1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26 error_codes,
27 manifest::{
28 validate_hello_capability_grammar, validate_hello_confirm_operations,
29 validate_hello_display_name, validate_hello_event_declarations,
30 validate_hello_flow_operations, validate_hello_operation_deferral,
31 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32 validate_hello_self_signal_declarations, validate_role_declarations_value,
33 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34 ProviderRole,
35 },
36 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
37 scope::{
38 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
39 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
40 SCOPE_SYNC_OP,
41 },
42 session::{
43 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
44 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
45 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
46 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
47 },
48 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
49 Priority, RouteTarget, PROTOCOL_VERSION,
50};
51use tokio::time::{timeout_at, Instant};
52use tracing::{debug, info, instrument::WithSubscriber, warn};
53
54use crate::{
55 caller_identity::CallerIdentity,
56 capability_requirements::{
57 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
58 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
59 RequirementStatus, RuntimeModule,
60 },
61 child_roster::DaemonShutdownFlag,
62 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
63 forwarding::{
64 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
65 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
66 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
67 },
68 observability::{
69 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
70 },
71 privileged_prompt::{prompt_text, PrivilegedOperation},
72 provenance::{
73 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
74 },
75 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
76 router::{daemon_error_flags, RouteCtx, RouterError},
77 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
78 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
79 stderr_tail::{CaptureState, TailEntry},
80 supervise::{
81 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
82 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
83 },
84 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
85};
86
87pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
93
94const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
95const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
96const CAP_PING_PONG: &str = "ping_pong_v1";
97const CAP_SESSION_ATTACH: &str = "session_attach_v1";
98const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
99
100const SUBC_CONTROL_OPS: &[&str] = &[
101 "claim.mint",
102 "claim.present",
103 ops::DAEMON_SHUTDOWN_INTENT,
104 ops::ACTIVITY_RECORD,
105 ops::ACTIVITY_QUERY,
106 ops::ACTIVITY_SUBSCRIBE,
107 ops::SERVER_DESCRIBE,
108 ops::CATALOG_LIST,
109 ops::ROUTE_OPEN,
110 ops::ROUTE_POLL,
111 ops::ROUTE_CLOSING,
112 ops::ROUTE_CLOSED,
113 ops::SUPERVISOR_LIST,
114 ops::SUPERVISOR_RESTART,
115 ops::SUPERVISOR_SWAP,
116 ops::SUPERVISOR_RELOAD,
117 ops::SUPERVISOR_RESCAN,
118 ops::SUPERVISOR_RELEASE_RESERVED,
119 ops::SUPERVISOR_SET_ENABLED,
120 ops::SUPERVISOR_HEALTH_PROBE,
121 ops::SUPERVISOR_HEALTH,
122 ops::SUPERVISOR_STDERR_TAIL,
123 ops::SUPERVISOR_TERMINALS,
124 ops::SUPERVISOR_ROUTES,
125 ops::SUPERVISOR_PROVENANCE,
126 ops::SUPERVISOR_SPAWN_SNAPSHOT,
127 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
128];
129
130const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[
137 ops::SUPERVISOR_RESCAN_CHECKED,
138 "config.status",
139 "config.plan",
140 "config.apply",
141 "config.stamp",
142 "config.unstamp",
143 "config.history",
144 "config.rollback",
145];
146
147mod channel_operations;
148#[cfg(all(test, unix, feature = "test-support"))]
149#[path = "control/channel_operations_tests.rs"]
150mod channel_operations_tests;
151pub(crate) mod claims;
152#[cfg(test)]
153#[path = "control/config_advertisement_tests.rs"]
154mod config_advertisement_tests;
155mod config_gate;
156mod scope_expiry;
157
158mod agent_commands;
159mod rescan_checked;
160use rescan_checked::{RescanMode, RescanPlan};
161
162const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
163 "claim.mint",
164 "claim.present",
165 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
166 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
167 "supervisor.live_roots",
168 "supervisor.agent_commands",
169 SCOPE_SYNC_OP,
170 SCOPE_APPLY_OP,
171 SCOPE_DESCRIBE_OP,
172 subc_protocol::scope::ROUTE_AUTHORIZE_OP,
173 "operator.confirm",
174];
175
176const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
180
181const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
182
183pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
191
192pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
207
208pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
226
227const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
228const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
229
230fn reload_verdict(
231 configured: &Path,
232 spawned_from: Option<&Path>,
233 image: subc_control::RunningImageAgreement,
234) -> PendingReloadVerdict {
235 let path = match spawned_from {
236 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
237 ReloadPathAgreement::Match
238 }
239 Some(spawned_from) => ReloadPathAgreement::Mismatch {
240 configured: configured.to_path_buf(),
241 spawned_from: spawned_from.to_path_buf(),
242 },
243 None => ReloadPathAgreement::Unavailable {
244 reason: if matches!(
245 image,
246 subc_control::RunningImageAgreement::Unavailable {
247 reason: subc_control::RunningImageUnavailableReason::NotRunning
248 }
249 ) {
250 ReloadPathUnavailableReason::NotRunning
251 } else {
252 ReloadPathUnavailableReason::SpawnedPathUnavailable
253 },
254 },
255 };
256 PendingReloadVerdict::new(path, image)
257}
258
259fn launch_spec_agreement(
260 configured: &crate::supervise::ModuleSpec,
261 running: bool,
262 spawned: Option<&crate::supervise::LaunchSpecDigest>,
263) -> subc_control::LaunchSpecAgreement {
264 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
265 if !running {
266 return Agreement::Unavailable {
267 reason: Reason::NotRunning,
268 };
269 }
270 match spawned {
271 None => Agreement::Unavailable {
272 reason: Reason::SpawnedSpecUnavailable,
273 },
274 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
275 Agreement::Match
276 }
277 Some(_) => Agreement::Mismatch,
278 }
279}
280
281fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
282 if configured == spawned_from {
283 return true;
284 }
285 #[cfg(windows)]
286 {
287 match (
293 std::fs::canonicalize(configured),
294 std::fs::canonicalize(spawned_from),
295 ) {
296 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
297 _ => false,
298 }
299 }
300 #[cfg(not(windows))]
301 {
302 false
303 }
304}
305
306#[derive(Clone)]
307struct DaemonProvenanceFacts {
308 build: DaemonBuildProvenance,
309 pid: Option<u32>,
310 started_at_ms: Option<u64>,
311 start_clock: Option<crate::clock::StartClock>,
312 executable_path: Option<PathBuf>,
313 executable_identity: Option<SpawnedFileIdentity>,
314 process_start_time: Option<u64>,
315 probe: ExecutableIdentityProbe,
316}
317
318impl Default for DaemonProvenanceFacts {
319 fn default() -> Self {
320 Self {
321 build: DaemonBuildProvenance {
322 build_git_sha: None,
323 build_lock_digest: None,
324 },
325 pid: None,
326 started_at_ms: None,
327 start_clock: None,
328 executable_path: None,
329 executable_identity: None,
330 process_start_time: None,
331 probe: ExecutableIdentityProbe::default(),
332 }
333 }
334}
335
336#[derive(Debug, Clone)]
337struct SupervisorRescanContext {
338 supervisor: Supervisor,
339 config_path: PathBuf,
340 configured_port: Option<u16>,
341 storage_config: Option<crate::daemon_config::StorageConfig>,
342 admission_facts_carrier_module_id: Option<String>,
343 admission_facts_targets: Option<Vec<String>>,
344 scope_authority_owners: Vec<String>,
345 federated_scope_owner: String,
346 privileged_operator_prompt: PrivilegedOperatorPrompt,
347 generation: Arc<std::sync::atomic::AtomicU64>,
353}
354
355const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
376 "reloading",
377 "supervisor_not_live",
378 "registration_not_active",
379 "no_forwarding_connection",
380 "relay_send_failed",
381];
382
383#[cfg(test)]
385struct ActivityRequestPause {
386 connection: ConnectionId,
387 started: tokio::sync::Notify,
388 cancelled: tokio::sync::Notify,
389 cleanup_started: std::sync::atomic::AtomicBool,
390 cancelled_after_cleanup: std::sync::atomic::AtomicBool,
391}
392
393#[cfg(test)]
394struct ActivityRequestCancellation(Arc<ActivityRequestPause>);
395
396#[cfg(test)]
397impl Drop for ActivityRequestCancellation {
398 fn drop(&mut self) {
399 self.0.cancelled_after_cleanup.store(
400 self.0
401 .cleanup_started
402 .load(std::sync::atomic::Ordering::Acquire),
403 std::sync::atomic::Ordering::Release,
404 );
405 self.0.cancelled.notify_one();
406 }
407}
408
409#[cfg(test)]
411type TestPause = (
412 tokio::sync::oneshot::Sender<()>,
413 std::sync::mpsc::Receiver<()>,
414);
415
416#[cfg(test)]
418type AsyncTestPause = (
419 tokio::sync::oneshot::Sender<()>,
420 tokio::sync::oneshot::Receiver<()>,
421);
422
423#[derive(Clone)]
425pub struct ControlHandler {
426 #[cfg(test)]
427 activity_request_pause: Arc<Mutex<Option<Arc<ActivityRequestPause>>>>,
428 #[cfg(test)]
432 cleanup_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
433 #[cfg(test)]
436 hello_ingress_pause: Arc<Mutex<Option<AsyncTestPause>>>,
437 #[cfg(test)]
439 agent_commands_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
440 claims: Arc<claims::Claims>,
441 registry: Arc<Registry>,
442 config_runtime: Option<Arc<crate::config_gate::runtime::RuntimeOwner>>,
443 config_review_admission: Arc<dyn crate::config_gate::ReviewAdmission>,
444 config_review_admission_is_default: bool,
445 activity: Option<crate::activity::ActivityLedger>,
446 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
447 forwarding: Arc<ForwardingTable>,
448 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
449 supervisor: SupervisorHandle,
450 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
451 agent_declarations: agent_commands::Queues,
452 list_admission: Arc<tokio::sync::Semaphore>,
453 subc_capabilities: Arc<[String]>,
454 route_bind_relay_timeout: Duration,
458 route_bind_relay_timeouts: BTreeMap<String, Duration>,
462 route_bind_breakers: RouteBindBreakers,
465 route_bind_concurrency: RouteBindConcurrency,
468 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
473 route_bind_breaker_threshold: u32,
475 route_bind_breaker_cooldown: Duration,
477 health_probe_timeout: Duration,
478 storage_config: Option<crate::daemon_config::StorageConfig>,
481 machine_id: Option<crate::machine_id::MachineId>,
485 admission_facts_carrier_module_id: Option<String>,
486 admission_facts_targets: Option<Vec<String>>,
487 scopes: Arc<RwLock<ScopeTable>>,
491 scope_deadline_added: Arc<tokio::sync::Notify>,
495 scope_authority_owners: Vec<String>,
498 federated_scope_owner: String,
499 privileged_operator_prompt: PrivilegedOperatorPrompt,
501 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
504 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
505 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
506 captured_peers: Arc<Mutex<HashMap<ConnectionId, subc_os::local_peer::CapturedPeer>>>,
507 config_desktop_stamps:
508 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
509 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
510 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
511 pub(crate) operator_apps:
512 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
513 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
518 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
524 #[cfg(test)]
526 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
527 #[cfg(test)]
528 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
529 #[cfg(test)]
530 recovery_exit_test_after_step: Option<fn(usize) -> Result<(), crate::config_gate::GateError>>,
531 #[cfg(test)]
532 recovery_exit_test_completed: Option<fn(u64)>,
533 #[cfg(test)]
534 apply_reconcile_pause: Arc<Mutex<Option<AsyncTestPause>>>,
535 #[cfg(test)]
536 recovery_exit_test_quiet_retry: bool,
537 #[cfg(test)]
538 recovery_exit_test_retry_pause: Option<fn(u64) -> Option<PathBuf>>,
539 #[cfg(test)]
540 config_delivery_test_wakes:
541 Option<tokio::sync::mpsc::UnboundedSender<(&'static str, tokio::time::Instant)>>,
542 #[cfg(all(test, unix))]
543 config_delivery_test_idle:
544 Option<tokio::sync::mpsc::UnboundedSender<tokio::sync::oneshot::Sender<()>>>,
545 #[allow(dead_code)]
549 #[cfg(windows)]
550 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
551 #[allow(dead_code)]
552 #[cfg(windows)]
553 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
554 #[allow(dead_code)]
558 #[cfg(windows)]
559 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
560 #[cfg(all(test, windows))]
563 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
564 #[cfg(test)]
565 caller_lookup: Option<TestCallerLookup>,
566 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
569 rescan: Option<SupervisorRescanContext>,
570 connected_clients: ConnectedClients,
571 counters: DaemonCounters,
572 local_endpoint_unavailable: Option<String>,
575 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
576 daemon_shutdown: DaemonShutdownFlag,
581 daemon_provenance: DaemonProvenanceFacts,
582 #[cfg(test)]
583 control_dispatch_delay: Option<Duration>,
584 #[cfg(test)]
585 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
586 #[cfg(test)]
590 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
591 #[cfg(test)]
597 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
598 #[cfg(test)]
599 hello_ack_to_publication_hold: Option<Arc<std::sync::Barrier>>,
600 #[cfg(test)]
601 hello_before_publication_hold: Option<Arc<std::sync::Barrier>>,
602 #[cfg(test)]
603 hello_after_launch_check_hold: Option<Arc<std::sync::Barrier>>,
604 #[cfg(test)]
605 hello_inside_publication_hold: Option<Arc<std::sync::Barrier>>,
606}
607
608impl fmt::Debug for ControlHandler {
609 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
610 f.debug_struct("ControlHandler")
611 .field("registry", &self.registry)
612 .field("forwarding", &self.forwarding)
613 .field("process_liveness", &self.process_liveness.is_some())
614 .field("supervisor", &self.supervisor)
615 .field("subc_capabilities", &self.subc_capabilities)
616 .finish()
617 }
618}
619
620struct RouteOpenRequest {
621 target: RouteTarget,
622 identity: BindIdentity,
623 consumer_identity: Option<ConsumerIdentity>,
624 consumer_capabilities: Option<Vec<String>>,
625 role_versions: Option<BTreeMap<String, String>>,
626 busy_refusals: Option<bool>,
627 admission_facts: Option<serde_json::Value>,
628 scope: Option<ScopeSelector>,
629}
630
631#[derive(Clone, Copy)]
632enum RouteOpenPhase {
633 Admission,
634 ProjectRoot,
635 EgressReservation,
636 BindSend,
637 BindAckWait,
638}
639
640struct RouteOpenTimings {
644 module_id: String,
645 principal: String,
646 admission: Duration,
647 project_root: Duration,
648 egress_reservation: Duration,
649 bind_send: Duration,
650 bind_ack_wait: Duration,
651 phase: Option<RouteOpenPhase>,
652 phase_started_at: Instant,
653}
654
655impl RouteOpenTimings {
656 fn new() -> Self {
657 Self {
658 module_id: String::new(),
659 principal: "unresolved".to_string(),
661 admission: Duration::ZERO,
662 project_root: Duration::ZERO,
663 egress_reservation: Duration::ZERO,
664 bind_send: Duration::ZERO,
665 bind_ack_wait: Duration::ZERO,
666 phase: Some(RouteOpenPhase::Admission),
667 phase_started_at: Instant::now(),
668 }
669 }
670
671 fn finish(&mut self) {
672 let elapsed = self.phase_started_at.elapsed();
673 match self.phase.take() {
674 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
675 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
676 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
677 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
678 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
679 None => {}
680 }
681 }
682
683 fn enter(&mut self, phase: RouteOpenPhase) {
684 self.finish();
685 self.phase = Some(phase);
686 self.phase_started_at = Instant::now();
687 }
688}
689
690struct RouteBindReservationGuard {
691 forwarding: Arc<ForwardingTable>,
692 endpoint: ModuleEndpointId,
693 relay_corr: u64,
694 armed: bool,
695}
696
697struct ModuleControlRpcGuard {
698 forwarding: Arc<ForwardingTable>,
699 endpoint: ModuleEndpointId,
700 corr: u64,
701 armed: bool,
702}
703
704impl ModuleControlRpcGuard {
705 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
706 Self {
707 forwarding,
708 endpoint,
709 corr,
710 armed: true,
711 }
712 }
713
714 fn disarm(&mut self) {
715 self.armed = false;
716 }
717}
718
719impl Drop for ModuleControlRpcGuard {
720 fn drop(&mut self) {
721 if self.armed {
722 let _ = self
723 .forwarding
724 .cancel_module_control_rpc(self.endpoint, self.corr);
725 }
726 }
727}
728
729impl RouteBindReservationGuard {
730 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
731 Self {
732 forwarding,
733 endpoint,
734 relay_corr,
735 armed: true,
736 }
737 }
738
739 fn release_and_disarm(&mut self) {
740 if !self.armed {
741 return;
742 }
743 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
744 self.endpoint,
745 self.relay_corr,
746 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
747 ) {
748 send_goodbye_target_best_effort(
749 &self.forwarding.counters(),
750 &target,
751 "canceled route.bind",
752 );
753 }
754 self.armed = false;
755 }
756
757 fn disarm(&mut self) {
758 self.armed = false;
759 }
760}
761
762impl Drop for RouteBindReservationGuard {
763 fn drop(&mut self) {
764 self.release_and_disarm();
765 }
766}
767
768#[derive(Debug, Clone, Default)]
798pub(crate) struct RouteBindBreakers {
799 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
800}
801
802#[derive(Debug, Clone, Default)]
803pub(crate) struct RouteBindConcurrency {
804 modules: Arc<Mutex<HashMap<String, usize>>>,
805}
806
807struct RouteBindConcurrencyGuard {
808 concurrency: RouteBindConcurrency,
809 module_id: String,
810}
811
812impl RouteBindConcurrency {
813 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
816 let mut modules = self
817 .modules
818 .lock()
819 .expect("route.bind concurrency mutex poisoned");
820 let in_flight = modules.entry(module_id.to_string()).or_default();
821 if *in_flight >= limit {
822 return Err(*in_flight);
823 }
824 *in_flight += 1;
825 Ok(RouteBindConcurrencyGuard {
826 concurrency: self.clone(),
827 module_id: module_id.to_string(),
828 })
829 }
830}
831
832impl Drop for RouteBindConcurrencyGuard {
833 fn drop(&mut self) {
834 let mut modules = self
835 .concurrency
836 .modules
837 .lock()
838 .expect("route.bind concurrency mutex poisoned");
839 let remove = {
840 let in_flight = modules
841 .get_mut(&self.module_id)
842 .expect("admitted route.bind has a concurrency entry");
843 *in_flight -= 1;
844 *in_flight == 0
845 };
846 if remove {
847 modules.remove(&self.module_id);
848 }
849 }
850}
851
852#[derive(Debug, Default)]
853struct ModuleBreakerState {
854 consecutive_timeouts: u32,
856 cooldown_until: Option<Instant>,
859 probe_in_flight: Option<Arc<()>>,
864}
865
866enum RouteBindAdmission<'a> {
868 Admitted {
869 guard: RouteBindBreakerGuard<'a>,
870 probe: bool,
873 },
874 Refused {
875 consecutive_timeouts: u32,
876 retry_in: Duration,
880 probe_in_flight: bool,
881 },
882}
883
884struct RouteBindBreakerGuard<'a> {
891 breakers: RouteBindBreakers,
892 module_id: &'a str,
893 probe_token: Option<Arc<()>>,
894 settled: bool,
895}
896
897impl RouteBindBreakerGuard<'_> {
898 fn record_accepted(&mut self) -> bool {
902 self.settled = true;
903 self.breakers.record_accepted(self.module_id)
904 }
905
906 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
909 self.settled = true;
910 self.breakers.record_timeout(
911 self.module_id,
912 self.probe_token.as_ref(),
913 threshold,
914 cooldown,
915 )
916 }
917
918 fn record_inconclusive(&mut self) {
927 self.settled = true;
928 self.breakers
929 .record_inconclusive(self.module_id, self.probe_token.as_ref());
930 }
931}
932
933impl Drop for RouteBindBreakerGuard<'_> {
934 fn drop(&mut self) {
935 if !self.settled {
936 self.breakers
937 .record_inconclusive(self.module_id, self.probe_token.as_ref());
938 }
939 }
940}
941
942struct BreakerOpened {
946 consecutive_timeouts: u32,
947 reopened_after_probe: bool,
950}
951
952impl RouteBindBreakers {
953 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
954 self.modules
955 .lock()
956 .expect("route.bind breaker mutex poisoned")
957 }
958
959 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
962 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
963 probe: probe_token.is_some(),
964 guard: RouteBindBreakerGuard {
965 breakers: self.clone(),
966 module_id,
967 probe_token,
968 settled: false,
969 },
970 };
971
972 let mut modules = self.lock();
973 let Some(state) = modules.get_mut(module_id) else {
974 return admitted(None);
975 };
976 let Some(cooldown_until) = state.cooldown_until else {
977 return admitted(None);
978 };
979 if state.probe_in_flight.is_some() {
980 return RouteBindAdmission::Refused {
981 consecutive_timeouts: state.consecutive_timeouts,
982 retry_in: Duration::ZERO,
983 probe_in_flight: true,
984 };
985 }
986 let now = Instant::now();
987 if now < cooldown_until {
988 return RouteBindAdmission::Refused {
989 consecutive_timeouts: state.consecutive_timeouts,
990 retry_in: cooldown_until - now,
991 probe_in_flight: false,
992 };
993 }
994 let token = Arc::new(());
995 state.probe_in_flight = Some(Arc::clone(&token));
996 admitted(Some(token))
997 }
998
999 fn record_accepted(&self, module_id: &str) -> bool {
1000 self.lock()
1001 .remove(module_id)
1002 .is_some_and(|state| state.cooldown_until.is_some())
1003 }
1004
1005 fn record_timeout(
1006 &self,
1007 module_id: &str,
1008 probe_token: Option<&Arc<()>>,
1009 threshold: u32,
1010 cooldown: Duration,
1011 ) -> Option<BreakerOpened> {
1012 let mut modules = self.lock();
1013 let state = modules.entry(module_id.to_string()).or_default();
1014 let was_open = state.cooldown_until.is_some();
1015 let was_probe = Self::owns_probe(state, probe_token);
1016 if was_probe {
1017 state.probe_in_flight = None;
1018 }
1019 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
1020 if state.consecutive_timeouts < threshold {
1021 return None;
1022 }
1023 state.cooldown_until = Some(Instant::now() + cooldown);
1024 Some(BreakerOpened {
1025 consecutive_timeouts: state.consecutive_timeouts,
1026 reopened_after_probe: was_open && was_probe,
1027 })
1028 }
1029
1030 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
1031 state
1035 .probe_in_flight
1036 .as_ref()
1037 .zip(token)
1038 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
1039 }
1040
1041 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
1042 if let Some(state) = self.lock().get_mut(module_id) {
1043 if Self::owns_probe(state, probe_token) {
1044 state.probe_in_flight = None;
1045 }
1046 }
1047 }
1048
1049 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
1064 self.lock()
1065 .remove(module_id)
1066 .map(|state| state.consecutive_timeouts)
1067 .filter(|discarded| *discarded > 0)
1068 }
1069
1070 fn open_snapshot(&self) -> Option<serde_json::Value> {
1077 let now = Instant::now();
1078 let modules = self.lock();
1079 let open = modules
1080 .iter()
1081 .filter_map(|(module_id, state)| {
1082 let cooldown_until = state.cooldown_until?;
1083 Some((
1084 module_id.clone(),
1085 serde_json::json!({
1086 "consecutive_timeouts": state.consecutive_timeouts,
1087 "cooldown_remaining_ms":
1088 cooldown_until.saturating_duration_since(now).as_millis() as u64,
1089 "probe_in_flight": state.probe_in_flight.is_some(),
1090 }),
1091 ))
1092 })
1093 .collect::<serde_json::Map<String, serde_json::Value>>();
1094 (!open.is_empty()).then_some(serde_json::Value::Object(open))
1095 }
1096}
1097
1098impl ControlHandler {
1099 pub(crate) fn install_registration_service(
1100 &self,
1101 router: std::sync::Weak<crate::Router>,
1102 auth: crate::server::ServerAuth,
1103 ) {
1104 self.supervisor.install_registration_service(router, auth);
1105 }
1106 pub fn new(registry: Arc<Registry>) -> Self {
1107 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
1108 }
1109
1110 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
1111 let counters = forwarding.counters();
1112 let route_bind_breakers = forwarding.route_bind_breakers();
1116 let route_bind_concurrency = forwarding.route_bind_concurrency();
1117 let route_outages = forwarding.route_outages();
1118 Self {
1119 registry,
1120 #[cfg(test)]
1121 activity_request_pause: Arc::new(Mutex::new(None)),
1122 #[cfg(test)]
1123 cleanup_pause: Arc::new(Mutex::new(None)),
1124 #[cfg(test)]
1125 hello_ingress_pause: Arc::new(Mutex::new(None)),
1126 #[cfg(test)]
1127 agent_commands_pause: Arc::new(Mutex::new(None)),
1128 claims: Arc::new(claims::Claims::default()),
1129 config_runtime: None,
1130 config_review_admission: Arc::new(crate::config_gate::MissingReviewAdmission),
1131 config_review_admission_is_default: true,
1132 activity: None,
1133 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1134 forwarding,
1135 process_liveness: None,
1136 supervisor: SupervisorHandle::new(),
1137 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1138 env!("CARGO_PKG_VERSION").to_owned(),
1139 )),
1140 agent_declarations: agent_commands::Queues::default(),
1141 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1142 subc_capabilities: Arc::from([
1143 CAP_MANIFEST_REGISTRATION.to_string(),
1144 CAP_CHANNEL_LIFECYCLE.to_string(),
1145 CAP_PING_PONG.to_string(),
1146 CAP_SESSION_ATTACH.to_string(),
1147 CAP_ADMISSION_FACTS_RELAY.to_string(),
1148 CAP_SCOPES_V1.to_string(),
1149 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1150 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1151 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1152 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1153 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1154 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1155 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1156 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1157 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1158 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1159 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1160 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1161 ]),
1162 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1163 route_bind_relay_timeouts: BTreeMap::new(),
1164 route_bind_breakers,
1165 route_bind_concurrency,
1166 route_outages,
1167 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1168 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1169 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1170 storage_config: None,
1171 machine_id: None,
1172 admission_facts_carrier_module_id: None,
1173 admission_facts_targets: None,
1174 scopes: Arc::new(RwLock::new(ScopeTable::new(
1175 crate::daemon_config::default_scope_authority_owners(),
1176 ))),
1177 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1178 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1179 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1180 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1181 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1182 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1183 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1184 captured_peers: Arc::new(Mutex::new(HashMap::new())),
1185 config_desktop_stamps: Arc::new(Mutex::new(HashMap::new())),
1186 local_connections: Arc::new(Mutex::new(HashSet::new())),
1187 operator_app_policy: Default::default(),
1188 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1189 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1190 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1191 #[cfg(test)]
1192 app_open_test_platform: None,
1193 #[cfg(test)]
1194 app_open_test_launch: None,
1195 #[cfg(test)]
1196 recovery_exit_test_after_step: None,
1197 #[cfg(test)]
1198 recovery_exit_test_completed: None,
1199 #[cfg(test)]
1200 apply_reconcile_pause: Arc::new(Mutex::new(None)),
1201 #[cfg(test)]
1202 recovery_exit_test_quiet_retry: false,
1203 #[cfg(test)]
1204 recovery_exit_test_retry_pause: None,
1205 #[cfg(test)]
1206 config_delivery_test_wakes: None,
1207 #[cfg(all(test, unix))]
1208 config_delivery_test_idle: None,
1209 #[cfg(windows)]
1210 windows_verify: None,
1211 #[cfg(windows)]
1212 windows_flash: None,
1213 #[cfg(windows)]
1214 windows_track: None,
1215 #[cfg(all(test, windows))]
1216 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1217 #[cfg(test)]
1218 caller_lookup: None,
1219 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1220 rescan: None,
1221 connected_clients: ConnectedClients::new(),
1222 local_endpoint_unavailable: None,
1223 counters,
1224 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1225 daemon_shutdown: DaemonShutdownFlag::default(),
1226 daemon_provenance: DaemonProvenanceFacts::default(),
1227 #[cfg(test)]
1228 control_dispatch_delay: None,
1229 #[cfg(test)]
1230 provenance_probe_override: None,
1231 #[cfg(test)]
1232 route_bind_relay_sent_hold: None,
1233 #[cfg(test)]
1234 hello_registry_to_forwarding_hold: None,
1235 #[cfg(test)]
1236 hello_ack_to_publication_hold: None,
1237 #[cfg(test)]
1238 hello_before_publication_hold: None,
1239 #[cfg(test)]
1240 hello_after_launch_check_hold: None,
1241 #[cfg(test)]
1242 hello_inside_publication_hold: None,
1243 }
1244 }
1245
1246 pub fn with_storage_config(
1249 mut self,
1250 storage_config: Option<crate::daemon_config::StorageConfig>,
1251 ) -> Self {
1252 self.storage_config = storage_config;
1253 self
1254 }
1255
1256 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1259 self.activity = Some(crate::activity::ActivityLedger::open(
1260 run_dir.join("activity.jsonl"),
1261 incarnation,
1262 ));
1263 self.supervisor.set_activity(self.activity.clone());
1264 self
1265 }
1266
1267 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1270 self.machine_id = machine_id;
1271 self
1272 }
1273
1274 pub fn with_admission_facts_config(
1278 mut self,
1279 carrier_module_id: Option<String>,
1280 targets: Option<Vec<String>>,
1281 ) -> Self {
1282 self.admission_facts_carrier_module_id = carrier_module_id;
1283 self.admission_facts_targets = targets;
1284 self
1285 }
1286
1287 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1291 self.scopes = Arc::new(RwLock::new(
1292 ScopeTable::new(owners.iter().cloned())
1293 .with_federated_owner(self.federated_scope_owner.clone()),
1294 ));
1295 self.scope_authority_owners = owners;
1296 self
1297 }
1298
1299 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1301 self.scopes = Arc::new(RwLock::new(
1302 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1303 .with_federated_owner(owner.clone()),
1304 ));
1305 self.federated_scope_owner = owner;
1306 self
1307 }
1308
1309 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1311 self.privileged_operator_prompt = mode;
1312 self
1313 }
1314
1315 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1318 self.route_bind_relay_timeout = timeout;
1319 self
1320 }
1321
1322 pub fn with_route_bind_relay_timeouts(
1328 mut self,
1329 timeouts: impl IntoIterator<Item = (String, Duration)>,
1330 ) -> Self {
1331 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1332 self
1333 }
1334
1335 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1341 self.route_bind_relay_timeouts
1342 .get(module_id)
1343 .copied()
1344 .unwrap_or(self.route_bind_relay_timeout)
1345 }
1346
1347 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1355 self.route_bind_breaker_threshold = threshold.max(1);
1356 self.route_bind_breaker_cooldown = cooldown;
1357 self
1358 }
1359
1360 #[cfg(test)]
1361 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1362 self.health_probe_timeout = timeout;
1363 self
1364 }
1365
1366 #[cfg(test)]
1367 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1368 self.control_dispatch_delay = Some(delay);
1369 self
1370 }
1371
1372 #[cfg(test)]
1376 pub(crate) fn with_route_bind_relay_sent_hold(
1377 mut self,
1378 hold: Arc<tokio::sync::Barrier>,
1379 ) -> Self {
1380 self.route_bind_relay_sent_hold = Some(hold);
1381 self
1382 }
1383
1384 #[cfg(test)]
1389 pub(crate) fn with_hello_registry_to_forwarding_hold(
1390 mut self,
1391 hold: Arc<std::sync::Barrier>,
1392 ) -> Self {
1393 self.hello_registry_to_forwarding_hold = Some(hold);
1394 self
1395 }
1396
1397 #[cfg(test)]
1398 fn hold_between_registry_and_forwarding(&self) {
1399 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1400 hold.wait();
1401 hold.wait();
1402 }
1403 }
1404
1405 #[cfg(test)]
1408 fn hold_between_ack_and_publication(&self) {
1409 if let Some(hold) = &self.hello_ack_to_publication_hold {
1410 hold.wait();
1411 hold.wait();
1412 }
1413 }
1414
1415 #[cfg(all(test, unix))]
1416 pub(crate) fn with_hello_before_publication_hold(
1417 mut self,
1418 hold: Arc<std::sync::Barrier>,
1419 ) -> Self {
1420 self.hello_before_publication_hold = Some(hold);
1421 self
1422 }
1423
1424 #[cfg(all(test, unix))]
1425 pub(crate) fn with_hello_after_launch_check_hold(
1426 mut self,
1427 hold: Arc<std::sync::Barrier>,
1428 ) -> Self {
1429 self.hello_after_launch_check_hold = Some(hold);
1430 self
1431 }
1432
1433 #[cfg(all(test, unix))]
1434 pub(crate) fn with_hello_inside_publication_hold(
1435 mut self,
1436 hold: Arc<std::sync::Barrier>,
1437 ) -> Self {
1438 self.hello_inside_publication_hold = Some(hold);
1439 self
1440 }
1441
1442 pub fn with_process_liveness(
1443 mut self,
1444 process_liveness: Arc<dyn ModuleProcessLiveness>,
1445 ) -> Self {
1446 self.process_liveness = Some(process_liveness);
1447 self
1448 }
1449
1450 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1451 self.supervisor = supervisor;
1452 if let Some(owner) = &self.config_runtime {
1453 self.supervisor.install_binding_retirement(owner);
1454 }
1455 self.supervisor.set_activity(self.activity.clone());
1456 self
1457 }
1458
1459 pub fn with_daemon_provenance(
1460 mut self,
1461 pid: u32,
1462 started_at_ms: u64,
1463 executable_path: Option<PathBuf>,
1464 build_git_sha: Option<String>,
1465 build_lock_digest: Option<String>,
1466 ) -> Self {
1467 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1468 let process_start_time = process_start_time(pid);
1469 self.daemon_provenance = DaemonProvenanceFacts {
1470 build: DaemonBuildProvenance {
1471 build_git_sha,
1472 build_lock_digest,
1473 },
1474 pid: Some(pid),
1475 started_at_ms: Some(started_at_ms),
1476 start_clock: None,
1477 executable_path,
1478 executable_identity,
1479 process_start_time,
1480 probe: ExecutableIdentityProbe::default(),
1481 };
1482 self
1483 }
1484
1485 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1486 self.daemon_provenance.start_clock = Some(clock);
1487 self
1488 }
1489
1490 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1492 self.scopes
1493 .write()
1494 .unwrap_or_else(|poisoned| poisoned.into_inner())
1495 .set_wall_clock(Arc::new(clock));
1496 self
1497 }
1498
1499 #[cfg(test)]
1500 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1501 self.provenance_probe_override = Some(result);
1502 self
1503 }
1504
1505 pub fn with_capability_config(
1509 self,
1510 modules: impl IntoIterator<Item = (String, bool)>,
1511 reserved_capabilities: BTreeMap<String, String>,
1512 ) -> Self {
1513 self.capability_evaluator
1514 .configure(modules, reserved_capabilities);
1515 self
1516 }
1517
1518 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1522 self.daemon_shutdown = flag;
1523 self
1524 }
1525
1526 pub fn with_supervisor_rescan(
1527 mut self,
1528 supervisor: Supervisor,
1529 config_path: impl Into<PathBuf>,
1530 configured_port: Option<u16>,
1531 ) -> Self {
1532 self.rescan = Some(SupervisorRescanContext {
1533 supervisor,
1534 config_path: config_path.into(),
1535 configured_port,
1536 storage_config: self.storage_config.clone(),
1537 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1538 admission_facts_targets: self.admission_facts_targets.clone(),
1539 scope_authority_owners: self.scope_authority_owners.clone(),
1540 federated_scope_owner: self.federated_scope_owner.clone(),
1541 privileged_operator_prompt: self.privileged_operator_prompt,
1542 generation: Arc::default(),
1543 });
1544 self
1545 }
1546
1547 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1548 self.connected_clients = connected_clients;
1549 self
1550 }
1551
1552 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1555 self.local_endpoint_unavailable = reason;
1556 self
1557 }
1558
1559 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1560 Arc::clone(&self.forwarding)
1561 }
1562
1563 pub(crate) fn counters(&self) -> DaemonCounters {
1564 self.counters.clone()
1565 }
1566
1567 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1570 let handler = Arc::downgrade(&self);
1571 let evaluator = self.capability_evaluator.clone();
1572 tokio::spawn(async move {
1573 loop {
1574 evaluator.wait_for_change_or_deadline().await;
1575 let Some(handler) = handler.upgrade() else {
1576 break;
1577 };
1578 handler.refresh_capability_requirements();
1579 }
1580 });
1581 }
1582
1583 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1589 let context = scope_expiry::ScopeExpiryContext::new(&self);
1590 let deadline_added = self.scope_deadline_added.clone();
1591 tokio::spawn(async move {
1592 loop {
1593 if !context.has_deadlines() {
1594 deadline_added.notified().await;
1595 continue;
1596 }
1597 tokio::time::sleep(Duration::from_secs(1)).await;
1598 if let Err(error) = context.sweep() {
1599 warn!(%error, "scope expiry sweep failed");
1600 }
1601 }
1602 });
1603 }
1604
1605 #[cfg(test)]
1606 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1607 scope_expiry::ScopeExpiryContext::new(self).sweep()
1608 }
1609
1610 fn publish_scope_changes(
1611 &self,
1612 changes: &[crate::scopes::ScopeTagChange],
1613 expired: &[crate::scopes::ScopeExpired],
1614 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1615 scope_expiry::ScopeExpiryContext::new(self).publish_scope_changes(changes, expired)
1616 }
1617
1618 fn runtime_capability_snapshot(
1619 &self,
1620 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1621 let runtime = self
1622 .supervisor
1623 .list()
1624 .into_iter()
1625 .map(|module| {
1626 let status = module.status().map_err(|err| {
1627 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1628 })?;
1629 Ok(RuntimeModule {
1630 module_id: status.module_id,
1631 state: status.state,
1632 enabled: status.enabled,
1633 })
1634 })
1635 .collect::<Result<Vec<_>, RouterError>>()?;
1636 let registrations = self.registry.list_registered_modules().map_err(|err| {
1642 RouterError::backend(
1643 0,
1644 0,
1645 format!("failed to list capability registrations: {err}"),
1646 )
1647 })?;
1648 let registrations = registrations
1649 .into_iter()
1650 .map(|registration| RegisteredModule {
1651 module_id: registration.manifest.module_id,
1652 module_version: registration.manifest.module_version,
1653 capabilities: registration.manifest.capabilities,
1654 })
1655 .collect();
1656 Ok((runtime, registrations))
1657 }
1658
1659 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1666 let cached_registration = RegisteredModule {
1667 module_id: registration.manifest.module_id.clone(),
1668 module_version: registration.manifest.module_version.clone(),
1669 capabilities: registration.manifest.capabilities.clone(),
1670 };
1671 let change = self.capability_evaluator.record_hello(&cached_registration);
1672 log_manifest_change(&cached_registration.module_id, &change);
1673 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1674 self.enforce_capability_denies();
1675 }
1676 self.refresh_capability_requirements();
1677 }
1678
1679 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1683 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1684 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1685 self.supervisor.set_swap_promotion_observer(observer);
1686 }
1687
1688 pub fn refresh_capability_requirements(&self) {
1689 match self.runtime_capability_snapshot() {
1690 Ok((runtime, registrations)) => {
1691 log_requirement_events(
1692 self.capability_evaluator
1693 .evaluate_now(&runtime, ®istrations),
1694 &self.daemon_shutdown,
1695 );
1696 }
1697 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1698 }
1699 }
1700
1701 fn enforce_capability_denies(&self) {
1705 let registrations = match self.registry.list_registered_modules() {
1706 Ok(snapshot) => snapshot,
1707 Err(err) => {
1708 warn!(error = %err, "failed to read registrations for capability deny census");
1709 return;
1710 }
1711 };
1712 let manifests = registrations
1713 .into_iter()
1714 .map(|registration| {
1715 (
1716 registration.manifest.module_id.clone(),
1717 registration.manifest,
1718 )
1719 })
1720 .collect::<BTreeMap<_, _>>();
1721 let census = match self.forwarding.route_census(None) {
1722 Ok(census) => census,
1723 Err(err) => {
1724 warn!(error = %err, "failed to read route census for capability deny enforcement");
1725 return;
1726 }
1727 };
1728
1729 for (target_module_id, routes) in census {
1730 let Some(target_manifest) = manifests.get(&target_module_id) else {
1731 continue;
1732 };
1733 let mut closed_routes = Vec::new();
1734 let mut goodbyes = Vec::new();
1735 for route in routes {
1736 let Principal::Reserved {
1737 module_id: opening_module_id,
1738 } = &route.principal
1739 else {
1740 continue;
1741 };
1742 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1743 continue;
1744 };
1745 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1746 continue;
1747 };
1748
1749 match self.forwarding.release_client_route(
1750 route.goodbye_target.connection_id,
1751 route.goodbye_target.channel,
1752 route.goodbye_target.epoch,
1753 ) {
1754 Ok(RouteRelease::Removed(module_goodbye)) => {
1755 warn!(
1756 opening_module_id,
1757 target_module_id,
1758 capability,
1759 "force-closing route because an attested capability deny edge now matches"
1760 );
1761 goodbyes.push(module_goodbye);
1762 goodbyes.push(route.goodbye_target.clone());
1763 closed_routes.push(route);
1764 }
1765 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1766 Err(err) => warn!(
1767 opening_module_id,
1768 target_module_id,
1769 capability,
1770 error = %err,
1771 "failed to force-close capability-denied route"
1772 ),
1773 }
1774 }
1775
1776 if closed_routes.is_empty() {
1777 continue;
1778 }
1779 send_route_control_pushes(
1780 &self.forwarding,
1781 closed_routes,
1782 ClientControlPush::RouteClosed {
1783 module_id: target_module_id,
1784 channels: Vec::new(),
1785 reason: RouteCloseReason::CapabilityDenied,
1786 drained: false,
1787 abandoned: 0,
1788 excluded_subscriptions: 0,
1789 terminal: Some(false),
1790 },
1791 );
1792 self.emit_route_goodbyes(goodbyes);
1795 }
1796 }
1797
1798 fn not_ready_reason(
1804 &self,
1805 registration: &crate::registry::ModuleRegistration,
1806 ) -> Option<NotReadyReason> {
1807 if !registration.ready {
1808 return Some(NotReadyReason {
1809 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1810 capability: None,
1811 });
1812 }
1813 self.first_unprovided_required_capability(registration)
1814 .map(|capability| NotReadyReason {
1815 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1816 capability: Some(capability),
1817 })
1818 }
1819
1820 fn first_unprovided_required_capability(
1838 &self,
1839 registration: &crate::registry::ModuleRegistration,
1840 ) -> Option<String> {
1841 let required = registration
1842 .manifest
1843 .capabilities
1844 .iter()
1845 .flat_map(|declarations| declarations.requires.iter())
1846 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1847 .map(|requirement| requirement.capability.as_str())
1848 .collect::<BTreeSet<_>>();
1849 if required.is_empty() {
1850 return None;
1851 }
1852 let module_id = registration.manifest.module_id.as_str();
1853 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1854 if required
1855 .iter()
1856 .any(|capability| verdict(capability).is_none())
1857 {
1858 self.refresh_capability_requirements();
1859 }
1860 required
1861 .into_iter()
1862 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1863 .map(str::to_string)
1864 }
1865
1866 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1867 self.capability_evaluator
1868 .statuses()
1869 .into_iter()
1870 .map(capability_requirement_status)
1871 .collect()
1872 }
1873
1874 fn deregister_connection(
1881 &self,
1882 connection_id: ConnectionId,
1883 reason: RegistrationEndReason,
1884 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1885 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
1886 spawn.end_primary_registration();
1887 }
1888 self.registry
1889 .deregister_connection_with_reason(connection_id, reason)
1890 }
1891
1892 fn rollback_unpublished_hello(
1897 &self,
1898 connection: ConnectionId,
1899 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1900 self.registry.deregister_connection_with_reason(
1901 connection,
1902 RegistrationEndReason::RegistrationFailed,
1903 )
1904 }
1905
1906 fn request_refusal_close(
1907 &self,
1908 connection: ConnectionId,
1909 sink: Option<&crate::FrameSink>,
1910 reason: CloseReason,
1911 ) {
1912 if let Some(sink) = sink {
1913 sink.terminate_ingress();
1914 }
1915 if self.claims.close_notifications_enabled() {
1916 self.forwarding.request_connection_close(connection, reason);
1917 }
1918 }
1919
1920 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1921 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1922 return None;
1923 }
1924 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1925 parse_client_control_request(&frame.body)
1926 else {
1927 return None;
1928 };
1929 Some(target_module_id(&target).to_string())
1930 }
1931
1932 pub(crate) fn route_open_capacity_refusal(
1933 &self,
1934 ctx: &RouteCtx,
1935 frame: &Frame,
1936 target_module_id: &str,
1937 in_flight: usize,
1938 limit: usize,
1939 ) -> Result<Frame, RouterError> {
1940 let caller = self.route_open_caller_from_frame(ctx.connection_id, frame);
1943 self.route_open_admission_refusal_frame(
1944 ctx,
1945 &caller,
1946 frame,
1947 target_module_id,
1948 "open_admission_full",
1949 (in_flight, limit),
1950 format!(
1951 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1952 ),
1953 )
1954 }
1955
1956 fn route_open_target_capacity_refusal(
1957 &self,
1958 ctx: &RouteCtx,
1959 caller: &RouteOpenCaller,
1960 frame: &Frame,
1961 target_module_id: &str,
1962 in_flight: usize,
1963 ) -> Result<Frame, RouterError> {
1964 self.route_open_admission_refusal_frame(
1965 ctx,
1966 caller,
1967 frame,
1968 target_module_id,
1969 "target_binds_full",
1970 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1971 format!(
1972 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1973 ),
1974 )
1975 }
1976
1977 #[allow(clippy::too_many_arguments)]
1985 fn route_open_admission_refusal_frame(
1986 &self,
1987 ctx: &RouteCtx,
1988 caller: &RouteOpenCaller,
1989 frame: &Frame,
1990 target_module_id: &str,
1991 reason: &'static str,
1992 (in_flight, limit): (usize, usize),
1993 message: impl Into<String>,
1994 ) -> Result<Frame, RouterError> {
1995 let code = error_codes::TARGET_UNAVAILABLE;
1996 self.counters.increment_route_open_refused(code);
1997 info!(
1998 target: "control",
1999 code,
2000 reason,
2001 module_id = ?target_module_id,
2002 connection_id = ctx.connection_id.get(),
2003 principal = %caller.principal,
2004 scope = %caller.scope,
2005 harness = %caller.harness,
2006 in_flight,
2007 limit,
2008 "route.open refused"
2009 );
2010 control_error_frame(frame, code, message.into())
2011 }
2012
2013 #[cfg(test)]
2019 pub fn handle_control(
2020 &self,
2021 connection_id: ConnectionId,
2022 frame: Frame,
2023 ) -> Result<Vec<Frame>, RouterError> {
2024 match frame.header.ty {
2025 FrameType::Ping => Ok(vec![pong(&frame)?]),
2026 FrameType::Hello => self.handle_hello(connection_id, None, frame),
2027 FrameType::Goodbye => self.handle_goodbye(connection_id),
2028 ty => Ok(vec![control_error_frame(
2029 &frame,
2030 "unsupported_control_frame",
2031 format!("unsupported channel-0 frame {ty:?}"),
2032 )?]),
2033 }
2034 }
2035
2036 pub async fn handle_control_frame(
2037 &self,
2038 ctx: &RouteCtx,
2039 frame: Frame,
2040 ) -> Result<Vec<Frame>, RouterError> {
2041 self.handle_control_frame_timed(ctx, frame, None).await
2042 }
2043
2044 #[cfg(any(test, feature = "test-support"))]
2045 pub fn with_test_operator_app_policy(
2046 mut self,
2047 policy: crate::operator_apps::TestOperatorAppPolicy,
2048 ) -> Self {
2049 self.operator_app_policy = policy.into();
2050 self
2051 }
2052
2053 #[cfg(all(windows, any(test, feature = "test-support")))]
2057 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
2058 self.windows_verify = Some(verify);
2059 self
2060 }
2061
2062 #[cfg(all(windows, any(test, feature = "test-support")))]
2064 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
2065 self.windows_flash = Some(flash);
2066 self
2067 }
2068
2069 #[cfg(all(windows, any(test, feature = "test-support")))]
2073 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
2074 self.windows_track = Some(track);
2075 self
2076 }
2077
2078 pub(crate) fn record_captured_peer(
2079 &self,
2080 connection: ConnectionId,
2081 peer: subc_os::local_peer::CapturedPeer,
2082 ) {
2083 self.record_observed_peer(connection, peer.observation().clone());
2084 let identity = self.operator_app_policy.config_identity(peer.clone());
2085 #[cfg(windows)]
2086 let identity = identity.or_else(|| self.config_windows_launch_identity(&peer));
2087 if let Some(identity) = identity {
2088 self.config_desktop_stamps
2091 .lock()
2092 .unwrap_or_else(|p| p.into_inner())
2093 .insert(connection, identity);
2094 }
2095 self.captured_peers
2096 .lock()
2097 .unwrap_or_else(|p| p.into_inner())
2098 .insert(connection, peer.clone());
2099 #[cfg(windows)]
2100 if let Some(stamp) = self.stamp_windows_launch(&peer) {
2101 self.operator_apps
2102 .lock()
2103 .unwrap_or_else(|p| p.into_inner())
2104 .insert(connection, stamp);
2105 }
2106 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
2107 self.operator_apps
2108 .lock()
2109 .unwrap_or_else(|p| p.into_inner())
2110 .insert(connection, stamp);
2111 }
2112 }
2113
2114 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
2115 self.local_connections
2116 .lock()
2117 .unwrap_or_else(|p| p.into_inner())
2118 .insert(connection);
2119 }
2120
2121 fn connection_operator_app(
2124 &self,
2125 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
2126 ) -> Option<subc_protocol::peer::OperatorApp> {
2127 stamp
2128 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
2129 .map(|stamp| stamp.app.clone())
2130 }
2131
2132 fn handle_connection_describe(
2133 &self,
2134 ctx: &RouteCtx,
2135 frame: Frame,
2136 ) -> Result<Vec<Frame>, RouterError> {
2137 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
2138 return Ok(vec![control_error_frame(
2139 &frame,
2140 "invalid_control_body",
2141 "malformed connection.describe body",
2142 )?]);
2143 }
2144 let local = self
2145 .local_connections
2146 .lock()
2147 .unwrap_or_else(|p| p.into_inner())
2148 .contains(&ctx.connection_id);
2149 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
2150 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
2151 Ok(vec![control_response_body_frame(
2152 &frame,
2153 &subc_control::ConnectionDescribeReply::new(local, operator_app),
2154 ops::CONNECTION_DESCRIBE,
2155 )?])
2156 }
2157
2158 pub(crate) fn record_observed_peer(
2159 &self,
2160 connection: ConnectionId,
2161 peer: subc_os::local_peer::ObservedPeer,
2162 ) {
2163 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
2164 let peer = match peer {
2165 OsPeer::Unobserved => ObservedPeer::unobserved(),
2166 OsPeer::Observed(peer) => {
2167 let observed = match peer.user {
2168 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
2169 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
2170 };
2171 observed
2172 .with_exe(
2173 peer.exe
2174 .and_then(|path| path.into_os_string().into_string().ok()),
2175 )
2176 .with_signing_id(peer.signing_id)
2177 }
2178 };
2179 self.observed_peers
2180 .lock()
2181 .unwrap_or_else(|p| p.into_inner())
2182 .insert(connection, peer);
2183 }
2184
2185 pub(crate) fn record_caller_endpoints(
2186 &self,
2187 connection: ConnectionId,
2188 endpoints: Option<(SocketAddr, SocketAddr)>,
2189 ) {
2190 let mut callers = self
2191 .caller_endpoints
2192 .lock()
2193 .unwrap_or_else(|p| p.into_inner());
2194 callers.remove(&connection);
2195 if let Some(endpoints) = endpoints {
2196 callers.insert(connection, endpoints);
2197 }
2198 }
2199
2200 #[cfg(test)]
2201 pub(crate) fn with_caller_lookup_for_test(
2202 mut self,
2203 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2204 ) -> Self {
2205 self.caller_lookup = Some(Arc::new(lookup));
2206 self
2207 }
2208
2209 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2210 let endpoints = self
2211 .caller_endpoints
2212 .lock()
2213 .unwrap_or_else(|p| p.into_inner())
2214 .get(&connection)
2215 .copied();
2216 #[cfg(test)]
2217 let lookup = self.caller_lookup.clone();
2218 tokio::task::spawn_blocking(move || {
2219 #[cfg(test)]
2220 if let Some(lookup) = lookup {
2221 return lookup(endpoints);
2222 }
2223 endpoints
2224 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2225 .unwrap_or_default()
2226 })
2227 .await
2228 .unwrap_or_default()
2229 }
2230
2231 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2232 frame.header.channel == 0
2233 && frame.header.ty == FrameType::Request
2234 && self
2235 .forwarding
2236 .module_endpoint_for_connection(ctx.connection_id)
2237 .is_ok_and(|endpoint| endpoint.is_none())
2238 && parse_client_control_request(&frame.body)
2239 .ok()
2240 .is_some_and(|request| privileged_operation(&request).is_some())
2241 }
2242
2243 pub(crate) async fn handle_control_frame_timed(
2244 &self,
2245 ctx: &RouteCtx,
2246 frame: Frame,
2247 dispatch_started_at: Option<StdInstant>,
2248 ) -> Result<Vec<Frame>, RouterError> {
2249 let first = ctx.egress.first_ingress_frame();
2250 self.handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
2251 .await
2252 }
2253
2254 pub(crate) async fn handle_control_frame_ingress(
2255 &self,
2256 ctx: &RouteCtx,
2257 frame: Frame,
2258 dispatch_started_at: Option<StdInstant>,
2259 first: bool,
2260 ) -> Result<Vec<Frame>, RouterError> {
2261 if ctx.egress.ingress_terminal() {
2262 return Ok(Vec::new());
2263 }
2264 #[cfg(test)]
2265 self.claims.record_dispatch(frame.header.corr);
2266 let claim_binding = self.claims.binding(ctx.connection_id);
2267 let mut prefix_attempt = (frame.header.ty == FrameType::Hello
2268 && claim_binding
2269 .as_ref()
2270 .is_some_and(|binding| binding.prefix_module_id().is_some()))
2271 .then(|| self.prefix_hello_attempt(ctx));
2272 if claim_binding
2273 .as_ref()
2274 .is_some_and(|binding| binding.prefix_module_id().is_some())
2275 {
2276 let registered = self
2277 .forwarding
2278 .module_endpoint_for_connection(ctx.connection_id)
2279 .map_err(RouterError::Forwarding)?
2280 .is_some();
2281 if !registered && frame.header.ty != FrameType::Hello {
2282 return self
2283 .refuse_claim_connection(ctx, &frame, "claim_hello_required")
2284 .await;
2285 }
2286 if registered && frame.header.ty == FrameType::Goodbye {
2287 ctx.egress.terminate_ingress();
2288 let result = self.handle_goodbye(ctx.connection_id);
2289 self.claims.connection_closed(ctx.connection_id);
2290 self.request_refusal_close(
2291 ctx.connection_id,
2292 Some(&ctx.egress),
2293 CloseReason::new("claim_registration_ended", "prefix registration ended"),
2294 );
2295 return result;
2296 }
2297 if registered && frame.header.ty == FrameType::Request {
2298 let allowed =
2299 serde_json::from_slice::<ControlOpProbe>(&frame.body).is_ok_and(|probe| {
2300 matches!(
2301 probe.op.as_str(),
2302 "catalog.update"
2303 | "catalog.update_role_declarations"
2304 | "scope.describe"
2305 | "supervisor.live_roots"
2306 | "activity.record"
2307 )
2308 });
2309 if !allowed {
2310 return Ok(vec![control_error_frame(
2311 &frame,
2312 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2313 "prefix registrations have only brokered module operations",
2314 )?]);
2315 }
2316 }
2317 if registered
2318 && frame.header.ty == FrameType::Push
2319 && !serde_json::from_slice::<ControlOpProbe>(&frame.body)
2320 .is_ok_and(|probe| probe.op == "route.status")
2321 {
2322 return Ok(vec![control_error_frame(
2323 &frame,
2324 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2325 "prefix registrations may push only route.status for routes they serve",
2326 )?]);
2327 }
2328 }
2329 match frame.header.ty {
2330 FrameType::Ping => Ok(vec![pong(&frame)?]),
2331 FrameType::Hello => {
2332 if claim_binding
2333 .as_ref()
2334 .is_some_and(|binding| binding.prefix_module_id().is_none())
2335 {
2336 return self
2337 .refuse_claim_connection(ctx, &frame, "claim_hello_not_allowed")
2338 .await;
2339 }
2340 let channel_spawn = self.supervisor.registration_spawn(ctx.connection_id);
2344 if let Some(spawn) = &channel_spawn {
2345 spawn.wait_launch().await;
2346 if !spawn.launch_ready() {
2349 ctx.egress.terminate_ingress();
2350 return Ok(Vec::new());
2351 }
2352 #[cfg(test)]
2353 self.pause_hello_ingress_for_test().await;
2354 } else if claim_binding.is_none() {
2355 if let Ok(hello) = serde_json::from_slice::<serde_json::Value>(&frame.body) {
2356 if let (Some(module), Some(nonce)) = (
2357 hello
2358 .get("manifest")
2359 .and_then(|value| value.get("module_id"))
2360 .and_then(|value| value.as_str()),
2361 hello.get("launch_nonce").and_then(|value| value.as_str()),
2362 ) {
2363 if !self
2364 .config_runtime
2365 .as_ref()
2366 .is_some_and(|owner| owner.module_is_gated(module))
2367 && !self.supervisor.wait_program_binding(module, nonce).await
2368 {
2369 self.request_refusal_close(
2370 ctx.connection_id,
2371 Some(&ctx.egress),
2372 CloseReason::new(
2373 "program_refused",
2374 "the spawned program binding failed",
2375 ),
2376 );
2377 return Ok(vec![control_error_frame(
2378 &frame,
2379 if self
2380 .config_runtime
2381 .as_ref()
2382 .is_some_and(|owner| owner.is_recovering())
2383 {
2384 "config_gated"
2385 } else {
2386 "program_refused"
2387 },
2388 "the spawned program binding failed",
2389 )?]);
2390 }
2391 }
2392 }
2393 }
2394 let handler = self.clone();
2395 let connection = ctx.connection_id;
2396 let egress = ctx.egress.clone();
2397 let correlation = frame.header.corr;
2398 let dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
2399 let span = tracing::Span::current();
2400 let hello_binding = claim_binding.clone();
2401 let replies = tokio::task::spawn_blocking(move || {
2402 tracing::dispatcher::with_default(&dispatcher, || {
2403 span.in_scope(|| {
2404 handler.handle_hello_checked(
2405 connection,
2406 Some(egress),
2407 frame,
2408 true,
2409 hello_binding,
2410 channel_spawn,
2411 )
2412 })
2413 })
2414 })
2415 .await
2416 .map_err(|error| RouterError::backend(0, correlation, error.to_string()))??;
2417 #[cfg(test)]
2418 if prefix_attempt.is_some() {
2419 self.claims.pause_prefix_commit_for_test(false).await;
2420 }
2421 if let Some(attempt) = &mut prefix_attempt {
2422 let accepted = !replies
2423 .iter()
2424 .any(|reply| reply.header.ty == FrameType::Error)
2425 && self
2426 .registry
2427 .get_module_by_connection(ctx.connection_id)
2428 .map_err(|err| RouterError::backend(0, correlation, err.to_string()))?
2429 .is_some();
2430 if accepted {
2431 attempt.accepted = true;
2432 } else {
2433 ctx.egress.terminate_ingress();
2434 for reply in replies {
2435 ctx.egress.send_flushed(reply).await?;
2436 }
2437 return Ok(Vec::new());
2438 }
2439 }
2440 Ok(replies)
2441 }
2442 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2443 FrameType::Cancel => {
2444 if self
2450 .agent_declarations
2451 .cancel(ctx.connection_id, frame.header.corr)
2452 || self
2453 .forwarding
2454 .operator_confirms()
2455 .cancel(ctx.connection_id, frame.header.corr)
2456 || self.cancel_config_approval(ctx.connection_id, frame.header.corr)
2457 || self
2458 .supervisor
2459 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2460 || self.cancel_activity_subscription(ctx, &frame)?
2461 {
2462 Ok(Vec::new())
2463 } else {
2464 Ok(vec![control_error_frame(
2465 &frame,
2466 "unknown_subscription",
2467 "no subscription has this correlation id",
2468 )?])
2469 }
2470 }
2471 FrameType::Request => {
2472 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2475 let claim_present = op.as_ref().is_some_and(|probe| probe.op == "claim.present");
2476 if let Some(refusal) = self
2477 .recovery_request_refusal(
2478 ctx,
2479 &frame,
2480 op.as_ref().map(|probe| probe.op.as_str()),
2481 )
2482 .await?
2483 {
2484 if claim_present {
2485 self.reclaim_presented_expiry(&frame);
2486 return self
2487 .refuse_claim_connection(ctx, &frame, "config_gated")
2488 .await;
2489 }
2490 return Ok(vec![refusal]);
2491 }
2492 if claim_present {
2493 return self.handle_claim_present(ctx, &frame, first).await;
2494 }
2495 if op.as_ref().is_some_and(|probe| probe.op == "claim.mint") {
2496 return self.handle_claim_mint(ctx, &frame).await;
2497 }
2498 if let Some(refusal) = self.registration_operation_refusal(
2499 ctx,
2500 &frame,
2501 op.as_ref().map(|probe| probe.op.as_str()),
2502 )? {
2503 return Ok(vec![refusal]);
2504 }
2505 if let Some(probe) = op
2506 .as_ref()
2507 .filter(|probe| config_gate::is_module_config_op(&probe.op))
2508 {
2509 return self
2510 .handle_module_config_request(ctx, frame, &probe.op)
2511 .await;
2512 }
2513 if op
2514 .as_ref()
2515 .is_some_and(|probe| config_gate::is_config_op(&probe.op))
2516 {
2517 return self.handle_config_gate_request(ctx, frame).await;
2518 }
2519 if op
2520 .as_ref()
2521 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2522 {
2523 return self.handle_activity_record(ctx, frame).await;
2524 }
2525 if op.as_ref().is_some_and(|probe| {
2526 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2527 }) {
2528 return self.handle_activity_read(ctx, frame).await;
2529 }
2530 if op
2531 .as_ref()
2532 .is_some_and(|probe| probe.op == "operator.confirm")
2533 {
2534 return self.handle_operator_confirm(ctx, frame);
2535 }
2536 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2537 return self
2538 .observe_control_wait(
2539 ctx,
2540 None,
2541 ops::APP_OPEN,
2542 self.handle_app_open(ctx, frame),
2543 )
2544 .await;
2545 }
2546 if op
2547 .as_ref()
2548 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2549 {
2550 return self.handle_connection_describe(ctx, frame);
2551 }
2552 if op
2553 .as_ref()
2554 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2555 {
2556 return self.handle_scope_served_by(ctx.connection_id, frame);
2557 }
2558 if self
2559 .forwarding
2560 .module_endpoint_for_connection(ctx.connection_id)
2561 .map_err(RouterError::Forwarding)?
2562 .is_some()
2563 {
2564 if !is_known_module_request_op(&frame.body) {
2565 return Ok(vec![control_error_frame(
2566 &frame,
2567 "unsupported_control_frame",
2568 "module-originated channel-0 REQUEST is not supported",
2569 )?]);
2570 }
2571 let request = match parse_module_control_request_from_module(&frame.body) {
2572 Ok(request) => request,
2573 Err((err, ControlRequestBodyError::UnknownOp)) => {
2574 return Ok(vec![control_error_frame(
2575 &frame,
2576 "unsupported_control_frame",
2577 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2578 )?])
2579 }
2580 Err((err, ControlRequestBodyError::InvalidBody)) => {
2581 return Ok(vec![control_error_frame(
2582 &frame,
2583 "invalid_control_body",
2584 format!("malformed module control body: {err}"),
2585 )?])
2586 }
2587 };
2588 let op = module_control_request_op(&request);
2589 let corr = frame.header.corr;
2590 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2591 if let ModuleControlRequestFromModule::AgentCommands {
2592 started,
2593 finished,
2594 known,
2595 } = request
2596 {
2597 return self
2598 .agent_declarations
2599 .enqueue(
2600 self,
2601 ctx,
2602 frame,
2603 crate::resource_totals::Declaration {
2604 started,
2605 finished,
2606 known,
2607 },
2608 )
2609 .await;
2610 }
2611 let result =
2612 self.handle_module_control_request(ctx.connection_id, frame, request);
2613 log_slow_control_dispatch(
2614 dispatch_started_at,
2615 op,
2616 ctx.connection_id,
2617 corr,
2618 None,
2619 );
2620 return result;
2621 }
2622
2623 if is_known_module_request_op(&frame.body) {
2624 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2625 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2626 let code = if is_agents {
2627 "not_supervised"
2628 } else if op
2629 .as_ref()
2630 .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2631 {
2632 error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2633 } else {
2634 "not_registered"
2635 };
2636 return Ok(vec![control_error_frame(
2637 &frame,
2638 code,
2639 "module-originated control requests require the module's own registered connection",
2640 )?]);
2641 }
2642
2643 let request = match parse_client_control_request(&frame.body) {
2644 Ok(request) => request,
2645 Err((err, ControlRequestBodyError::UnknownOp)) => {
2646 return Ok(vec![control_error_frame(
2647 &frame,
2648 "unknown_control_op",
2649 format!("unknown client control op: {err}"),
2650 )?])
2651 }
2652 Err((err, ControlRequestBodyError::InvalidBody)) => {
2653 return Ok(vec![control_error_frame(
2654 &frame,
2655 "invalid_control_body",
2656 format!("malformed client control body: {err}"),
2657 )?])
2658 }
2659 };
2660 let op = client_control_request_op(&request);
2661 let corr = frame.header.corr;
2662 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2663 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2664 #[cfg(test)]
2665 if let Some(delay) = self.control_dispatch_delay {
2666 tokio::time::sleep(delay).await;
2667 }
2668 let result = self
2669 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2670 .await;
2671 if let Some(timings) = route_open_timings.as_mut() {
2672 timings.finish();
2673 }
2674 log_slow_control_dispatch(
2675 dispatch_started_at,
2676 op,
2677 ctx.connection_id,
2678 corr,
2679 route_open_timings.as_ref(),
2680 );
2681 result
2682 }
2683 FrameType::Push => {
2684 let Some(endpoint) = self
2685 .forwarding
2686 .module_endpoint_for_connection(ctx.connection_id)
2687 .map_err(RouterError::Forwarding)?
2688 else {
2689 return Ok(vec![control_error_frame(
2690 &frame,
2691 "unsupported_control_frame",
2692 "client-originated channel-0 PUSH is not supported",
2693 )?]);
2694 };
2695 self.handle_status_update(endpoint, frame)
2696 }
2697 FrameType::Response | FrameType::Error
2698 if self
2699 .forwarding
2700 .module_endpoint_for_connection(ctx.connection_id)
2701 .map_err(RouterError::Forwarding)?
2702 .is_some() =>
2703 {
2704 self.handle_module_relay_response(ctx.connection_id, frame)
2705 }
2706 ty => Ok(vec![control_error_frame(
2707 &frame,
2708 "unsupported_control_frame",
2709 format!("unsupported channel-0 frame {ty:?}"),
2710 )?]),
2711 }
2712 }
2713
2714 #[cfg(test)]
2715 fn pause_cleanup_for_test(&self, stage: u8) {
2716 let mut slot = self.cleanup_pause.lock().unwrap_or_else(|p| p.into_inner());
2717 if slot.as_ref().is_some_and(|(at, _)| *at == stage) {
2718 let (_, (reached, release)) = slot.take().unwrap();
2719 drop(slot);
2720 let _ = reached.send(());
2721 let _ = release.recv_timeout(Duration::from_secs(5));
2722 }
2723 }
2724
2725 #[cfg(test)]
2726 async fn pause_hello_ingress_for_test(&self) {
2727 let pause = self
2728 .hello_ingress_pause
2729 .lock()
2730 .unwrap_or_else(|p| p.into_inner())
2731 .take();
2732 if let Some((reached, release)) = pause {
2733 let _ = reached.send(());
2734 let _ = release.await;
2735 }
2736 }
2737
2738 pub fn cleanup_connection(
2739 &self,
2740 connection_id: ConnectionId,
2741 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2742 self.agent_declarations.close(connection_id);
2756 #[cfg(test)]
2757 self.pause_cleanup_for_test(0);
2758 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
2759 spawn.retire();
2760 }
2761 self.supervisor
2762 .forget_registration_connection(connection_id);
2763 #[cfg(test)]
2764 self.pause_cleanup_for_test(1);
2765 self.local_connections
2766 .lock()
2767 .unwrap_or_else(|p| p.into_inner())
2768 .remove(&connection_id);
2769 self.caller_endpoints
2770 .lock()
2771 .unwrap_or_else(|p| p.into_inner())
2772 .remove(&connection_id);
2773 self.build_class_warned
2774 .lock()
2775 .unwrap_or_else(|p| p.into_inner())
2776 .remove(&connection_id);
2777 self.observed_peers
2778 .lock()
2779 .unwrap_or_else(|p| p.into_inner())
2780 .remove(&connection_id);
2781 self.captured_peers
2782 .lock()
2783 .unwrap_or_else(|p| p.into_inner())
2784 .remove(&connection_id);
2785 self.config_desktop_stamps
2786 .lock()
2787 .unwrap_or_else(|p| p.into_inner())
2788 .remove(&connection_id);
2789 self.operator_apps
2790 .lock()
2791 .unwrap_or_else(|p| p.into_inner())
2792 .remove(&connection_id);
2793 self.forwarding
2794 .operator_confirms()
2795 .connection_closed(connection_id);
2796 let cleanup = self.cleanup_connection_with_end_reason(
2797 connection_id,
2798 RegistrationEndReason::ConnectionClosed,
2799 );
2800 self.claims.connection_closed(connection_id);
2803 cleanup
2804 }
2805
2806 fn cleanup_connection_with_end_reason(
2807 &self,
2808 connection_id: ConnectionId,
2809 requested_reason: RegistrationEndReason,
2810 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2811 #[cfg(test)]
2812 if let Some(pause) = self
2813 .activity_request_pause
2814 .lock()
2815 .unwrap()
2816 .as_ref()
2817 .filter(|pause| pause.connection == connection_id)
2818 {
2819 pause
2820 .cleanup_started
2821 .store(true, std::sync::atomic::Ordering::Release);
2822 }
2823 if let Some(runtime) = &self.config_runtime {
2824 runtime.disconnected(connection_id);
2825 }
2826 self.agent_declarations.close(connection_id);
2827 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2828 self.registry
2829 .get_module_by_connection(connection_id)?
2830 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2831 .and_then(|module| module.registration_end_reason().ok().flatten())
2832 .unwrap_or(requested_reason)
2833 } else {
2834 requested_reason
2835 };
2836 let crash_closed = self
2837 .registry
2838 .get_module_by_connection(connection_id)?
2839 .and_then(|registration| {
2840 self.forwarding
2841 .module_endpoint_for_connection(connection_id)
2842 .ok()
2843 .flatten()
2844 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2845 .map(|routes| (registration.manifest.module_id, routes))
2846 });
2847 let crash_closed = crash_closed.map(|(module_id, routes)| {
2848 let terminal = match self.supervisor.get(&module_id) {
2849 None => false,
2850 Some(module) => match module.will_recover_after_connection_loss() {
2851 Ok(will_recover) => !will_recover,
2852 Err(err) => {
2853 warn!(
2854 %module_id,
2855 error = %err,
2856 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2857 );
2858 false
2859 }
2860 },
2861 };
2862 let (reason, drained) = match self
2866 .forwarding
2867 .daemon_drain_result_for_connection(connection_id)
2868 {
2869 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2870 Ok(None) => (RouteCloseReason::Crash, false),
2871 Err(err) => {
2872 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2873 (RouteCloseReason::Crash, false)
2874 }
2875 };
2876 (module_id, routes, reason, drained, terminal)
2877 });
2878 let registrations = self.deregister_connection(connection_id, end_reason);
2879 let cleanup = if crash_closed.is_some() {
2880 self.forwarding.cleanup_connection_counted(connection_id)
2881 } else {
2882 self.forwarding
2888 .cleanup_connection(connection_id)
2889 .map(|released| crate::forwarding::ConnectionCleanup {
2890 released,
2891 abandoned_relays: 0,
2892 })
2893 };
2894 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2899 let abandoned = cleanup
2900 .as_ref()
2901 .map_or(0, |cleanup| cleanup.abandoned_relays);
2902 send_route_control_pushes(
2903 &self.forwarding,
2904 routes,
2905 ClientControlPush::RouteClosed {
2906 module_id,
2907 channels: Vec::new(),
2908 reason,
2909 drained,
2910 abandoned,
2911 excluded_subscriptions: 0,
2912 terminal: Some(terminal),
2913 },
2914 );
2915 }
2916 if let Ok(cleanup) = cleanup {
2917 self.emit_route_goodbyes(cleanup.released);
2918 }
2919 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2923 crate::supervise::notify_registration_release();
2924 self.capability_evaluator.wake_deadline_loop();
2925 self.refresh_capability_requirements();
2926 }
2927 self.supervisor.remove_spawn_subscribers(connection_id);
2928 self.activity_streams
2929 .lock()
2930 .unwrap_or_else(|p| p.into_inner())
2931 .retain(|(id, _), task| {
2932 if *id == connection_id {
2933 task.abort();
2934 false
2935 } else {
2936 true
2937 }
2938 });
2939 self.hello_launch_nonces
2942 .lock()
2943 .unwrap_or_else(|poisoned| poisoned.into_inner())
2944 .forget(connection_id);
2945 self.scopes
2946 .write()
2947 .unwrap_or_else(|poisoned| poisoned.into_inner())
2948 .release_connection(connection_id);
2949 registrations
2950 }
2951
2952 pub(crate) fn handle_route_goodbye(
2953 &self,
2954 connection_id: ConnectionId,
2955 route_channel: u16,
2956 route_epoch: u32,
2957 ) -> Result<bool, RouterError> {
2958 debug!(
2959 connection_id = connection_id.get(),
2960 route_channel, route_epoch, "handling route GOODBYE"
2961 );
2962 let RouteRelease::Removed(released_route) = self
2963 .forwarding
2964 .release_client_route(connection_id, route_channel, route_epoch)
2965 .map_err(RouterError::Forwarding)?
2966 else {
2967 return Ok(false);
2968 };
2969 self.emit_route_goodbyes(vec![released_route]);
2970 Ok(true)
2971 }
2972
2973 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2974 scope_expiry::ScopeExpiryContext::new(self).emit_route_goodbyes(released_routes);
2975 }
2976
2977 fn send_abandoned_route_bind_goodbye(
2989 &self,
2990 module_sink: &crate::FrameSink,
2991 negotiated_ver: u8,
2992 module_channel: u16,
2993 module_epoch: u32,
2994 ) {
2995 let frame = match Frame::build_with_version(
2996 negotiated_ver,
2997 FrameType::Goodbye,
2998 control_flags(),
2999 module_channel,
3000 module_epoch,
3001 0,
3002 Vec::new(),
3003 ) {
3004 Ok(frame) => frame,
3005 Err(err) => {
3006 warn!(
3007 route_channel = module_channel,
3008 error = %err,
3009 "failed to build GOODBYE for abandoned route.bind"
3010 );
3011 return;
3012 }
3013 };
3014 crate::forwarding::send_module_route_goodbye(
3015 &self.counters,
3016 module_sink,
3017 frame,
3018 None,
3019 "abandoned route.bind",
3020 );
3021 }
3022
3023 #[cfg(test)]
3024 fn handle_hello(
3025 &self,
3026 connection_id: ConnectionId,
3027 egress: Option<crate::router::FrameSink>,
3028 frame: Frame,
3029 ) -> Result<Vec<Frame>, RouterError> {
3030 let binding = self.claims.binding(connection_id);
3031 let channel_spawn = self.supervisor.registration_spawn(connection_id);
3032 self.handle_hello_checked(connection_id, egress, frame, false, binding, channel_spawn)
3033 }
3034
3035 fn handle_hello_checked(
3036 &self,
3037 connection_id: ConnectionId,
3038 sink: Option<crate::FrameSink>,
3039 frame: Frame,
3040 wait_for_gate: bool,
3041 prefix_binding: Option<claims::ClaimBinding>,
3042 channel_spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3043 ) -> Result<Vec<Frame>, RouterError> {
3044 let primary_sink = channel_spawn.as_ref().and_then(|_| sink.clone());
3045 #[cfg(test)]
3046 if prefix_binding.is_some() {
3047 self.claims.pause_prefix_hello_for_test(5);
3048 match self.claims.take_prefix_hello_failure_for_test() {
3049 Some("error") => {
3050 return Err(RouterError::backend(
3051 0,
3052 frame.header.corr,
3053 "injected HELLO failure",
3054 ))
3055 }
3056 Some("panic") => panic!("injected HELLO task panic"),
3057 _ => {}
3058 }
3059 }
3060 if let Some(binding) = &prefix_binding {
3061 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3062 let module = value
3063 .as_ref()
3064 .and_then(|value| value["manifest"]["module_id"].as_str());
3065 if !binding.admit_prefix_hello(module)
3066 || !self.prefix_binding_live(binding, connection_id)
3067 || !module.is_some_and(|module| {
3068 self.supervisor
3069 .prefix_claim_authorized(&binding.registration.module_id, module)
3070 })
3071 || binding.registration.check().is_err()
3072 {
3073 return Ok(vec![control_error_frame(
3074 &frame,
3075 "claim_prefix_refused",
3076 "HELLO must name the prefix claim's exact module id once",
3077 )?]);
3078 }
3079 }
3080 struct RefuseUnregisteredChannel {
3081 spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3082 sink: Option<crate::FrameSink>,
3083 }
3084 impl Drop for RefuseUnregisteredChannel {
3085 fn drop(&mut self) {
3086 if let Some(spawn) = &self.spawn {
3087 if !spawn.registered() {
3088 if let Some(sink) = &self.sink {
3089 sink.terminate_ingress();
3090 }
3091 spawn.retire();
3092 }
3093 }
3094 }
3095 }
3096 let _channel_attempt = RefuseUnregisteredChannel {
3097 spawn: channel_spawn.clone(),
3098 sink: sink.clone(),
3099 };
3100 if let Some(spawn) = &channel_spawn {
3101 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3102 if let Err(code) = spawn.admit_hello(
3103 value
3104 .as_ref()
3105 .and_then(|value| value["manifest"]["module_id"].as_str()),
3106 ) {
3107 return Ok(vec![control_error_frame(
3108 &frame,
3109 code,
3110 "registration channel HELLO refused",
3111 )?]);
3112 }
3113 }
3114 debug!(
3115 connection_id = connection_id.get(),
3116 corr = frame.header.corr,
3117 "handling HELLO"
3118 );
3119 if self
3123 .registry
3124 .get_module_by_connection(connection_id)
3125 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3126 .is_some()
3127 {
3128 return Ok(vec![control_error_frame(
3129 &frame,
3130 "invalid_hello",
3131 "connection is already registered as a module",
3132 )?]);
3133 }
3134 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
3135 Ok(value) => value,
3136 Err(err) => {
3137 return Ok(vec![control_error_frame(
3138 &frame,
3139 "invalid_hello",
3140 format!("malformed HELLO body: {err}"),
3141 )?])
3142 }
3143 };
3144 if let Some(module) = hello_value
3145 .get("manifest")
3146 .and_then(|value| value.get("module_id"))
3147 .and_then(|value| value.as_str())
3148 {
3149 let recovering = self
3150 .config_runtime
3151 .as_ref()
3152 .is_some_and(|owner| owner.check_recovery().unwrap_or(true));
3153 if recovering
3157 && channel_spawn.as_ref().is_none_or(|spawn| {
3158 !self.supervisor.has_live_verified_channel_binding(spawn)
3159 || spawn.check().is_err()
3160 })
3161 && !(channel_spawn.is_none()
3162 && hello_value
3163 .get("launch_nonce")
3164 .and_then(|value| value.as_str())
3165 .is_some_and(|nonce| {
3166 self.supervisor.spawned_consumer_authorized(module, nonce)
3167 && self
3168 .supervisor
3169 .has_live_verified_program_binding(module, nonce)
3170 }))
3171 {
3172 return Ok(vec![control_error_frame(
3173 &frame,
3174 "config_gated",
3175 "recovery admits only previously bound supervised launches",
3176 )?]);
3177 }
3178 if self
3179 .config_runtime
3180 .as_ref()
3181 .is_some_and(|owner| owner.module_is_gated(module))
3182 && (channel_spawn
3183 .as_ref()
3184 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3185 || self.config_runtime.as_ref().is_some_and(|owner| {
3186 !recovering && owner.check_module_hold(module).is_err()
3187 }))
3188 {
3189 return Ok(vec![control_error_frame(
3190 &frame,
3191 "reserved_module",
3192 "a gated identity belongs only to its supervised child",
3193 )?]);
3194 }
3195 }
3196 if channel_spawn.is_none() {
3197 if let (Some(module), Some(nonce)) = (
3198 hello_value
3199 .get("manifest")
3200 .and_then(|value| value.get("module_id"))
3201 .and_then(|value| value.as_str()),
3202 hello_value
3203 .get("launch_nonce")
3204 .and_then(|value| value.as_str()),
3205 ) {
3206 let binding = self.supervisor.has_program_binding_record(module, nonce);
3207 let required = self
3208 .config_runtime
3209 .as_ref()
3210 .is_some_and(|owner| owner.module_is_gated(module));
3211 if let Some(error) = (binding || required)
3212 .then(|| {
3213 self.supervisor
3214 .check_current_program_binding(module, nonce)
3215 .err()
3216 })
3217 .flatten()
3218 {
3219 let code = if error.code == "program_refused" {
3220 "program_refused"
3221 } else {
3222 "config_delivery_deferred"
3223 };
3224 self.request_refusal_close(
3225 connection_id,
3226 sink.as_ref(),
3227 CloseReason::new(code, error.detail.clone()),
3228 );
3229 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3230 }
3231 }
3232 }
3233 if let Err(err) = validate_hello_display_name(&hello_value) {
3234 return Ok(vec![control_error_frame(
3235 &frame,
3236 "invalid_manifest",
3237 err.to_string(),
3238 )?]);
3239 }
3240 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
3241 return Ok(vec![control_error_frame(
3242 &frame,
3243 "invalid_capability_grammar",
3244 err.to_string(),
3245 )?]);
3246 }
3247 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
3248 return Ok(vec![control_error_frame(
3249 &frame,
3250 "invalid_manifest",
3251 err.to_string(),
3252 )?]);
3253 }
3254 if let Err(err) = validate_hello_event_declarations(&hello_value) {
3255 return Ok(vec![control_error_frame(
3256 &frame,
3257 "invalid_event_declaration",
3258 err.to_string(),
3259 )?]);
3260 }
3261 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
3262 return Ok(vec![control_error_frame(
3263 &frame,
3264 "invalid_manifest",
3265 err.to_string(),
3266 )?]);
3267 }
3268 if let Some(provenance) = hello_value
3269 .get("manifest")
3270 .and_then(|manifest| manifest.get("provenance"))
3271 {
3272 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
3273 return Ok(vec![control_error_frame(
3274 &frame,
3275 "invalid_manifest",
3276 format!("malformed manifest provenance: {err}"),
3277 )?]);
3278 }
3279 }
3280 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
3281 let module_id = hello_value["manifest"]["module_id"]
3282 .as_str()
3283 .unwrap_or("<unknown>");
3284 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
3285 }
3286 if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
3287 let module_id = hello_value
3288 .get("manifest")
3289 .and_then(|m| m.get("module_id"))
3290 .and_then(serde_json::Value::as_str)
3291 .unwrap_or("unknown");
3292 warn!(module_id, %reason, "dropping operation_deferral from HELLO");
3293 }
3294 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
3295 let module_id = hello_value["manifest"]["module_id"]
3296 .as_str()
3297 .unwrap_or("<unknown>");
3298 warn!(module_id, %reason, "dropping flow_operations from HELLO");
3299 }
3300 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
3301 let module_id = hello_value["manifest"]["module_id"]
3302 .as_str()
3303 .unwrap_or("<unknown>");
3304 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
3305 }
3306 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
3307 Ok(hello) => hello,
3308 Err(err) => {
3309 return Ok(vec![control_error_frame(
3310 &frame,
3311 "invalid_hello",
3312 format!("malformed HELLO body: {err}"),
3313 )?])
3314 }
3315 };
3316
3317 if hello.protocol_ver != hello.manifest.protocol_ver {
3318 return Ok(vec![control_error_frame(
3319 &frame,
3320 "invalid_manifest",
3321 format!(
3322 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
3323 hello.protocol_ver, hello.manifest.protocol_ver
3324 ),
3325 )?]);
3326 }
3327
3328 if hello.manifest.module_id.trim().is_empty() {
3329 return Ok(vec![control_error_frame(
3330 &frame,
3331 "invalid_manifest",
3332 "manifest module_id must not be empty",
3333 )?]);
3334 }
3335
3336 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
3337 Ok(negotiated_ver) => negotiated_ver,
3338 Err(message) => {
3339 return Ok(vec![control_error_frame(
3340 &frame,
3341 "version_unsupported",
3342 message,
3343 )?])
3344 }
3345 };
3346
3347 let _publication_gate = if let Some(owner) = &self.config_runtime {
3353 let gate = owner
3354 .module_gate(&hello.manifest.module_id)
3355 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3356 Some(if wait_for_gate {
3357 gate.blocking_lock_owned()
3358 } else {
3359 match gate.try_lock_owned() {
3360 Ok(guard) => guard,
3361 Err(_) => {
3362 return Ok(vec![control_error_frame(
3363 &frame,
3364 "config_delivery_deferred",
3365 "module admission is busy",
3366 )?])
3367 }
3368 }
3369 })
3370 } else {
3371 None
3372 };
3373 #[cfg(test)]
3374 if let Some(hold) = &self.hello_before_publication_hold {
3375 hold.wait();
3376 hold.wait();
3377 }
3378 let module = &hello.manifest.module_id;
3379 let required = self
3380 .config_runtime
3381 .as_ref()
3382 .is_some_and(|owner| owner.module_is_gated(module));
3383 let recovering = self
3384 .config_runtime
3385 .as_ref()
3386 .is_some_and(|owner| owner.is_recovering());
3387 if recovering
3395 && channel_spawn.as_ref().is_none_or(|spawn| {
3396 !self.supervisor.has_live_verified_channel_binding(spawn) || spawn.check().is_err()
3397 })
3398 && !(channel_spawn.is_none()
3399 && hello.launch_nonce.as_deref().is_some_and(|nonce| {
3400 self.supervisor.spawned_consumer_authorized(module, nonce)
3401 && self
3402 .supervisor
3403 .has_live_verified_program_binding(module, nonce)
3404 }))
3405 {
3406 return Ok(vec![control_error_frame(
3407 &frame,
3408 "config_gated",
3409 "recovery admits only previously bound supervised launches",
3410 )?]);
3411 }
3412 if required
3413 && (channel_spawn
3414 .as_ref()
3415 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3416 || self
3417 .config_runtime
3418 .as_ref()
3419 .is_some_and(|owner| !recovering && owner.check_module_hold(module).is_err()))
3420 {
3421 return Ok(vec![control_error_frame(
3422 &frame,
3423 "reserved_module",
3424 "gated admission requires the current supervised child's registration channel and verified image binding",
3425 )?]);
3426 }
3427 if let Some(spawn) = &channel_spawn {
3428 if let Err(code) = spawn.check() {
3429 return Ok(vec![control_error_frame(
3430 &frame,
3431 code,
3432 "registration launch is no longer live",
3433 )?]);
3434 }
3435 } else if let Some(nonce) = hello.launch_nonce.as_deref() {
3436 if required || self.supervisor.has_program_binding_record(module, nonce) {
3437 if let Err(error) = self.supervisor.check_current_program_binding(module, nonce) {
3438 let code = if error.code == "program_refused" {
3439 "program_refused"
3440 } else {
3441 "config_delivery_deferred"
3442 };
3443 self.request_refusal_close(
3444 connection_id,
3445 sink.as_ref(),
3446 CloseReason::new(code, error.detail.clone()),
3447 );
3448 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3449 }
3450 }
3451 }
3452
3453 let swap_admission = match &channel_spawn {
3461 Some(spawn) if spawn.is_candidate() => SwapHelloAdmission::Candidate,
3462 Some(_) => SwapHelloAdmission::NotSwapping,
3463 None if prefix_binding.is_some() => SwapHelloAdmission::NotSwapping,
3464 None => self
3465 .supervisor
3466 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref()),
3467 };
3468 if swap_admission == SwapHelloAdmission::ProtocolNone {
3469 warn!(
3470 module_id = %hello.manifest.module_id,
3471 connection_id = connection_id.get(),
3472 "HELLO refused: the module_id belongs to a protocol:none module, which never registers"
3473 );
3474 return Ok(vec![control_error_frame(
3477 &frame,
3478 "swap_token_invalid",
3479 format!(
3480 "module_id '{}' is a protocol:none module; a protocol:none module may not register",
3481 hello.manifest.module_id
3482 ),
3483 )?]);
3484 }
3485 if swap_admission == SwapHelloAdmission::ChannelRequired {
3486 warn!(
3487 module_id = %hello.manifest.module_id,
3488 connection_id = connection_id.get(),
3489 "HELLO refused: the module_id is gated and the HELLO did not arrive over its registration channel"
3490 );
3491 return Ok(vec![control_error_frame(
3494 &frame,
3495 "swap_token_invalid",
3496 format!(
3497 "module_id '{}' is gated; it must register over its supervised registration channel",
3498 hello.manifest.module_id
3499 ),
3500 )?]);
3501 }
3502 if swap_admission == SwapHelloAdmission::Refused {
3503 warn!(
3504 module_id = %hello.manifest.module_id,
3505 connection_id = connection_id.get(),
3506 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
3507 );
3508 return Ok(vec![control_error_frame(
3509 &frame,
3510 "swap_token_invalid",
3511 format!(
3512 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
3513 hello.manifest.module_id
3514 ),
3515 )?]);
3516 }
3517 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
3518
3519 let verification = if swap_candidate && channel_spawn.is_none() {
3524 ReservedHelloVerification {
3525 nonce_verified: true,
3526 prefix_owner: None,
3527 }
3528 } else {
3529 let verification = if let Some(binding) = &prefix_binding {
3530 if !self.supervisor.prefix_claim_authorized(
3531 &binding.registration.module_id,
3532 &hello.manifest.module_id,
3533 ) {
3534 return Ok(vec![control_error_frame(
3535 &frame,
3536 "claim_prefix_refused",
3537 "prefix ownership changed",
3538 )?]);
3539 }
3540 Ok(ReservedHelloVerification {
3541 nonce_verified: true,
3542 prefix_owner: Some(binding.registration.module_id.clone()),
3543 })
3544 } else if channel_spawn.is_some() {
3545 self.supervisor
3546 .channel_hello_verification(&hello.manifest.module_id)
3547 } else {
3548 self.supervisor.reserved_hello_verification(
3549 &hello.manifest.module_id,
3550 hello.launch_nonce.as_deref(),
3551 )
3552 };
3553 match verification {
3554 Ok(verified) => verified,
3555 Err(rejection) => {
3556 let message = match rejection {
3557 ReservedHelloRejection::Exact { module_id } => format!(
3558 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
3559 ),
3560 ReservedHelloRejection::Prefix {
3561 prefix,
3562 owner_module_id,
3563 } => format!(
3564 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
3565 hello.manifest.module_id
3566 ),
3567 };
3568 return Ok(vec![control_error_frame(
3569 &frame,
3570 "reserved_module",
3571 message,
3572 )?]);
3573 }
3574 }
3575 };
3576
3577 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
3578 &hello.manifest.module_id,
3579 hello.manifest.capabilities.as_ref(),
3580 );
3581 if let Some(refusal) = reserved_capability_refusals.first() {
3582 let capability = refusal.capability.clone();
3583 let bound_module = refusal.claimants[0].clone();
3584 log_duplicate_claim_events(reserved_capability_refusals);
3585 return Ok(vec![control_error_frame(
3586 &frame,
3587 "reserved_capability",
3588 format!(
3589 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3590 capability, bound_module, hello.manifest.module_id
3591 ),
3592 )?]);
3593 }
3594
3595 if self
3598 .forwarding
3599 .connection_has_client_routes(connection_id)
3600 .map_err(RouterError::Forwarding)?
3601 {
3602 return Ok(vec![control_error_frame(
3603 &frame,
3604 "invalid_hello",
3605 "connection has open client routes and cannot also register as a module",
3606 )?]);
3607 }
3608
3609 self.hello_launch_nonces
3617 .lock()
3618 .unwrap_or_else(|poisoned| poisoned.into_inner())
3619 .record(
3620 connection_id,
3621 if prefix_binding.is_some() || channel_spawn.is_some() {
3622 None
3623 } else {
3624 hello.launch_nonce.as_deref()
3625 },
3626 );
3627 let control_ops = effective_module_control_ops(hello.control_ops);
3628 let prefix_submodule = verification.prefix_owner.is_some();
3629 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
3632 #[cfg(test)]
3633 struct PublicationFinished(Arc<std::sync::Barrier>);
3634 #[cfg(test)]
3635 impl Drop for PublicationFinished {
3636 fn drop(&mut self) {
3637 self.0.wait();
3638 }
3639 }
3640 #[cfg(test)]
3641 let _publication_finished = self.hello_after_launch_check_hold.as_ref().map(|hold| {
3642 hold.wait();
3643 hold.wait();
3644 PublicationFinished(hold.clone())
3645 });
3646 let publication_spawn = channel_spawn
3647 .as_ref()
3648 .or_else(|| prefix_binding.as_ref().map(|binding| &binding.registration));
3649 #[cfg(test)]
3650 if prefix_binding.is_some() {
3651 self.claims.pause_prefix_hello_for_test(7);
3652 }
3653 let _registration_publication = match publication_spawn {
3654 Some(spawn) => Some(match spawn.begin_publication() {
3655 Ok(guard) => guard,
3656 Err(code) => {
3657 return Ok(vec![control_error_frame(
3658 &frame,
3659 code,
3660 "registration ended before HELLO publication",
3661 )?])
3662 }
3663 }),
3664 None => None,
3665 };
3666 let (_prefix_live, _prefix_ownership) = if let Some(binding) = &prefix_binding {
3667 let live = self.claims.lock_prefix_binding(binding, connection_id);
3668 let ownership = self.supervisor.prefix_claim_publication(
3669 &binding.registration.module_id,
3670 &hello.manifest.module_id,
3671 );
3672 if live.is_none()
3673 || ownership.is_none()
3674 || !self.claim_primary_registered(binding.spawn, &binding.registration)
3675 || binding.prefix_module_id() != Some(hello.manifest.module_id.as_str())
3676 {
3677 return Ok(vec![control_error_frame(
3678 &frame,
3679 "claim_prefix_refused",
3680 "prefix authority ended before publication",
3681 )?]);
3682 }
3683 (live, ownership)
3684 } else {
3685 (None, None)
3686 };
3687 if let Some(spawn) = publication_spawn {
3688 self.forwarding
3689 .bind_spawn(connection_id, spawn)
3690 .map_err(RouterError::Forwarding)?;
3691 }
3692 #[cfg(test)]
3693 if prefix_binding.is_some() {
3694 self.claims.pause_prefix_hello_for_test(8);
3695 }
3696 if swap_candidate {
3697 let module_id = hello.manifest.module_id.clone();
3698 let reply = self.register_swap_candidate(
3699 connection_id,
3700 sink,
3701 &frame,
3702 hello.manifest,
3703 negotiated_ver,
3704 control_ops,
3705 hello_ack,
3706 #[cfg(windows)]
3707 hello.launch_nonce.as_deref(),
3708 )?;
3709 if let Some(spawn) = &channel_spawn {
3710 if self
3711 .registry
3712 .get_module_by_connection(connection_id)
3713 .ok()
3714 .flatten()
3715 .is_some()
3716 {
3717 if let Some(sink) = primary_sink {
3718 spawn.bind_primary_sink(sink);
3719 }
3720 spawn.hello_accepted();
3721 }
3722 } else if self
3723 .registry
3724 .get_module_by_connection(connection_id)
3725 .ok()
3726 .flatten()
3727 .is_some()
3728 {
3729 self.supervisor
3730 .retire_unused_registration_channel(&module_id, true);
3731 }
3732 return Ok(reply);
3733 }
3734 let registration = match self
3741 .registry
3742 .register_unpublished_verified_with_control_ops(
3743 hello.manifest,
3744 negotiated_ver,
3745 connection_id,
3746 control_ops,
3747 verification.nonce_verified,
3748 ) {
3749 Ok(registration) => registration,
3750 Err(RegistryError::DuplicateModuleId { module_id }) => {
3751 return Ok(vec![control_error_frame(
3752 &frame,
3753 "duplicate_module_id",
3754 format!(
3755 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
3756 ),
3757 )?])
3758 }
3759 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3760 return Ok(vec![control_error_frame(
3761 &frame,
3762 "invalid_module_id",
3763 err.to_string(),
3764 )?])
3765 }
3766 Err(err) => {
3767 return Ok(vec![control_error_frame(
3768 &frame,
3769 "registry_error",
3770 err.to_string(),
3771 )?])
3772 }
3773 };
3774 #[cfg(test)]
3775 self.hold_between_registry_and_forwarding();
3776
3777 let mut reply = Vec::new();
3778 let publication = self
3781 .registry
3782 .publish_registration_after(connection_id, || {
3783 if let Some(sink) = sink {
3784 let concurrency = manifest_concurrency(®istration.manifest);
3800 self.forwarding.register_module_connection_acked(
3801 connection_id,
3802 registration.manifest.module_id.clone(),
3803 negotiated_ver,
3804 concurrency,
3805 sink,
3806 crate::forwarding::ModuleHelloAck {
3807 frame: hello_ack,
3808 verified_prefix_owner: verification.prefix_owner,
3809 },
3810 )?;
3811 } else {
3812 reply.push(hello_ack);
3815 }
3816 #[cfg(test)]
3817 self.hold_between_ack_and_publication();
3818 Ok(())
3819 })
3820 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3821 let published = match publication {
3822 Ok(published) => published,
3823 Err(err) => {
3824 if matches!(
3831 self.rollback_unpublished_hello(connection_id),
3832 Ok(r) if !r.is_empty()
3833 ) {
3834 crate::supervise::notify_registration_release();
3835 }
3836 return Ok(vec![control_error_frame(
3837 &frame,
3838 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
3839 "invalid_hello"
3840 } else {
3841 forwarding_error_code(&err)
3842 },
3843 err.to_string(),
3844 )?]);
3845 }
3846 };
3847 if !published {
3848 debug!(
3849 module_id = %registration.manifest.module_id,
3850 connection_id = connection_id.get(),
3851 "registration was removed before it could be published"
3852 );
3853 return Ok(reply);
3854 }
3855
3856 drop(_prefix_ownership);
3857 drop(_prefix_live);
3858
3859 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
3868 info!(
3869 module_id = %registration.manifest.module_id,
3870 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
3871 );
3872 }
3873 if registration.manifest.registration_channel == Some(false)
3874 && self
3875 .supervisor
3876 .module_is_gated(®istration.manifest.module_id)
3877 && self
3878 .config_runtime
3879 .as_ref()
3880 .and_then(|owner| owner.registration_declaration(®istration.manifest.module_id))
3881 .as_deref()
3882 == Some("declared")
3883 {
3884 warn!(module_id = %registration.manifest.module_id,
3885 "HELLO registration_channel is false despite the approved channel declaration; registration remains admitted");
3886 }
3887
3888 self.apply_registration_capabilities(®istration);
3889
3890 self.supervisor
3891 .record_activity_registration(®istration.manifest.module_id);
3892
3893 info!(
3894 module_id = %registration.manifest.module_id,
3895 module_version = %registration.manifest.module_version,
3896 negotiated_ver,
3897 routable_provider = manifest_provides_routable_role(®istration.manifest),
3898 connection_id = connection_id.get(),
3899 "module registered"
3900 );
3901
3902 if let Some(spawn) = &channel_spawn {
3903 if let Some(sink) = primary_sink {
3904 spawn.bind_primary_sink(sink);
3905 }
3906 spawn.hello_accepted();
3907 } else if !prefix_submodule {
3908 self.supervisor
3909 .retire_unused_registration_channel(®istration.manifest.module_id, false);
3910 }
3911 #[cfg(windows)]
3912 if let Some(nonce) = hello.launch_nonce.as_deref() {
3913 self.supervisor
3914 .program_hello_accepted(®istration.manifest.module_id, nonce);
3915 }
3916
3917 #[cfg(test)]
3918 if let Some(hold) = &self.hello_inside_publication_hold {
3919 hold.wait();
3920 hold.wait();
3921 }
3922
3923 Ok(reply)
3924 }
3925
3926 #[allow(clippy::too_many_arguments)]
3943 fn register_swap_candidate(
3944 &self,
3945 connection_id: ConnectionId,
3946 sink: Option<crate::FrameSink>,
3947 frame: &Frame,
3948 manifest: ModuleManifest,
3949 negotiated_ver: u8,
3950 control_ops: Vec<String>,
3951 hello_ack: Frame,
3952 #[cfg(windows)] launch_nonce: Option<&str>,
3953 ) -> Result<Vec<Frame>, RouterError> {
3954 let module_id = manifest.module_id.clone();
3955 self.supervisor
3958 .with_swap_candidate_admission(&module_id, || {
3959 let reply = self.publish_swap_candidate(
3960 connection_id,
3961 sink,
3962 frame,
3963 manifest,
3964 negotiated_ver,
3965 control_ops,
3966 hello_ack,
3967 #[cfg(windows)]
3968 launch_nonce,
3969 );
3970 let admitted = self
3971 .registry
3972 .get_candidate(&module_id)
3973 .ok()
3974 .flatten()
3975 .is_some_and(|candidate| candidate.connection_id == connection_id);
3976 (reply, admitted)
3977 })
3978 .unwrap_or_else(|| {
3979 Ok(vec![control_error_frame(
3980 frame,
3981 "swap_token_invalid",
3982 "swap ended before candidate HELLO publication",
3983 )?])
3984 })
3985 }
3986
3987 #[allow(clippy::too_many_arguments)]
3988 fn publish_swap_candidate(
3989 &self,
3990 connection_id: ConnectionId,
3991 sink: Option<crate::FrameSink>,
3992 frame: &Frame,
3993 manifest: ModuleManifest,
3994 negotiated_ver: u8,
3995 control_ops: Vec<String>,
3996 hello_ack: Frame,
3997 #[cfg(windows)] launch_nonce: Option<&str>,
3998 ) -> Result<Vec<Frame>, RouterError> {
3999 let module_id = manifest.module_id.clone();
4000 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
4001 manifest,
4002 negotiated_ver,
4003 connection_id,
4004 control_ops,
4005 ) {
4006 Ok(registration) => registration,
4007 Err(RegistryError::DuplicateModuleId { module_id }) => {
4008 return Ok(vec![control_error_frame(
4009 frame,
4010 "duplicate_module_id",
4011 format!(
4012 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
4013 ),
4014 )?])
4015 }
4016 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
4017 return Ok(vec![control_error_frame(
4018 frame,
4019 "invalid_module_id",
4020 err.to_string(),
4021 )?])
4022 }
4023 Err(err) => {
4024 return Ok(vec![control_error_frame(
4025 frame,
4026 "registry_error",
4027 err.to_string(),
4028 )?])
4029 }
4030 };
4031 #[cfg(test)]
4032 self.hold_between_registry_and_forwarding();
4033 let mut reply = Vec::new();
4034 let publication = self
4035 .registry
4036 .publish_registration_after(connection_id, || {
4037 if let Some(sink) = sink {
4038 let concurrency = manifest_concurrency(®istration.manifest);
4042 self.forwarding.register_candidate_module_connection_acked(
4043 connection_id,
4044 module_id.clone(),
4045 negotiated_ver,
4046 concurrency,
4047 sink,
4048 hello_ack,
4049 )?;
4050 } else {
4051 reply.push(hello_ack);
4052 }
4053 #[cfg(test)]
4054 self.hold_between_ack_and_publication();
4055 Ok(())
4056 })
4057 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4058 let published = match publication {
4059 Ok(published) => published,
4060 Err(err) => {
4061 if matches!(
4062 self.rollback_unpublished_hello(connection_id),
4063 Ok(r) if !r.is_empty()
4064 ) {
4065 crate::supervise::notify_registration_release();
4066 }
4067 return Ok(vec![control_error_frame(
4068 frame,
4069 forwarding_error_code(&err),
4070 err.to_string(),
4071 )?]);
4072 }
4073 };
4074 if !published {
4075 debug!(
4076 module_id = %module_id,
4077 connection_id = connection_id.get(),
4078 "swap candidate registration was removed before it could be published"
4079 );
4080 return Ok(reply);
4081 }
4082 info!(
4083 module_id = %module_id,
4084 module_version = %registration.manifest.module_version,
4085 negotiated_ver,
4086 ready = registration.ready,
4087 connection_id = connection_id.get(),
4088 "swap candidate registered; not routable until cutover"
4089 );
4090 #[cfg(windows)]
4091 if let Some(nonce) = launch_nonce {
4092 self.supervisor.program_hello_accepted(&module_id, nonce);
4093 }
4094 Ok(reply)
4095 }
4096
4097 fn build_hello_ack(
4098 &self,
4099 frame: &Frame,
4100 negotiated_ver: u8,
4101 module_id: &str,
4102 ) -> Result<Frame, RouterError> {
4103 let ack = ModuleHelloAckBody {
4104 negotiated_ver,
4105 subc_ops: module_subc_ops(),
4106 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
4107 storage: self
4108 .storage_config
4109 .as_ref()
4110 .map(|cfg| cfg.descriptor_for(module_id)),
4111 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4112 };
4113 let body = serde_json::to_vec(&ack).map_err(|err| {
4114 RouterError::backend(
4115 0,
4116 frame.header.corr,
4117 format!("failed to encode HELLO_ACK: {err}"),
4118 )
4119 })?;
4120
4121 Frame::build_with_version(
4122 negotiated_ver,
4123 FrameType::HelloAck,
4124 control_flags(),
4125 0,
4126 0,
4127 frame.header.corr,
4128 body,
4129 )
4130 .map_err(RouterError::FrameBuild)
4131 }
4132
4133 async fn handle_client_control_request(
4134 &self,
4135 ctx: &RouteCtx,
4136 frame: Frame,
4137 request: ClientControlRequest,
4138 route_open_timings: &mut Option<RouteOpenTimings>,
4139 ) -> Result<Vec<Frame>, RouterError> {
4140 if let Some(owner) = &self.config_runtime {
4141 let module = match &request {
4142 ClientControlRequest::SupervisorRestart { module_id, .. }
4143 | ClientControlRequest::SupervisorReload { module_id }
4144 | ClientControlRequest::SupervisorSwap { module_id, .. }
4145 | ClientControlRequest::SupervisorSetEnabled {
4146 module_id,
4147 enabled: true,
4148 } => Some(module_id.as_str()),
4149 _ => None,
4150 };
4151 let rescan = matches!(
4152 &request,
4153 ClientControlRequest::SupervisorRescan { preview: false }
4154 | ClientControlRequest::SupervisorRescanChecked { preview: false, .. }
4155 );
4156 let held = if module.is_some() || rescan {
4157 let module = module.map(str::to_owned);
4158 owner
4159 .on_blocking(move |owner| owner.checked_admission_hold(module.as_deref()))
4160 .await
4161 } else {
4162 Ok(())
4163 };
4164 if let Err(error) = held {
4165 if error.code != "config_delivery_deferred" {
4166 let code = if error.code == "store_tampered" {
4167 "store_tampered"
4168 } else {
4169 "config_gated"
4170 };
4171 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
4172 }
4173 }
4174 }
4175 if let Some(operation) = privileged_operation(&request) {
4176 let caller = self.resolve_caller(ctx.connection_id).await;
4177 let op = client_control_request_op(&request);
4178 let text = prompt_text(operation, &caller);
4179 match self.privileged_operator_prompt {
4180 PrivilegedOperatorPrompt::Log => {
4181 info!(target: "subc_daemon::privileged_prompt", operation = op,
4182 target = ?operation, caller = %caller.description(), summary = %text,
4183 "privileged request would prompt the operator");
4184 }
4185 PrivilegedOperatorPrompt::Enforce => {
4186 let key =
4187 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
4188 let stamp = self
4189 .operator_apps
4190 .lock()
4191 .unwrap_or_else(|p| p.into_inner())
4192 .get(&ctx.connection_id)
4193 .cloned();
4194 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
4195 &self.operator_app_policy,
4196 &self.launch_table,
4197 stamp.as_ref(),
4198 )
4199 .then(|| self.shortcut_context(stamp, None, None));
4200 let outcome = self
4201 .forwarding
4202 .operator_confirms()
4203 .approve_privileged_with_shortcut(
4204 ctx,
4205 &frame,
4206 op,
4207 text,
4208 caller.description(),
4209 key,
4210 shortcut,
4211 )
4212 .await;
4213 if outcome.code != "confirmed" {
4214 return Ok(vec![outcome.frame(&frame)]);
4215 }
4216 }
4217 }
4218 }
4219 match request {
4220 ClientControlRequest::DaemonShutdownIntent { intent } => {
4221 if intent.as_str() != Some("restart") {
4222 return Ok(vec![control_error_frame(
4223 &frame,
4224 "invalid_params",
4225 "intent must be 'restart'",
4226 )?]);
4227 }
4228 if let Some(owner) = &self.config_runtime {
4233 match owner
4234 .on_blocking(|owner| owner.assign_daemon_restart())
4235 .await
4236 {
4237 Ok(Some(generation)) => {
4238 info!(
4239 generation,
4240 "daemon restart will warm up approved generation"
4241 )
4242 }
4243 Ok(None) => {}
4244 Err(error) => warn!(
4245 %error,
4246 "daemon restart intent not recorded; the next start boots the last active generation"
4247 ),
4248 }
4249 }
4250 let response = ClientControlResponse::DaemonShutdownIntent {
4251 intent: "restart".to_owned(),
4252 expires_at_ms: self.supervisor.announce_daemon_restart(),
4253 };
4254 Ok(vec![control_response_body_frame(
4255 &frame,
4256 &response,
4257 "ClientControlResponse::DaemonShutdownIntent",
4258 )?])
4259 }
4260 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
4261 ClientControlRequest::CatalogList { module_id } => {
4262 self.handle_catalog_list(frame, module_id)
4263 }
4264 ClientControlRequest::RouteOpen {
4265 target,
4266 identity,
4267 consumer_identity,
4268 consumer_capabilities,
4269 role_versions,
4270 busy_refusals,
4271 admission_facts,
4272 scope,
4273 ..
4274 } => {
4275 self.handle_route_open(
4276 ctx,
4277 frame,
4278 RouteOpenRequest {
4279 target,
4280 identity,
4281 consumer_identity,
4282 consumer_capabilities,
4283 role_versions,
4284 busy_refusals,
4285 admission_facts,
4286 scope,
4287 },
4288 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
4289 )
4290 .await
4291 }
4292 ClientControlRequest::RoutePoll {
4293 route_channel,
4294 route_epoch,
4295 kind,
4296 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
4297 ClientControlRequest::SupervisorList {} => {
4298 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
4299 self.agent_declarations.refuse(
4300 self,
4301 ctx,
4302 control_error_frame(
4303 &frame,
4304 "busy",
4305 "supervisor list work is at capacity; retry",
4306 )?,
4307 );
4308 return Ok(Vec::new());
4309 };
4310 let handler = self.clone();
4312 let sink = ctx.egress.clone();
4313 tokio::spawn(
4314 async move {
4315 let _permit = permit;
4316 let result = handler.handle_supervisor_list(frame.clone()).await;
4317 let replies = match result {
4318 Ok(replies) => replies,
4319 Err(error) => {
4320 control_error_frame(&frame, "backend_error", error.to_string())
4321 .into_iter()
4322 .collect()
4323 }
4324 };
4325 for reply in replies {
4326 if !matches!(
4327 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
4328 .await,
4329 Ok(Ok(()))
4330 ) {
4331 break;
4332 }
4333 }
4334 }
4335 .with_current_subscriber(),
4336 );
4337 Ok(Vec::new())
4338 }
4339 ClientControlRequest::SupervisorSpawnSnapshot {} => {
4340 self.handle_supervisor_spawn_snapshot(frame)
4341 }
4342 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
4343 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
4344 }
4345 ClientControlRequest::SupervisorRestart {
4346 module_id,
4347 drain_timeout_ms,
4348 } => {
4349 self.log_supervisor_request_received(
4350 ctx,
4351 frame.header.corr,
4352 ops::SUPERVISOR_RESTART,
4353 Some(&module_id),
4354 None,
4355 )?;
4356 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4357 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
4358 .await
4359 }
4360 ClientControlRequest::SupervisorSwap {
4361 module_id,
4362 ready_timeout_ms,
4363 } => {
4364 self.log_supervisor_request_received(
4365 ctx,
4366 frame.header.corr,
4367 ops::SUPERVISOR_SWAP,
4368 Some(&module_id),
4369 None,
4370 )?;
4371 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4372 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
4373 .await
4374 }
4375 ClientControlRequest::SupervisorReload { module_id } => {
4376 self.log_supervisor_request_received(
4377 ctx,
4378 frame.header.corr,
4379 ops::SUPERVISOR_RELOAD,
4380 Some(&module_id),
4381 None,
4382 )?;
4383 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4384 self.handle_supervisor_reload(frame, module_id, requester)
4385 .await
4386 }
4387 ClientControlRequest::SupervisorRescan { preview } => {
4388 if !preview {
4389 self.log_supervisor_request_received(
4390 ctx,
4391 frame.header.corr,
4392 ops::SUPERVISOR_RESCAN,
4393 None,
4394 None,
4395 )?;
4396 }
4397 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4398 let operation = self.handle_supervisor_rescan(
4399 frame,
4400 if preview {
4401 RescanMode::Preview
4402 } else {
4403 RescanMode::Execute
4404 },
4405 requester,
4406 );
4407 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
4408 .await
4409 }
4410 ClientControlRequest::SupervisorRescanChecked {
4411 preview,
4412 candidate,
4413 expect,
4414 } => {
4415 self.observe_control_wait(
4416 ctx,
4417 None,
4418 ops::SUPERVISOR_RESCAN,
4419 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
4420 )
4421 .await
4422 }
4423 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
4424 self.log_supervisor_request_received(
4425 ctx,
4426 frame.header.corr,
4427 ops::SUPERVISOR_RELEASE_RESERVED,
4428 Some(&module_id),
4429 None,
4430 )?;
4431 self.handle_supervisor_release_reserved(frame, module_id)
4432 .await
4433 }
4434 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
4435 self.log_supervisor_request_received(
4436 ctx,
4437 frame.header.corr,
4438 ops::SUPERVISOR_SET_ENABLED,
4439 Some(&module_id),
4440 Some(enabled),
4441 )?;
4442 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4443 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
4444 .await
4445 }
4446 ClientControlRequest::SupervisorHealthProbe { module_id } => {
4447 self.handle_supervisor_health_probe(ctx, frame, module_id)
4448 .await
4449 }
4450 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
4451 ClientControlRequest::SupervisorRoutes { module_id } => {
4452 self.handle_supervisor_routes(frame, module_id)
4453 }
4454 ClientControlRequest::SupervisorProvenance { module_id } => {
4455 self.handle_supervisor_provenance(ctx, frame, module_id)
4456 .await
4457 }
4458 ClientControlRequest::SupervisorStderrTail {
4459 module_id,
4460 max_lines,
4461 max_bytes,
4462 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
4463 ClientControlRequest::SupervisorTerminals { module_id } => {
4464 self.handle_supervisor_terminals(ctx, frame, module_id)
4465 .await
4466 }
4467 }
4468 }
4469
4470 fn log_supervisor_request_received(
4471 &self,
4472 ctx: &RouteCtx,
4473 corr: u64,
4474 op: &'static str,
4475 module_id: Option<&str>,
4476 enabled: Option<bool>,
4477 ) -> Result<(), RouterError> {
4478 let caller = self.supervisor_requester(ctx, corr)?;
4479 let caller = principal_label(&caller);
4480 info!(
4483 target: "control",
4484 op,
4485 module_id,
4486 enabled,
4487 connection_id = ctx.connection_id.get(),
4488 caller = %caller,
4489 "supervisor request received"
4490 );
4491 Ok(())
4492 }
4493
4494 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
4495 Ok(self
4496 .registry
4497 .get_module_by_connection(ctx.connection_id)
4498 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4499 .map(|registration| Principal::Reserved {
4500 module_id: registration.manifest.module_id,
4501 })
4502 .unwrap_or(Principal::Direct))
4503 }
4504
4505 fn handle_module_control_request(
4506 &self,
4507 connection_id: ConnectionId,
4508 frame: Frame,
4509 request: ModuleControlRequestFromModule,
4510 ) -> Result<Vec<Frame>, RouterError> {
4511 match request {
4512 ModuleControlRequestFromModule::RouteAuthorize {
4513 principal,
4514 scope,
4515 target,
4516 expected,
4517 } => self.handle_route_authorize(
4518 connection_id,
4519 frame,
4520 principal,
4521 scope,
4522 target,
4523 expected,
4524 ),
4525 ModuleControlRequestFromModule::CatalogUpdate {
4526 provides,
4527 capabilities,
4528 ready,
4529 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
4530 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
4531 self.handle_catalog_update_role_declarations(
4532 connection_id,
4533 frame,
4534 role_declarations,
4535 )
4536 }
4537 ModuleControlRequestFromModule::LiveRoots {} => {
4538 let registered = self
4539 .registry
4540 .get_module_by_connection(connection_id)
4541 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4542 let Some(registration) = registered else {
4543 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
4544 };
4545 let response = self
4546 .forwarding
4547 .live_roots(®istration.manifest.module_id)
4548 .map_err(RouterError::Forwarding)?;
4549 Ok(vec![control_response_body_frame(
4550 &frame,
4551 &response,
4552 "ModuleControlResponseToModule::LiveRoots",
4553 )?])
4554 }
4555 ModuleControlRequestFromModule::AgentCommands {
4556 started,
4557 finished,
4558 known,
4559 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
4560 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
4561 self.handle_scope_sync(connection_id, frame, generation, scopes)
4562 }
4563 ModuleControlRequestFromModule::ScopeApply {
4564 generation,
4565 upsert,
4566 end,
4567 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
4568 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
4569 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
4570 }
4571 _ => Ok(vec![control_error_frame(
4572 &frame,
4573 "unsupported_control_frame",
4574 "unsupported module control operation",
4575 )?]),
4576 }
4577 }
4578
4579 fn handle_scope_served_by(
4580 &self,
4581 connection_id: ConnectionId,
4582 frame: Frame,
4583 ) -> Result<Vec<Frame>, RouterError> {
4584 let registered = self
4585 .registry
4586 .get_module_by_connection(connection_id)
4587 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4588 let Some(registered) = registered else {
4589 return Ok(vec![control_error_frame(
4590 &frame,
4591 "not_registered",
4592 "scope.served_by requires an active module registration owned by this connection",
4593 )?]);
4594 };
4595 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
4596 {
4597 Ok(request) => request,
4598 Err(err) => {
4599 return Ok(vec![control_error_frame(
4600 &frame,
4601 "invalid_control_body",
4602 format!("malformed scope.served_by body: {err}"),
4603 )?])
4604 }
4605 };
4606 let answer = self
4607 .scopes
4608 .read()
4609 .unwrap_or_else(|poisoned| poisoned.into_inner())
4610 .served_by(
4611 ®istered.manifest.module_id,
4612 &request.scope_ref,
4613 request.scope_epoch,
4614 &request.module_id,
4615 );
4616 let reply = subc_control::ScopeServedByReply::new(
4617 answer,
4618 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4619 );
4620 Ok(vec![control_response_body_frame(
4621 &frame,
4622 &reply,
4623 "ScopeServedByReply",
4624 )?])
4625 }
4626
4627 fn handle_operator_confirm(
4628 &self,
4629 ctx: &RouteCtx,
4630 frame: Frame,
4631 ) -> Result<Vec<Frame>, RouterError> {
4632 use crate::operator_confirm::{audit, Outcome};
4633 let registration = self
4634 .registry
4635 .get_module_by_connection(ctx.connection_id)
4636 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4637 let Some(registration) = registration else {
4638 let outcome = Outcome::refusal("not_registered");
4639 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4640 return Ok(vec![outcome.frame(&frame)]);
4641 };
4642 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
4643 Ok(request) => request,
4644 Err(_) => {
4645 let outcome = Outcome::refusal("invalid_control_body");
4646 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4647 return Ok(vec![outcome.frame(&frame)]);
4648 }
4649 };
4650 let declared = matches!(
4651 request.kind,
4652 Some(subc_protocol::session::OperatorConfirmKind::Operation)
4653 ) && request.operation.as_ref().is_some_and(|operation| {
4654 registration
4655 .manifest
4656 .confirm_operations
4657 .as_ref()
4658 .is_some_and(|operations| operations.contains(operation))
4659 });
4660 let module_id = registration.manifest.module_id;
4661 let nonce = self
4665 .hello_launch_nonces
4666 .lock()
4667 .unwrap_or_else(|p| p.into_inner())
4668 .nonce(ctx.connection_id)
4669 .map(str::to_owned);
4670 let nonce_proven = self
4673 .confirm_channel_proven(ctx.connection_id, &module_id)
4674 .unwrap_or_else(|| {
4675 nonce.as_deref().is_some_and(|nonce| {
4676 self.supervisor
4677 .spawned_consumer_authorized(&module_id, nonce)
4678 })
4679 });
4680 let confirms = self.forwarding.operator_confirms();
4681 self.forwarding
4682 .with_operator_route(
4683 ctx.connection_id,
4684 request.route_channel,
4685 request.route_epoch,
4686 |binding| {
4687 let shortcut = binding
4688 .filter(|binding| {
4689 nonce_proven
4690 && crate::operator_confirm::valid_summary(&request.summary)
4691 && declared
4692 && crate::operator_confirm::ShortcutContext::route_eligible(
4693 &self.operator_app_policy,
4694 &self.launch_table,
4695 binding,
4696 )
4697 })
4698 .map(|binding| {
4699 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
4700 &self.operator_app_policy,
4701 binding,
4702 );
4703 let stamp = if phone.is_some() {
4704 None
4705 } else {
4706 binding.operator_app.clone()
4707 };
4708 self.shortcut_context(
4709 stamp,
4710 phone,
4711 Some((request.route_channel, request.route_epoch)),
4712 )
4713 });
4714 confirms.admit_with_shortcut(
4715 ctx,
4716 frame,
4717 module_id,
4718 nonce_proven,
4719 request,
4720 binding,
4721 shortcut,
4722 )
4723 },
4724 )
4725 .map_err(RouterError::Forwarding)
4726 }
4727
4728 fn shortcut_context(
4729 &self,
4730 stamp: Option<crate::operator_apps::OperatorAppStamp>,
4731 phone_fingerprint: Option<String>,
4732 route: Option<(u16, u32)>,
4733 ) -> crate::operator_confirm::ShortcutContext {
4734 crate::operator_confirm::ShortcutContext {
4735 ledger: self.activity.clone(),
4736 forwarding: self.forwarding.clone(),
4737 launches: self.launch_table.clone(),
4738 stamp,
4739 phone_fingerprint,
4740 route,
4741 connections: self.operator_apps.clone(),
4742 #[cfg(test)]
4743 hooks: Default::default(),
4744 }
4745 }
4746
4747 #[cfg(any(test, feature = "test-support"))]
4749 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
4750 self.operator_app_policy.phone_enabled = enabled;
4751 self
4752 }
4753
4754 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
4757 let mut scopes = ScopeRecords::from(scopes);
4758 for (index, record) in scopes.records.iter_mut().enumerate() {
4759 if let Some(refusal) = Self::canonicalize_workdir(record) {
4760 scopes.refusals.insert(index, refusal);
4761 }
4762 }
4763 scopes
4764 }
4765
4766 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
4767 let workdir = record.workdir.as_mut()?;
4768 let path = std::path::Path::new(workdir);
4769 if !path.is_absolute() {
4770 return Some((
4771 error_codes::INVALID_WORKDIR,
4772 "workdir must be absolute".to_string(),
4773 ));
4774 }
4775 match ProjectRootId::from_path_allowing_missing(path) {
4776 Ok(root) => match root.as_path().to_str() {
4777 Some(canonical) => {
4778 *workdir = canonical.to_string();
4779 None
4780 }
4781 None => Some((
4782 error_codes::INVALID_WORKDIR,
4783 "canonical workdir is not UTF-8".to_string(),
4784 )),
4785 },
4786 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
4787 }
4788 }
4789
4790 fn handle_scope_sync(
4794 &self,
4795 connection_id: ConnectionId,
4796 frame: Frame,
4797 generation: u64,
4798 scopes: Vec<ScopeRecord>,
4799 ) -> Result<Vec<Frame>, RouterError> {
4800 self.handle_scope_change(connection_id, frame, generation, scopes, None)
4801 }
4802
4803 fn handle_scope_change(
4804 &self,
4805 connection_id: ConnectionId,
4806 frame: Frame,
4807 generation: u64,
4808 scopes: Vec<ScopeRecord>,
4809 end: Option<Vec<ScopeEnd>>,
4810 ) -> Result<Vec<Frame>, RouterError> {
4811 let is_apply = end.is_some();
4812 let Some(registration) = self
4813 .registry
4814 .get_module_by_connection(connection_id)
4815 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4816 else {
4817 return Ok(vec![control_error_frame(
4818 &frame,
4819 "not_registered",
4820 "scope.sync and scope.apply require an active module registration owned by this connection",
4821 )?]);
4822 };
4823 let owner = registration.manifest.module_id;
4824 let scopes = Self::prepare_scope_records(scopes);
4825 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
4826 let is_current_launch = |connection: ConnectionId| {
4827 self.connection_is_current_launch(&owner, connection, current_nonce.as_deref())
4828 };
4829 let mut table = self
4834 .scopes
4835 .write()
4836 .unwrap_or_else(|poisoned| poisoned.into_inner());
4837 let outcome = match end {
4838 Some(end) => table.apply(
4839 &owner,
4840 connection_id,
4841 is_current_launch,
4842 generation,
4843 scopes,
4844 end,
4845 ),
4846 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
4847 };
4848 let drained = match &outcome {
4849 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
4850 Err(_) => Vec::new(),
4851 };
4852 let has_deadlines = outcome.is_ok() && table.has_deadlines();
4853 drop(table);
4854 if has_deadlines {
4855 self.scope_deadline_added.notify_one();
4856 }
4857 match outcome {
4858 Ok(applied) => {
4859 let counts = ScopeOutcomeCounts::of(&applied.results);
4860 info!(
4861 owner = %owner,
4862 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
4863 generation,
4864 records = applied.results.len(),
4865 created = counts.created,
4866 replaced = counts.replaced,
4867 updated = counts.updated,
4868 unchanged = counts.unchanged,
4869 refused = counts.refused,
4870 ended = applied.ended.len(),
4871 tag_changes = applied.tag_changes.len(),
4872 routes_closed = drained.len(),
4873 "scope change accepted"
4874 );
4875 for refused in applied
4881 .results
4882 .iter()
4883 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
4884 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
4885 {
4886 warn!(
4887 owner = %owner,
4888 generation,
4889 scope_ref = %refused.scope_ref,
4890 scope_epoch = refused.scope_epoch,
4891 code = refused.code.as_deref().unwrap_or(""),
4892 "scope record refused"
4893 );
4894 }
4895 self.close_scope_drained_routes(drained);
4896 let response = if is_apply {
4897 ModuleControlResponseToModule::ScopeApply {
4898 generation,
4899 results: applied.results,
4900 end_results: applied.end_results,
4901 ended: applied.ended,
4902 }
4903 } else {
4904 ModuleControlResponseToModule::ScopeSync {
4905 generation,
4906 results: applied.results,
4907 ended: applied.ended,
4908 }
4909 };
4910 Ok(vec![control_response_body_frame(
4911 &frame,
4912 &response,
4913 "scope change response",
4914 )?])
4915 }
4916 Err(refusal) => {
4917 info!(
4918 owner = %owner,
4919 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
4920 generation,
4921 code = refusal.code,
4922 "scope change refused"
4923 );
4924 Ok(vec![control_error_frame(
4925 &frame,
4926 refusal.code,
4927 refusal.message,
4928 )?])
4929 }
4930 }
4931 }
4932
4933 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
4935 scope_expiry::ScopeExpiryContext::new(self).close_scope_drained_routes(drained);
4936 }
4937
4938 fn handle_route_authorize(
4939 &self,
4940 connection_id: ConnectionId,
4941 frame: Frame,
4942 principal: Principal,
4943 scope: subc_protocol::scope::RouteAuthorizeScope,
4944 target: String,
4945 expected: subc_protocol::scope::RouteAuthorizeExpected,
4946 ) -> Result<Vec<Frame>, RouterError> {
4947 use crate::scopes::admission::{admit_route, AdmissionMode};
4948 use subc_protocol::scope::{
4949 RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
4950 };
4951 let registry = self
4952 .registry
4953 .route_admission_snapshot()
4954 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4955 let owner_configured = match &scope.owner {
4956 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4957 _ => false,
4958 };
4959 let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
4960 let answer = {
4961 let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
4962 let permitted = registry
4963 .iter()
4964 .find(|r| r.connection_id == connection_id)
4965 .is_some_and(|r| {
4966 scopes.is_federated_owner(&Principal::Reserved {
4967 module_id: r.manifest.module_id.clone(),
4968 }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
4969 caps.provides
4970 .iter()
4971 .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
4972 })
4973 });
4974 if !permitted {
4975 drop(scopes);
4976 return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
4977 "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
4978 }
4979 admit_route(
4980 &scopes,
4981 ®istry,
4982 &principal,
4983 &target,
4984 Some(&selector),
4985 owner_configured,
4986 AdmissionMode::Authorize(&expected),
4987 )
4988 };
4989 let response = match answer {
4990 Ok(admission) => {
4991 let mut response = RouteAuthorizeResponse::new(
4992 RouteAuthorizeOutcome::Authorized,
4993 "route admission authorized",
4994 );
4995 response.content_version = admission.content_version;
4996 response
4997 }
4998 Err(refusal) => refusal.response(),
4999 };
5000 Ok(vec![control_response_body_frame(
5001 &frame,
5002 &response,
5003 "RouteAuthorizeResponse",
5004 )?])
5005 }
5006
5007 fn handle_scope_describe(
5010 &self,
5011 connection_id: ConnectionId,
5012 frame: Frame,
5013 owner: Principal,
5014 scope_ref: String,
5015 ) -> Result<Vec<Frame>, RouterError> {
5016 let registered = self
5017 .registry
5018 .get_module_by_connection(connection_id)
5019 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5020 let Some(registered) = registered else {
5021 return Ok(vec![control_error_frame(
5022 &frame,
5023 "not_registered",
5024 "scope.describe requires an active module registration owned by this connection",
5025 )?]);
5026 };
5027 let scopes = self
5028 .scopes
5029 .read()
5030 .unwrap_or_else(|poisoned| poisoned.into_inner());
5031 if scopes.is_federated_owner(&owner)
5032 && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
5033 {
5034 return Ok(vec![control_error_frame(
5035 &frame,
5036 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
5037 "scope.describe of the federated owner requires scope-federated/v1",
5038 )?]);
5039 }
5040 let mut description = scopes.describe(&owner, &scope_ref);
5041 drop(scopes);
5042 if owner
5043 != (Principal::Reserved {
5044 module_id: registered.manifest.module_id.clone(),
5045 })
5046 {
5047 if let Some(facts) = description
5048 .stamp
5049 .as_mut()
5050 .and_then(|stamp| stamp.attributes.federated.as_mut())
5051 {
5052 facts.stamp_required = false;
5053 }
5054 }
5055 if !registered
5058 .manifest
5059 .capabilities
5060 .as_ref()
5061 .is_some_and(|caps| {
5062 caps.provides
5063 .iter()
5064 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5065 })
5066 {
5067 if let Some(stamp) = description.stamp.as_mut() {
5068 stamp.attributes.on_behalf_of_agent = None;
5069 }
5070 }
5071 if !registered
5072 .manifest
5073 .capabilities
5074 .as_ref()
5075 .is_some_and(|caps| {
5076 caps.provides
5077 .iter()
5078 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5079 })
5080 {
5081 if let Some(stamp) = description.stamp.as_mut() {
5082 stamp.attributes.flow_version = None;
5083 stamp.attributes.flow_code_hash = None;
5084 stamp.attributes.flow_install_id = None;
5085 }
5086 }
5087 let owner_configured = match &owner {
5088 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5093 _ => false,
5094 };
5095 let response = ModuleControlResponseToModule::ScopeDescribe {
5096 status: description.status,
5097 scope_epoch: description.scope_epoch,
5098 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
5099 owner_synced: description.owner_synced,
5100 owner_configured,
5101 scope: description.stamp,
5102 };
5103 Ok(vec![control_response_body_frame(
5104 &frame,
5105 &response,
5106 "ModuleControlResponseToModule::ScopeDescribe",
5107 )?])
5108 }
5109
5110 fn handle_catalog_update_role_declarations(
5111 &self,
5112 connection_id: ConnectionId,
5113 frame: Frame,
5114 role_declarations: serde_json::Value,
5115 ) -> Result<Vec<Frame>, RouterError> {
5116 if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
5117 return Ok(vec![control_error_frame(
5118 &frame,
5119 "invalid_role_declarations",
5120 err.to_string(),
5121 )?]);
5122 }
5123 let declarations = serde_json::from_value(role_declarations)
5124 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5125 if self
5126 .registry
5127 .replace_role_declarations_for_connection(connection_id, declarations)
5128 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5129 .is_none()
5130 {
5131 return Ok(vec![control_error_frame(
5132 &frame,
5133 "not_registered",
5134 "catalog.update_role_declarations requires an active module registration owned by this connection",
5135 )?]);
5136 }
5137 control_response_body_frame(
5138 &frame,
5139 &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
5140 "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
5141 )
5142 .map(|frame| vec![frame])
5143 }
5144
5145 fn handle_catalog_update(
5146 &self,
5147 connection_id: ConnectionId,
5148 frame: Frame,
5149 provides: Vec<ProviderRole>,
5150 capabilities: Option<CapabilityDeclarations>,
5151 ready: Option<bool>,
5152 ) -> Result<Vec<Frame>, RouterError> {
5153 self.refresh_capability_requirements();
5154 let Some(registration) = self
5155 .registry
5156 .get_module_by_connection(connection_id)
5157 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5158 else {
5159 return Ok(vec![control_error_frame(
5160 &frame,
5161 "not_registered",
5162 "catalog.update requires an active module registration owned by this connection",
5163 )?]);
5164 };
5165
5166 if let Some(message) =
5167 catalog_update_frozen_field_message(®istration.manifest, &provides)
5168 {
5169 return Ok(vec![control_error_frame(
5170 &frame,
5171 "catalog_update_frozen_field",
5172 message,
5173 )?]);
5174 }
5175
5176 let mut candidate = registration.manifest.clone();
5177 candidate.provides = provides.clone();
5178 candidate.capabilities = capabilities
5179 .clone()
5180 .or_else(|| registration.manifest.capabilities.clone());
5181 if let Some(rich_ui) = &candidate.rich_ui {
5182 if let Err(err) = rich_ui.validate() {
5183 return Ok(vec![control_error_frame(
5184 &frame,
5185 "invalid_manifest",
5186 err.to_string(),
5187 )?]);
5188 }
5189 }
5190 if let Err(err) = candidate.validate_capability_grammar() {
5191 return Ok(vec![control_error_frame(
5192 &frame,
5193 "invalid_capability_grammar",
5194 err.to_string(),
5195 )?]);
5196 }
5197
5198 let mut conflicts = self
5201 .capability_evaluator
5202 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
5203 if let Some(conflict) = conflicts.first() {
5204 let message = format!(
5205 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
5206 conflict.capability, conflict.claimants[0], candidate.module_id
5207 );
5208 for conflict in &mut conflicts {
5209 conflict.source = DuplicateClaimSource::CatalogUpdate;
5210 }
5211 log_duplicate_claim_events(conflicts);
5212 return Ok(vec![control_error_frame(
5213 &frame,
5214 "reserved_capability",
5215 message,
5216 )?]);
5217 }
5218
5219 let updated = match self.registry.replace_catalog_for_connection(
5220 connection_id,
5221 provides,
5222 capabilities,
5223 ready,
5224 ) {
5225 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => {
5226 return Ok(vec![control_error_frame(
5227 &frame,
5228 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5229 "scope-federated/v1 is fixed at HELLO; restart the module to change it",
5230 )?])
5231 }
5232 result => {
5233 result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5234 }
5235 };
5236 if updated.is_none() {
5237 return Ok(vec![control_error_frame(
5238 &frame,
5239 "not_registered",
5240 "catalog.update requires an active module registration owned by this connection",
5241 )?]);
5242 }
5243 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
5244 log_duplicate_claim_events(
5245 self.capability_evaluator
5246 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
5247 );
5248 }
5249 if capability_census_trigger(
5250 registration.manifest.capabilities.as_ref(),
5251 updated
5252 .as_ref()
5253 .and_then(|entry| entry.manifest.capabilities.as_ref()),
5254 ) {
5255 self.enforce_capability_denies();
5256 }
5257 self.refresh_capability_requirements();
5258
5259 let response = ModuleControlResponseToModule::CatalogUpdate {};
5260 control_response_body_frame(
5261 &frame,
5262 &response,
5263 "ModuleControlResponseToModule::CatalogUpdate",
5264 )
5265 .map(|frame| vec![frame])
5266 }
5267
5268 pub(crate) fn with_activity_ledger_config(
5269 mut self,
5270 run_dir: &Path,
5271 incarnation: String,
5272 config: crate::activity::ActivityConfig,
5273 ) -> Self {
5274 self.activity = Some(crate::activity::ActivityLedger::open_config(
5275 run_dir.join("activity.jsonl"),
5276 incarnation,
5277 config,
5278 ));
5279 self.supervisor.set_activity(self.activity.clone());
5280 self
5281 }
5282
5283 pub(crate) async fn configure_activity(
5284 &self,
5285 config: crate::activity::ActivityConfig,
5286 ) -> Result<(), &'static str> {
5287 match &self.activity {
5288 Some(ledger) => ledger.configure(config).await,
5289 None => Ok(()),
5290 }
5291 }
5292
5293 fn cancel_activity_subscription(
5294 &self,
5295 ctx: &RouteCtx,
5296 frame: &Frame,
5297 ) -> Result<bool, RouterError> {
5298 let task = self
5299 .activity_streams
5300 .lock()
5301 .unwrap_or_else(|p| p.into_inner())
5302 .remove(&(ctx.connection_id, frame.header.corr));
5303 let Some(task) = task else {
5304 return Ok(false);
5305 };
5306 task.abort();
5307 let end = Frame::build_with_version(
5308 response_version(frame),
5309 FrameType::StreamEnd,
5310 control_flags(),
5311 0,
5312 0,
5313 frame.header.corr,
5314 Vec::new(),
5315 )
5316 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
5317 let sink = ctx.egress.clone();
5318 tokio::spawn(async move {
5319 let _ = sink.send(end).await;
5320 });
5321 Ok(true)
5322 }
5323
5324 async fn handle_activity_read(
5325 &self,
5326 ctx: &RouteCtx,
5327 frame: Frame,
5328 ) -> Result<Vec<Frame>, RouterError> {
5329 #[cfg(test)]
5330 {
5331 let pause = self
5332 .activity_request_pause
5333 .lock()
5334 .unwrap()
5335 .clone()
5336 .filter(|pause| pause.connection == ctx.connection_id);
5337 if let Some(pause) = pause {
5338 let _cancelled = ActivityRequestCancellation(pause.clone());
5339 pause.started.notify_one();
5340 std::future::pending::<()>().await;
5341 }
5342 }
5343 use subc_control::{
5344 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
5345 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
5346 };
5347 #[derive(Deserialize)]
5348 struct Identity {
5349 #[serde(default)]
5350 consumer_identity: Option<ConsumerIdentity>,
5351 }
5352 let mut bound_spawn = None;
5356 let legacy = match self.activity_reader(ctx.connection_id) {
5357 channel_operations::ActivityReader::Module { module_id, spawn } => {
5358 bound_spawn = Some(spawn);
5359 Some(Ok(Principal::Reserved { module_id }))
5360 }
5361 channel_operations::ActivityReader::Refused => {
5363 return Ok(vec![control_error_frame(
5364 &frame,
5365 ACTIVITY_READ_FORBIDDEN,
5366 "helper, relay and retired connections cannot read activity",
5367 )?]);
5368 }
5369 channel_operations::ActivityReader::Legacy => None,
5370 };
5371 let principal = match legacy {
5375 Some(principal) => principal,
5376 None => match self
5377 .forwarding
5378 .module_id_for_connection(ctx.connection_id)
5379 .map_err(RouterError::Forwarding)?
5380 {
5381 Some(module_id) => Ok(Principal::Reserved { module_id }),
5382 None => {
5383 let identity: Identity = match serde_json::from_slice(&frame.body) {
5384 Ok(identity) => identity,
5385 Err(_) => {
5386 return Ok(vec![control_error_frame(
5387 &frame,
5388 ACTIVITY_MALFORMED,
5389 "malformed activity read",
5390 )?])
5391 }
5392 };
5393 self.route_open_principal(
5394 ctx.connection_id,
5395 &frame,
5396 identity.consumer_identity,
5397 )?
5398 }
5399 },
5400 };
5401 let admitted = matches!(principal, Ok(Principal::Direct))
5402 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
5403 if !admitted {
5404 return Ok(vec![control_error_frame(
5405 &frame,
5406 ACTIVITY_READ_FORBIDDEN,
5407 "activity reads require an operator or trusted mirror principal",
5408 )?]);
5409 }
5410 let Some(ledger) = &self.activity else {
5411 return Ok(vec![control_error_frame(
5412 &frame,
5413 ACTIVITY_UNAVAILABLE,
5414 "activity ledger unavailable",
5415 )?]);
5416 };
5417 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
5418 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
5419 {
5420 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
5421 Ok(request) => ledger.query(request).await,
5422 Err(_) => Err(ACTIVITY_MALFORMED),
5423 };
5424 return match result {
5425 Ok(reply) => Ok(vec![control_response_body_frame(
5426 &frame,
5427 &reply,
5428 ops::ACTIVITY_QUERY,
5429 )?]),
5430 Err(code) => Ok(vec![control_error_frame(
5431 &frame,
5432 code,
5433 "activity query refused",
5434 )?]),
5435 };
5436 }
5437 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
5438 Ok(request) => request,
5439 Err(_) => {
5440 return Ok(vec![control_error_frame(
5441 &frame,
5442 ACTIVITY_MALFORMED,
5443 "malformed activity subscription",
5444 )?])
5445 }
5446 };
5447 let mut subscription = match ledger.subscribe(request.since).await {
5448 Ok(subscription) => subscription,
5449 Err(code) => {
5450 return Ok(vec![control_error_frame(
5451 &frame,
5452 code,
5453 "activity subscription refused",
5454 )?])
5455 }
5456 };
5457 let key = (ctx.connection_id, frame.header.corr);
5458 let sink = ctx.egress.clone();
5459 let mut streams = self
5460 .activity_streams
5461 .lock()
5462 .unwrap_or_else(|p| p.into_inner());
5463 streams.retain(|_, task| !task.is_finished());
5464 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
5465 return Ok(vec![control_error_frame(
5466 &frame,
5467 "duplicate_subscription",
5468 "activity subscription correlation id already in use",
5469 )?]);
5470 }
5471 let stream = async move {
5472 loop {
5473 let entry = if let Some(entry) = subscription.replay.pop_front() {
5474 Some(entry)
5475 } else {
5476 subscription.receiver.recv().await
5477 };
5478 let Some(entry) = entry else {
5479 break;
5480 };
5481 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
5482 return;
5483 };
5484 let Ok(next) = Frame::build_with_version(
5485 response_version(&frame),
5486 FrameType::StreamData,
5487 control_flags(),
5488 0,
5489 0,
5490 frame.header.corr,
5491 body,
5492 ) else {
5493 return;
5494 };
5495 if sink.send(next).await.is_err() {
5496 return;
5497 }
5498 }
5499 if let Some(code) = subscription.terminal_code() {
5502 if let Ok(end) =
5503 control_error_frame(&frame, code, "activity subscriber fell behind")
5504 {
5505 let _ = sink.send(end).await;
5506 }
5507 }
5508 };
5509 let task = tokio::spawn(async move {
5513 match bound_spawn {
5514 Some(spawn) => {
5515 tokio::select! {
5516 () = stream => {}
5517 () = spawn.wait_closed() => {}
5518 }
5519 }
5520 None => stream.await,
5521 }
5522 });
5523 streams.insert(key, task.abort_handle());
5524 Ok(Vec::new())
5525 }
5526
5527 async fn handle_activity_record(
5528 &self,
5529 ctx: &RouteCtx,
5530 frame: Frame,
5531 ) -> Result<Vec<Frame>, RouterError> {
5532 use subc_control::{
5533 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
5534 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
5535 };
5536 let result = async {
5537 let module = self
5538 .forwarding
5539 .module_id_for_connection(ctx.connection_id)
5540 .ok()
5541 .flatten()
5542 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
5543 let request: ActivityRecordRequest =
5544 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
5545 crate::activity::validate(&module, &request)?;
5546 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
5547 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
5548 let at_ms = std::time::SystemTime::now()
5549 .duration_since(std::time::UNIX_EPOCH)
5550 .unwrap_or_default()
5551 .as_millis() as u64;
5552 ledger.record(module, request, actor, at_ms).await
5553 }
5554 .await;
5555 match result {
5556 Ok(entry) => Ok(vec![control_response_body_frame(
5557 &frame,
5558 &ActivityRecordResponse { seq: entry.seq },
5559 "activity.record",
5560 )?]),
5561 Err(code) => Ok(vec![control_error_frame(
5562 &frame,
5563 code,
5564 "activity recording refused",
5565 )?]),
5566 }
5567 }
5568
5569 fn activity_actor(
5570 &self,
5571 connection: ConnectionId,
5572 module: &str,
5573 on: Option<&subc_control::ActivityOn>,
5574 ) -> Result<subc_control::ActivityActor, &'static str> {
5575 use crate::forwarding::{DataRoute, DataRouteState};
5576 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
5577 let principal = |id: &str| Principal::Reserved {
5578 module_id: id.to_owned(),
5579 };
5580 let mut actor = ActivityActor {
5581 principal: principal(module),
5582 scope_owner: None,
5583 agent_id: None,
5584 run_id: None,
5585 flow_id: None,
5586 project_root: None,
5587 workdir: None,
5588 project_id: None,
5589 };
5590 let scope = match on {
5591 None => return Ok(actor),
5592 Some(ActivityOn::Scope(scope)) => {
5593 let selector = subc_protocol::scope::ScopeSelector::new(
5594 principal(module),
5595 scope.scope_ref.clone(),
5596 Some(scope.epoch),
5597 );
5598 self.scopes
5599 .read()
5600 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5601 .attribute(&principal(module), module, &selector, true)
5602 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5603 }
5604 Some(ActivityOn::Route(route)) => {
5605 let DataRoute::Module(DataRouteState::Bound(binding)) = self
5606 .forwarding
5607 .lookup_data_route(connection, route.channel, route.epoch)
5608 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5609 else {
5610 return Err(ACTIVITY_ACTOR_UNKNOWN);
5611 };
5612 if binding.module_id != module {
5613 return Err(ACTIVITY_ACTOR_UNKNOWN);
5614 }
5615 actor.principal = binding.principal.clone();
5616 actor.project_root = binding
5617 .project_root
5618 .as_ref()
5619 .map(|root| root.as_path().to_path_buf());
5620 let Some(bound) = &binding.scope else {
5621 return Ok(actor);
5622 };
5623 let owner = principal(&bound.owner);
5624 let selector = subc_protocol::scope::ScopeSelector::new(
5625 owner.clone(),
5626 bound.scope_ref.clone(),
5627 Some(bound.tag.scope_epoch),
5628 );
5629 let admitted = self
5630 .scopes
5631 .read()
5632 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5633 .attribute(&owner, module, &selector, true)
5634 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
5635 if admitted.tag != bound.tag {
5636 return Err(ACTIVITY_ACTOR_UNKNOWN);
5637 }
5638 admitted
5639 }
5640 };
5641 actor.scope_owner = Some(scope.stamp.owner);
5642 actor.agent_id = scope.stamp.attributes.agent_id;
5643 actor.run_id = scope.stamp.attributes.run_id;
5644 actor.flow_id = scope.stamp.attributes.flow_id;
5645 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
5646 actor.project_id = scope.stamp.project_id;
5647 Ok(actor)
5648 }
5649
5650 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5651 self.refresh_capability_requirements();
5652 let mut counters = self.counters.snapshot();
5658 let losses = self.resource_tracker.loss_diagnostics();
5659 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
5660 if let Some(obj) = counters.as_object_mut() {
5661 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
5662 }
5663 }
5664 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
5665 obj.insert(
5666 "activity_unparseable_lines".into(),
5667 ledger.read_errors().into(),
5668 );
5669 obj.insert(
5670 "activity_lifecycle_append_failures".into(),
5671 ledger.lifecycle_failures().into(),
5672 );
5673 obj.insert(
5674 "activity_started_digest_skips".into(),
5675 ledger.lifecycle_digest_skips().into(),
5676 );
5677 }
5678 if let (Ok((connections_with_routes, max)), Some(obj)) = (
5679 self.forwarding.client_route_concentration(),
5680 counters.as_object_mut(),
5681 ) {
5682 obj.insert(
5683 "client_connections_with_routes".into(),
5684 connections_with_routes.into(),
5685 );
5686 obj.insert("max_routes_on_one_connection".into(), max.into());
5687 }
5688 if let (Some(open_breakers), Some(obj)) = (
5694 self.route_bind_breakers.open_snapshot(),
5695 counters.as_object_mut(),
5696 ) {
5697 obj.insert("route_bind_breakers_open".into(), open_breakers);
5698 }
5699 if let (Some(reason), Some(obj)) = (
5704 self.local_endpoint_unavailable.clone(),
5705 counters.as_object_mut(),
5706 ) {
5707 obj.insert("local_endpoint_unavailable".into(), reason.into());
5708 }
5709 let response = ClientControlResponse::ServerDescribe {
5710 protocol_ver: PROTOCOL_VERSION,
5711 subc_ops: subc_ops(),
5712 capabilities: self.describe_capabilities(),
5713 connected_clients: self.connected_clients.count(),
5714 counters: Some(counters),
5715 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
5716 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
5717 capability_requirements: self.capability_requirement_statuses(),
5718 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
5719 config_path: self
5720 .rescan
5721 .as_ref()
5722 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
5723 };
5724 Ok(vec![control_response_body_frame(
5725 &frame,
5726 &response,
5727 "ClientControlResponse::ServerDescribe",
5728 )?])
5729 }
5730
5731 fn handle_catalog_list(
5732 &self,
5733 frame: Frame,
5734 module_id: Option<String>,
5735 ) -> Result<Vec<Frame>, RouterError> {
5736 let (generation, modules) = self
5737 .registry
5738 .list_modules_with_nonce_verification()
5739 .map_err(|err| {
5740 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
5741 })?;
5742 let entries = modules
5743 .into_iter()
5744 .filter(|(registration, _)| {
5745 module_id
5746 .as_deref()
5747 .map(|wanted| registration.manifest.module_id == wanted)
5748 .unwrap_or(true)
5749 })
5750 .map(|(registration, nonce_verified)| {
5751 let not_ready = self.not_ready_reason(®istration);
5752 let roles = registration.manifest.provides;
5753 CatalogEntry::new(
5754 registration.manifest.module_id,
5755 roles,
5756 registration.control_ops,
5757 )
5758 .with_ready(not_ready.is_none())
5759 .with_display_name(registration.manifest.display_name)
5760 .with_not_ready(not_ready)
5761 .with_module_version(Some(registration.manifest.module_version))
5762 .with_provenance(registration.manifest.provenance)
5763 .with_capabilities(registration.manifest.capabilities)
5764 .with_self_signals(registration.manifest.self_signals)
5765 .with_events(registration.manifest.events)
5766 .with_operation_delivery(registration.manifest.operation_delivery)
5767 .with_operation_payload(registration.manifest.operation_payload)
5768 .with_flow_operations(registration.manifest.flow_operations)
5769 .with_operation_deferral(registration.manifest.operation_deferral)
5770 .with_confirm_operations(registration.manifest.confirm_operations)
5771 .with_rich_ui(registration.manifest.rich_ui)
5772 .with_role_declarations(registration.manifest.role_declarations)
5773 .with_nonce_verified(nonce_verified)
5774 })
5775 .collect();
5776 let response = ClientControlResponse::CatalogList {
5777 generation,
5778 modules: entries,
5779 subc_ops: subc_ops(),
5780 };
5781 Ok(vec![control_response_body_frame(
5782 &frame,
5783 &response,
5784 "ClientControlResponse::CatalogList",
5785 )?])
5786 }
5787
5788 fn attest_route_open_consumer(
5793 &self,
5794 connection_id: ConnectionId,
5795 consumer_identity: Option<&ConsumerIdentity>,
5796 ) -> Option<Principal> {
5797 if let Some(binding) = self.claims.binding(connection_id) {
5798 return (consumer_identity.is_none() && binding.consumer_live(connection_id)).then(
5799 || Principal::Reserved {
5800 module_id: binding.registration.module_id.clone(),
5801 },
5802 );
5803 }
5804 let Some(consumer_identity) = consumer_identity else {
5805 return Some(Principal::Direct);
5806 };
5807 self.supervisor
5808 .spawned_consumer_authorized(
5809 &consumer_identity.module_id,
5810 &consumer_identity.launch_nonce,
5811 )
5812 .then(|| Principal::Reserved {
5813 module_id: consumer_identity.module_id.clone(),
5814 })
5815 }
5816
5817 fn route_open_caller_from_frame(
5821 &self,
5822 connection_id: ConnectionId,
5823 frame: &Frame,
5824 ) -> RouteOpenCaller {
5825 let Ok(ClientControlRequest::RouteOpen {
5826 identity,
5827 consumer_identity,
5828 scope,
5829 ..
5830 }) = parse_client_control_request(&frame.body)
5831 else {
5832 return RouteOpenCaller::unknown();
5833 };
5834 let principal = self.attest_route_open_consumer(connection_id, consumer_identity.as_ref());
5835 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
5836 }
5837
5838 fn route_open_principal(
5839 &self,
5840 connection_id: ConnectionId,
5841 frame: &Frame,
5842 consumer_identity: Option<ConsumerIdentity>,
5843 ) -> Result<Result<Principal, Frame>, RouterError> {
5844 let attested = self.attest_route_open_consumer(connection_id, consumer_identity.as_ref());
5845 self.route_open_principal_attested(frame, consumer_identity, attested)
5846 }
5847
5848 fn route_open_principal_attested(
5852 &self,
5853 frame: &Frame,
5854 consumer_identity: Option<ConsumerIdentity>,
5855 attested: Option<Principal>,
5856 ) -> Result<Result<Principal, Frame>, RouterError> {
5857 if let Some(principal) = attested {
5858 return Ok(Ok(principal));
5859 }
5860 Ok(Err(control_error_frame(
5861 frame,
5862 "bad_consumer_identity",
5863 consumer_identity.map_or_else(
5864 || "connection claim is no longer authorized for routes".to_string(),
5865 |identity| format!(
5866 "consumer_identity for module_id '{}' conflicts with its connection claim or did not match a supervised launch nonce",
5867 identity.module_id
5868 ),
5869 ),
5870 )?))
5871 }
5872
5873 #[allow(clippy::too_many_arguments)]
5880 fn route_open_refusal_frame(
5881 &self,
5882 ctx: &RouteCtx,
5883 caller: &RouteOpenCaller,
5884 frame: &Frame,
5885 module_id: &str,
5886 reason: &'static str,
5887 code: &'static str,
5888 message: impl Into<String>,
5889 ) -> Result<Frame, RouterError> {
5890 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
5891 let detail = if matches!(
5892 reason,
5893 "registration_not_active"
5894 | "supervisor_not_live"
5895 | "no_forwarding_connection"
5896 | "reloading"
5897 ) {
5898 let expected_back = (reason == "reloading").then_some(true);
5901 self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
5902 } else {
5903 None
5904 };
5905 control_error_body_frame(
5906 frame,
5907 ErrorBody {
5908 code: code.to_string(),
5909 message: message.into(),
5910 detail,
5911 },
5912 )
5913 }
5914
5915 fn route_open_recovery_detail(
5916 &self,
5917 module_id: &str,
5918 corr: u64,
5919 detail: Option<serde_json::Value>,
5920 expected_back: Option<bool>,
5921 ) -> Result<Option<serde_json::Value>, RouterError> {
5922 let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
5923 return Ok(detail);
5924 };
5925 let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
5926 detail["state"] = serde_json::json!(status.state.to_string());
5927 if let Some(expected_back) = expected_back {
5928 detail["expected_back"] = serde_json::json!(expected_back);
5929 } else if let Some(module) = self.supervisor.get(module_id) {
5930 match module.will_recover_after_connection_loss() {
5931 Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
5932 Err(err) => warn!(
5933 %module_id,
5934 error = %err,
5935 "failed to read route.open recovery verdict; omitting expected_back"
5936 ),
5937 }
5938 }
5939 Ok(Some(detail))
5940 }
5941
5942 #[allow(clippy::too_many_arguments)]
5951 fn route_bind_rejection(
5952 &self,
5953 ctx: &RouteCtx,
5954 caller: &RouteOpenCaller,
5955 frame: &Frame,
5956 target_module_id: &str,
5957 body: ErrorBody,
5958 daemon_authored: bool,
5959 ) -> Result<Vec<Frame>, RouterError> {
5960 if daemon_authored {
5966 let scope_code = match body.code.as_str() {
5967 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5968 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5969 _ => None,
5970 };
5971 if let Some(code) = scope_code {
5972 self.observe_route_open_refusal(
5973 ctx,
5974 caller,
5975 target_module_id,
5976 "scope_changed_before_commit",
5977 code,
5978 );
5979 return Ok(vec![control_error_body_frame(frame, body)?]);
5980 }
5981 }
5982 self.counters
5983 .increment_route_open_refused("module_rejected");
5984 info!(
5985 target: "control",
5986 code = "module_rejected",
5987 module_code = ?body.code,
5988 module_id = ?target_module_id,
5989 connection_id = ctx.connection_id.get(),
5990 principal = %caller.principal,
5991 scope = %caller.scope,
5992 harness = %caller.harness,
5993 "route.open refused"
5994 );
5995 let refusal = if daemon_authored {
5996 control_error_body_frame(frame, body)?
5997 } else {
5998 control_relayed_error_body_frame(frame, body)?
5999 };
6000 Ok(vec![refusal])
6001 }
6002
6003 #[allow(clippy::too_many_arguments)]
6021 fn route_open_breaker_refusal_frame(
6022 &self,
6023 ctx: &RouteCtx,
6024 caller: &RouteOpenCaller,
6025 frame: &Frame,
6026 module_id: &str,
6027 consecutive_timeouts: u32,
6028 retry_in: Duration,
6029 probe_in_flight: bool,
6030 ) -> Result<Frame, RouterError> {
6031 self.counters
6032 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
6033 debug!(
6034 target: "control",
6035 code = "module_timeout",
6036 module_id = ?module_id,
6037 connection_id = ctx.connection_id.get(),
6038 principal = %caller.principal,
6039 scope = %caller.scope,
6040 harness = %caller.harness,
6041 consecutive_timeouts,
6042 retry_in_ms = retry_in.as_millis() as u64,
6043 probe_in_flight,
6044 "route.open refused by open bind-relay breaker"
6045 );
6046 let detail = if probe_in_flight {
6053 "one test route is already being tried; retry once it settles".to_string()
6054 } else {
6055 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
6056 };
6057 control_error_frame(
6058 frame,
6059 "module_timeout",
6060 format!(
6061 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
6062 timed out in a row); {detail}; its established routes are unaffected"
6063 ),
6064 )
6065 }
6066
6067 fn observe_route_open_refusal(
6080 &self,
6081 ctx: &RouteCtx,
6082 caller: &RouteOpenCaller,
6083 module_id: &str,
6084 reason: &'static str,
6085 code: &'static str,
6086 ) {
6087 self.counters.increment_route_open_refused(code);
6088 info!(
6089 target: "control",
6090 code,
6091 reason,
6092 module_id = ?module_id,
6093 connection_id = ctx.connection_id.get(),
6094 principal = %caller.principal,
6095 scope = %caller.scope,
6096 harness = %caller.harness,
6097 "route.open refused"
6098 );
6099 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
6100 self.route_outages.record_not_serving(module_id, reason);
6101 }
6102 }
6103
6104 fn observe_route_open_accept(
6149 &self,
6150 ctx: &RouteCtx,
6151 module_id: &str,
6152 principal: &str,
6153 scope: &str,
6154 ) {
6155 self.route_outages.record_accepted(module_id);
6156 self.counters.increment_route_open_accepted(principal);
6157 info!(
6158 target: "control",
6159 principal,
6160 module_id,
6161 connection_id = ctx.connection_id.get(),
6162 scope = %scope,
6163 "route.open accepted"
6164 );
6165 }
6166
6167 fn supervised_absent_route_open_refusal_frame(
6168 &self,
6169 ctx: &RouteCtx,
6170 caller: &RouteOpenCaller,
6171 frame: &Frame,
6172 module_id: &str,
6173 code: &'static str,
6174 status: &crate::supervise::ModuleStatus,
6175 ) -> Result<Frame, RouterError> {
6176 self.counters.increment_route_open_refused(code);
6177 info!(
6178 target: "control",
6179 code,
6180 reason = "supervised_not_registered",
6181 module_id = ?module_id,
6182 connection_id = ctx.connection_id.get(),
6183 principal = %caller.principal,
6184 scope = %caller.scope,
6185 harness = %caller.harness,
6186 state = %status.state,
6187 enabled = status.enabled,
6188 live = status.live,
6189 "route.open refused"
6190 );
6191 self.route_outages
6195 .record_not_serving(module_id, "supervised_not_registered");
6196 let message = if let Some(pid) = status.waiting_for_previous_pid {
6197 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
6198 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
6199 })
6200 } else {
6201 format!(
6202 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
6203 status.state, status.enabled, status.live
6204 )
6205 };
6206 let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
6210 control_error_body_frame(
6211 frame,
6212 ErrorBody {
6213 code: code.to_string(),
6214 message,
6215 detail: self.route_open_recovery_detail(
6216 module_id,
6217 frame.header.corr,
6218 None,
6219 expected_back,
6220 )?,
6221 },
6222 )
6223 }
6224
6225 async fn handle_route_open(
6226 &self,
6227 ctx: &RouteCtx,
6228 frame: Frame,
6229 request: RouteOpenRequest,
6230 timings: &mut RouteOpenTimings,
6231 ) -> Result<Vec<Frame>, RouterError> {
6232 let RouteOpenRequest {
6233 target,
6234 mut identity,
6235 consumer_identity,
6236 consumer_capabilities,
6237 role_versions,
6238 busy_refusals,
6239 admission_facts,
6240 scope,
6241 } = request;
6242 let target_module_id = target_module_id(&target).to_string();
6243 timings.module_id.clone_from(&target_module_id);
6244 if self
6245 .registry
6246 .get_module_by_connection(ctx.connection_id)
6247 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6248 .is_some()
6249 {
6250 return Ok(vec![control_error_frame(
6251 &frame,
6252 "invalid_request",
6253 "module connections cannot open client routes",
6254 )?]);
6255 }
6256 debug!(
6257 connection_id = ctx.connection_id.get(),
6258 corr = frame.header.corr,
6259 module_id = %target_module_id,
6260 "handling route.open"
6261 );
6262
6263 let attested =
6268 self.attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref());
6269 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
6270
6271 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
6277 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
6278 self.observe_route_open_refusal(
6279 ctx,
6280 &caller,
6281 &target_module_id,
6282 "invalid_role_versions",
6283 error_codes::INVALID_REQUEST,
6284 );
6285 return Ok(vec![control_error_body_frame(
6286 &frame,
6287 ErrorBody {
6288 code: error_codes::INVALID_REQUEST.to_string(),
6289 message: error.to_string(),
6290 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
6291 },
6292 )?]);
6293 }
6294
6295 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
6296 if status.waiting_for_previous_pid.is_some() {
6297 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6298 ctx,
6299 &caller,
6300 &frame,
6301 &target_module_id,
6302 "module_warming",
6303 &status,
6304 )?]);
6305 }
6306 }
6307
6308 let admission_registry = self
6327 .registry
6328 .route_admission_snapshot()
6329 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6330 let Some(registration) =
6331 crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
6332 else {
6333 if let Some((status, warming)) =
6334 self.supervisor_status(&target_module_id, frame.header.corr)?
6335 {
6336 if status.protocol == ModuleProtocol::None {
6343 return Ok(vec![self.route_open_refusal_frame(
6344 ctx,
6345 &caller,
6346 &frame,
6347 &target_module_id,
6348 "protocol_none",
6349 error_codes::MODULE_NO_PROTOCOL,
6350 format!(
6351 "module_id '{target_module_id}' is declared protocol: none; \
6352 it speaks no subc wire and serves no routes"
6353 ),
6354 )?]);
6355 }
6356 let code = if warming {
6357 "module_warming"
6358 } else {
6359 "target_unavailable"
6360 };
6361 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6362 ctx,
6363 &caller,
6364 &frame,
6365 &target_module_id,
6366 code,
6367 &status,
6368 )?]);
6369 }
6370 if let Some(removed_ago_ms) =
6371 self.supervisor.removal_tombstone_age_ms(&target_module_id)
6372 {
6373 return Ok(vec![self.route_open_refusal_frame(
6374 ctx,
6375 &caller,
6376 &frame,
6377 &target_module_id,
6378 "removed",
6379 error_codes::MODULE_REMOVED,
6380 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
6381 )?]);
6382 }
6383 return Ok(vec![self.route_open_refusal_frame(
6384 ctx,
6385 &caller,
6386 &frame,
6387 &target_module_id,
6388 "not_registered",
6389 error_codes::UNKNOWN_MODULE,
6390 format!("module_id '{target_module_id}' is not registered"),
6391 )?]);
6392 };
6393
6394 if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
6398 .is_err()
6399 {
6400 self.counters
6401 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
6402 info!(
6403 target: "control",
6404 code = error_codes::MODULE_WARMING,
6405 module_id = ?target_module_id,
6406 connection_id = ctx.connection_id.get(),
6407 principal = %caller.principal,
6408 scope = %caller.scope,
6409 harness = %caller.harness,
6410 reason = "declared_not_ready",
6411 "route.open refused"
6412 );
6413 self.route_outages
6416 .record_not_serving(&target_module_id, "declared_not_ready");
6417 return Ok(vec![control_error_body_frame(
6420 &frame,
6421 ErrorBody {
6422 code: error_codes::MODULE_WARMING.to_string(),
6423 message: format!(
6424 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
6425 ),
6426 detail: self.route_open_recovery_detail(
6427 &target_module_id,
6428 frame.header.corr,
6429 Some(serde_json::json!({ "reason": "declared_not_ready" })),
6430 Some(true),
6431 )?,
6432 },
6433 )?]);
6434 }
6435
6436 if let Some(capability) = self.first_unprovided_required_capability(registration) {
6455 self.counters
6456 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
6457 info!(
6458 target: "control",
6459 code = error_codes::MODULE_WARMING,
6460 module_id = ?target_module_id,
6461 connection_id = ctx.connection_id.get(),
6462 principal = %caller.principal,
6463 scope = %caller.scope,
6464 harness = %caller.harness,
6465 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6466 capability = %capability,
6467 "route.open refused"
6468 );
6469 return Ok(vec![control_error_body_frame(
6470 &frame,
6471 ErrorBody {
6472 code: error_codes::MODULE_WARMING.to_string(),
6473 message: format!(
6474 "module_id '{target_module_id}' requires capability '{capability}', \
6475 which no registered module provides; retry"
6476 ),
6477 detail: Some(serde_json::json!({
6478 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6479 "capability": capability,
6480 })),
6481 },
6482 )?]);
6483 }
6484
6485 if !target_has_required_role(&target, ®istration.manifest.provides) {
6486 return Ok(vec![self.route_open_refusal_frame(
6487 ctx,
6488 &caller,
6489 &frame,
6490 &target_module_id,
6491 "role_not_provided",
6492 "target_unavailable",
6493 format!("module_id '{target_module_id}' does not provide the requested target"),
6494 )?]);
6495 }
6496
6497 if registration.state != ChannelState::Active {
6498 return Ok(vec![self.route_open_refusal_frame(
6499 ctx,
6500 &caller,
6501 &frame,
6502 &target_module_id,
6503 "registration_not_active",
6504 "target_unavailable",
6505 format!("module_id '{target_module_id}' is not active"),
6506 )?]);
6507 }
6508
6509 if self
6510 .forwarding
6511 .module_is_draining(&target_module_id)
6512 .map_err(RouterError::Forwarding)?
6513 {
6514 return Ok(vec![self.route_open_refusal_frame(
6515 ctx,
6516 &caller,
6517 &frame,
6518 &target_module_id,
6519 "reloading",
6520 "module_reloading",
6521 format!("module_id '{target_module_id}' is reloading"),
6522 )?]);
6523 }
6524
6525 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
6526 process_liveness.process_live(&target_module_id) == Some(false)
6527 }) {
6528 if process_liveness.process_replacing(&target_module_id) {
6535 return Ok(vec![self.route_open_refusal_frame(
6536 ctx,
6537 &caller,
6538 &frame,
6539 &target_module_id,
6540 "reloading",
6541 "module_reloading",
6542 format!("module_id '{target_module_id}' is reloading"),
6543 )?]);
6544 }
6545 return Ok(vec![self.route_open_refusal_frame(
6546 ctx,
6547 &caller,
6548 &frame,
6549 &target_module_id,
6550 "supervisor_not_live",
6551 "target_unavailable",
6552 format!("module_id '{target_module_id}' is not live"),
6553 )?]);
6554 }
6555
6556 if !self
6557 .forwarding
6558 .has_live_module_connection(&target_module_id)
6559 .map_err(RouterError::Forwarding)?
6560 {
6561 return Ok(vec![self.route_open_refusal_frame(
6562 ctx,
6563 &caller,
6564 &frame,
6565 &target_module_id,
6566 "no_forwarding_connection",
6567 "target_unavailable",
6568 format!("module_id '{target_module_id}' has no live forwarding connection"),
6569 )?]);
6570 }
6571
6572 if let Some(error) =
6573 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
6574 {
6575 self.observe_route_open_refusal(
6576 ctx,
6577 &caller,
6578 &target_module_id,
6579 "op_not_allowed",
6580 "op_not_allowed",
6581 );
6582 return Ok(vec![error]);
6583 }
6584
6585 let principal =
6586 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
6587 Ok(principal) => principal,
6588 Err(error) => {
6589 self.observe_route_open_refusal(
6590 ctx,
6591 &caller,
6592 &target_module_id,
6593 "bad_consumer_identity",
6594 "bad_consumer_identity",
6595 );
6596 return Ok(vec![error]);
6597 }
6598 };
6599 timings.principal = principal_label(&principal);
6600
6601 let build_class = identity.build_class.take().filter(|build_class| {
6604 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
6605 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
6606 } else {
6607 if self
6608 .build_class_warned
6609 .lock()
6610 .unwrap_or_else(|p| p.into_inner())
6611 .insert(ctx.connection_id)
6612 {
6613 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
6614 "ignoring build_class from a non-Callosum opener");
6615 }
6616 false
6617 }
6618 });
6619
6620 if let Principal::Reserved {
6624 module_id: opening_module_id,
6625 } = &principal
6626 {
6627 if let Some(opening_registration) = admission_registry
6628 .iter()
6629 .find(|r| r.manifest.module_id == *opening_module_id)
6630 {
6631 if let Some(capability) =
6632 denied_capability(&opening_registration.manifest, ®istration.manifest)
6633 {
6634 warn!(
6635 opening_module_id,
6636 target_module_id,
6637 capability,
6638 "refusing route.open because an attested capability deny edge matches"
6639 );
6640 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6641 &frame,
6642 &target_module_id,
6643 "capability_deny_edge",
6644 "capability_forbidden",
6645 format!(
6646 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
6647 ),
6648 )?]);
6649 }
6650 }
6651 }
6652
6653 if admission_facts.is_some() {
6654 let carrier_matches = matches!(
6655 &principal,
6656 Principal::Reserved { module_id }
6657 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
6658 );
6659 if !carrier_matches {
6660 return Ok(vec![self.route_open_refusal_frame(
6661 ctx,
6662 &caller,
6663 &frame,
6664 &target_module_id,
6665 "admission_facts_carrier_not_permitted",
6666 "admission_facts_not_permitted",
6667 "admission facts may only be carried by the configured reserved module",
6668 )?]);
6669 }
6670
6671 let target_allowed = self
6672 .admission_facts_targets
6673 .as_ref()
6674 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
6675 if !target_allowed {
6676 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6677 &frame,
6678 &target_module_id,
6679 "admission_facts_target_not_listed",
6680 "admission_facts_target_not_allowed",
6681 format!(
6682 "admission facts are not permitted for target module_id '{target_module_id}'"
6683 ),
6684 )?]);
6685 }
6686
6687 }
6691
6692 let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
6698 None => (None, None, false),
6699 Some(selector) => {
6700 let owner_configured = match &selector.owner {
6701 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
6707 _ => false,
6708 };
6709 let mut scopes = self
6710 .scopes
6711 .write()
6712 .unwrap_or_else(|poisoned| poisoned.into_inner());
6713 let admitted = crate::scopes::admission::admit_route(
6714 &scopes,
6715 &admission_registry,
6716 &principal,
6717 &target_module_id,
6718 Some(selector),
6719 owner_configured,
6720 crate::scopes::admission::AdmissionMode::Open,
6721 )
6722 .map(|route| {
6723 let admission = route.scope.expect("scoped admission");
6724 let bound = scopes.reserve_bind(
6725 &admission,
6726 &principal,
6727 ctx.connection_id,
6728 selector.hold_presence,
6729 );
6730 (admission, bound)
6731 });
6732 match admitted {
6733 Ok((admission, bound)) => {
6734 (Some(bound), Some(admission.stamp), admission.stamp_required)
6735 }
6736 Err(refusal) => {
6737 return Ok(vec![self.route_open_refusal_frame(
6738 ctx,
6739 &caller,
6740 &frame,
6741 &target_module_id,
6742 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
6743 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
6744 refusal.message,
6745 )?]);
6746 }
6747 }
6748 }
6749 };
6750 if let Some(stamp) = &scope_stamp {
6753 caller.scope = route_open_scope_label(stamp);
6754 }
6755
6756 timings.enter(RouteOpenPhase::ProjectRoot);
6764 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
6765 Ok(project) => project,
6766 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6767 };
6768 timings.enter(RouteOpenPhase::Admission);
6769 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
6770 Ok(root) => root,
6771 Err(err) => {
6772 return Ok(vec![control_error_frame(
6773 &frame,
6774 "invalid_project_root",
6775 err.to_string(),
6776 )?])
6777 }
6778 };
6779 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
6785
6786 let _concurrency_guard = match self
6797 .route_bind_concurrency
6798 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
6799 {
6800 Ok(guard) => guard,
6801 Err(in_flight) => {
6802 return Ok(vec![self.route_open_target_capacity_refusal(
6803 ctx,
6804 &caller,
6805 &frame,
6806 &target_module_id,
6807 in_flight,
6808 )?]);
6809 }
6810 };
6811
6812 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
6815 RouteBindAdmission::Admitted { guard, probe } => {
6816 if probe {
6817 info!(
6818 module_id = %target_module_id,
6819 connection_id = ctx.connection_id.get(),
6820 "route.bind breaker half-open: admitting one probe"
6821 );
6822 }
6823 guard
6824 }
6825 RouteBindAdmission::Refused {
6826 consecutive_timeouts,
6827 retry_in,
6828 probe_in_flight,
6829 } => {
6830 return Ok(vec![self.route_open_breaker_refusal_frame(
6831 ctx,
6832 &caller,
6833 &frame,
6834 &target_module_id,
6835 consecutive_timeouts,
6836 retry_in,
6837 probe_in_flight,
6838 )?]);
6839 }
6840 };
6841
6842 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
6847 let relay_deadline = Instant::now() + route_bind_relay_timeout;
6848 let operator_app = self
6849 .operator_apps
6850 .lock()
6851 .unwrap_or_else(|p| p.into_inner())
6852 .get(&ctx.connection_id)
6853 .cloned();
6854 timings.enter(RouteOpenPhase::EgressReservation);
6855 let pending = match self
6856 .forwarding
6857 .begin_route_bind_relay_for(
6858 ctx.connection_id,
6859 ctx.egress.clone(),
6860 response_version(&frame),
6861 frame.header.corr,
6862 &target_module_id,
6863 principal.clone(),
6864 bound_scope,
6865 project_root,
6866 identity.harness.clone(),
6867 build_class,
6868 operator_app.clone(),
6869 relay_deadline,
6870 busy_refusals == Some(true),
6871 )
6872 .await
6873 {
6874 Ok(pending) => pending,
6875 Err(err) => {
6876 return Ok(vec![self.route_open_refusal_frame(
6877 ctx,
6878 &caller,
6879 &frame,
6880 &target_module_id,
6881 "relay_reservation_failed",
6882 forwarding_error_code(&err),
6883 err.to_string(),
6884 )?])
6885 }
6886 };
6887 timings.enter(RouteOpenPhase::Admission);
6888 let crate::forwarding::PendingRouteBindRelay {
6889 endpoint,
6890 module_sink,
6891 negotiated_ver,
6892 client_channel,
6893 client_epoch,
6894 module_channel,
6895 module_epoch,
6896 corr: relay_corr,
6897 receiver,
6898 } = pending;
6899 let mut reservation =
6900 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
6901
6902 let relay_registration = self
6907 .registry
6908 .get_module_by_connection(endpoint.connection_id)
6909 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6910 let relay_capabilities = relay_registration
6911 .as_ref()
6912 .and_then(|registration| registration.manifest.capabilities.as_ref());
6913 if let Some(selector) = &scope {
6914 if let Err(refusal) = self
6917 .scopes
6918 .read()
6919 .unwrap_or_else(|poisoned| poisoned.into_inner())
6920 .check_bind_proof(&principal, selector)
6921 {
6922 reservation.release_and_disarm();
6923 return Ok(vec![self.route_open_refusal_frame(
6924 ctx,
6925 &caller,
6926 &frame,
6927 &target_module_id,
6928 refusal.code,
6929 refusal.code,
6930 refusal.message,
6931 )?]);
6932 }
6933 }
6934 if let Some(stamp) = scope_stamp.as_ref() {
6935 if let Err(refusal) = crate::scopes::check_target_required_stamp(
6936 stamp_required,
6937 &target_module_id,
6938 relay_capabilities,
6939 )
6940 .and_then(|()| {
6941 crate::scopes::check_target_flow_support(
6942 stamp,
6943 &target_module_id,
6944 relay_capabilities,
6945 )
6946 })
6947 .and_then(|()| {
6948 crate::scopes::check_target_agent_run_support(
6949 stamp,
6950 &target_module_id,
6951 relay_capabilities,
6952 )
6953 }) {
6954 reservation.release_and_disarm();
6955 return Ok(vec![self.route_open_refusal_frame(
6956 ctx,
6957 &caller,
6958 &frame,
6959 &target_module_id,
6960 refusal.code,
6961 refusal.code,
6962 refusal.message,
6963 )?]);
6964 }
6965 }
6966
6967 if scope_stamp
6970 .as_ref()
6971 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
6972 && !crate::scopes::target_supports_federated(relay_capabilities)
6973 {
6974 scope_stamp = None;
6976 }
6977 if !relay_capabilities.is_some_and(|caps| {
6978 caps.provides
6979 .iter()
6980 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
6981 }) {
6982 if let Some(stamp) = scope_stamp.as_mut() {
6983 stamp.workdir = None;
6984 stamp.project_id = None;
6985 }
6986 }
6987 let worker_run = scope_stamp
6992 .as_ref()
6993 .is_some_and(crate::scopes::is_worker_run)
6994 && relay_capabilities.is_some_and(|caps| {
6995 caps.provides
6996 .iter()
6997 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
6998 });
6999 if !worker_run
7000 && !relay_capabilities.is_some_and(|caps| {
7001 caps.provides
7002 .iter()
7003 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
7004 })
7005 {
7006 if let Some(stamp) = scope_stamp.as_mut() {
7007 stamp.attributes.on_behalf_of_agent = None;
7008 }
7009 }
7010 if !relay_capabilities.is_some_and(|caps| {
7011 caps.provides
7012 .iter()
7013 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
7014 }) {
7015 if let Some(stamp) = scope_stamp.as_mut() {
7016 stamp.attributes.flow_version = None;
7017 stamp.attributes.flow_code_hash = None;
7018 stamp.attributes.flow_install_id = None;
7019 }
7020 }
7021 let call_mode = if worker_run
7022 || relay_capabilities
7023 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
7024 {
7025 Some(derive_call_mode(scope_stamp.as_ref()))
7026 } else {
7027 if let Some(stamp) = scope_stamp.as_mut() {
7028 stamp.expires_at_ms = None;
7029 }
7030 None
7031 };
7032
7033 debug!(
7034 connection_id = ctx.connection_id.get(),
7035 client_channel,
7036 client_epoch,
7037 module_channel,
7038 module_epoch,
7039 "reserved route handle pair"
7040 );
7041 let route_principal_label = principal_label(&principal);
7044 let relay = ModuleControlRequest::RouteBind {
7045 route_channel: module_channel,
7046 epoch: module_epoch,
7047 target,
7048 identity,
7049 principal: Some(principal),
7050 consumer_capabilities,
7051 role_versions,
7052 admission_facts,
7053 scope: scope_stamp,
7054 call_mode,
7055 };
7056 let peer = RouteBindPeer::new()
7057 .with_observed_peer(
7058 relay_capabilities
7059 .is_some_and(|caps| {
7060 caps.provides
7061 .iter()
7062 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
7063 })
7064 .then(|| {
7065 self.observed_peers
7066 .lock()
7067 .unwrap_or_else(|p| p.into_inner())
7068 .get(&ctx.connection_id)
7069 .cloned()
7070 .unwrap_or_else(ObservedPeer::unobserved)
7071 }),
7072 )
7073 .with_operator_app(
7074 self.connection_operator_app(operator_app.as_ref())
7075 .filter(|_| {
7076 relay_capabilities.is_some_and(|caps| {
7077 caps.provides
7078 .iter()
7079 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
7080 })
7081 }),
7082 );
7083 let relay_body = crate::bind_compat::encode_bind(
7084 &relay,
7085 &peer,
7086 &legacy_bind_project,
7087 relay_capabilities,
7088 )
7089 .map_err(|err| {
7090 RouterError::backend(
7091 0,
7092 frame.header.corr,
7093 format!("failed to encode route.bind request: {err}"),
7094 )
7095 })?;
7096 let relay_frame = Frame::build_with_version(
7097 negotiated_ver,
7098 FrameType::Request,
7099 control_flags(),
7100 0,
7101 0,
7102 relay_corr,
7103 relay_body,
7104 )
7105 .map_err(RouterError::FrameBuild)?;
7106
7107 timings.enter(RouteOpenPhase::BindSend);
7108 let sent = module_sink.send(relay_frame).await;
7109 timings.enter(RouteOpenPhase::Admission);
7110 if let Err(err) = sent {
7111 reservation.release_and_disarm();
7112 return Ok(vec![self.route_open_refusal_frame(
7113 ctx,
7114 &caller,
7115 &frame,
7116 &target_module_id,
7117 "relay_send_failed",
7118 "target_unavailable",
7119 err.to_string(),
7120 )?]);
7121 }
7122
7123 #[cfg(test)]
7124 if let Some(hold) = &self.route_bind_relay_sent_hold {
7125 hold.wait().await;
7129 hold.wait().await;
7130 }
7131
7132 if self
7137 .forwarding
7138 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
7139 .map_err(RouterError::Forwarding)?
7140 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
7141 {
7142 self.send_abandoned_route_bind_goodbye(
7143 &module_sink,
7144 negotiated_ver,
7145 module_channel,
7146 module_epoch,
7147 );
7148 }
7149
7150 timings.enter(RouteOpenPhase::BindAckWait);
7151 let outcome = timeout_at(relay_deadline, receiver).await;
7152 timings.enter(RouteOpenPhase::Admission);
7153 match outcome {
7154 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
7155 reservation.disarm();
7156 if breaker.record_accepted() {
7157 info!(
7158 module_id = %target_module_id,
7159 "route.bind breaker closed: the probe was accepted"
7160 );
7161 }
7162 self.observe_route_open_accept(
7163 ctx,
7164 &target_module_id,
7165 &route_principal_label,
7166 &caller.scope,
7167 );
7168 Ok(Vec::new())
7169 }
7170 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
7171 reservation.release_and_disarm();
7172 breaker.record_inconclusive();
7176 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
7177 }
7178 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
7179 reservation.release_and_disarm();
7180 breaker.record_inconclusive();
7181 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
7182 }
7183 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
7184 reservation.release_and_disarm();
7185 breaker.record_inconclusive();
7186 tracing::warn!(
7193 module_id = %target_module_id,
7194 "route.bind relay abandoned: {message}"
7195 );
7196 Ok(vec![self.route_open_refusal_frame(
7197 ctx,
7198 &caller,
7199 &frame,
7200 &target_module_id,
7201 "relay_abandoned",
7202 "target_unavailable",
7203 message,
7204 )?])
7205 }
7206 Ok(Err(_)) => {
7207 reservation.release_and_disarm();
7208 breaker.record_inconclusive();
7209 Ok(vec![self.route_open_refusal_frame(
7210 ctx,
7211 &caller,
7212 &frame,
7213 &target_module_id,
7214 "relay_waiter_canceled",
7215 "target_unavailable",
7216 "route.bind relay waiter was canceled before the module responded",
7217 )?])
7218 }
7219 Err(_) => {
7220 reservation.release_and_disarm();
7221 if let Some(opened) = breaker.record_timeout(
7225 self.route_bind_breaker_threshold,
7226 self.route_bind_breaker_cooldown,
7227 ) {
7228 warn!(
7229 module_id = %target_module_id,
7230 consecutive_timeouts = opened.consecutive_timeouts,
7231 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
7232 reopened_after_probe = opened.reopened_after_probe,
7233 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
7234 );
7235 }
7236 tracing::warn!(
7244 module_id = %target_module_id,
7245 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
7246 "route.bind relay timed out: module did not ack within budget"
7247 );
7248 Ok(vec![self.route_open_refusal_frame(
7249 ctx,
7250 &caller,
7251 &frame,
7252 &target_module_id,
7253 "relay_timed_out",
7254 "module_timeout",
7255 format!(
7256 "module_id '{target_module_id}' did not answer route.bind within {:?}",
7257 route_bind_relay_timeout
7258 ),
7259 )?])
7260 }
7261 }
7262 }
7263
7264 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7265 let response = ClientControlResponse::SupervisorSpawnSnapshot {
7266 snapshot: self.supervisor.spawn_snapshot(),
7267 };
7268 Ok(vec![control_response_body_frame(
7269 &frame,
7270 &response,
7271 "ClientControlResponse::SupervisorSpawnSnapshot",
7272 )?])
7273 }
7274
7275 fn handle_supervisor_spawn_subscribe(
7276 &self,
7277 ctx: &RouteCtx,
7278 frame: Frame,
7279 since: Option<SpawnCursor>,
7280 ) -> Result<Vec<Frame>, RouterError> {
7281 match self.supervisor.subscribe_spawns(
7282 ctx.connection_id,
7283 frame.header.corr,
7284 response_version(&frame),
7285 since,
7286 ctx.egress.clone(),
7287 ) {
7288 Ok(()) => Ok(Vec::new()),
7289 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
7290 Ok(vec![control_error_body_frame(
7291 &frame,
7292 ErrorBody {
7293 code: "spawn_cursor_incarnation_mismatch".to_string(),
7294 message: "spawn cursor belongs to a different daemon incarnation"
7295 .to_string(),
7296 detail: Some(serde_json::json!({
7297 "current_daemon_incarnation": current
7298 })),
7299 },
7300 )?])
7301 }
7302 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
7303 &frame,
7304 ErrorBody {
7305 code: "spawn_cursor_too_old".to_string(),
7306 message: "spawn cursor predates the retained event ring".to_string(),
7307 detail: Some(serde_json::json!({
7308 "oldest_retained_cursor": oldest
7309 })),
7310 },
7311 )?]),
7312 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
7313 0,
7314 frame.header.corr,
7315 format!("failed to open supervisor spawn subscription: {error}"),
7316 )),
7317 }
7318 }
7319
7320 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7321 let handler = self.clone();
7324 let runtime = tokio::runtime::Handle::current();
7325 let subscriber = tracing::dispatcher::get_default(Clone::clone);
7328 let corr = frame.header.corr;
7329 tokio::task::spawn_blocking(move || {
7330 runtime.block_on(
7331 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
7332 .with_subscriber(subscriber),
7333 )
7334 })
7335 .await
7336 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
7337 }
7338
7339 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7340 use crate::config_gate::runtime::ApprovedDaemonView;
7341 let publication = self
7342 .config_runtime
7343 .as_ref()
7344 .and_then(|owner| owner.polled_snapshot().ok());
7345 let approved = publication
7346 .as_ref()
7347 .map(|view| view.approved_daemon.clone())
7348 .unwrap_or_else(|| {
7349 if self.config_runtime.is_some() {
7350 ApprovedDaemonView::Unavailable
7351 } else {
7352 ApprovedDaemonView::NotStamped
7353 }
7354 });
7355 let generation = self
7356 .registry
7357 .generation()
7358 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7359 let supervised = self.supervisor.list();
7360 let mut statuses = BTreeMap::new();
7361 let mut not_running = BTreeSet::new();
7362 for module in &supervised {
7363 let status = module.status_for_control("list").map_err(|err| {
7364 RouterError::backend(
7365 0,
7366 frame.header.corr,
7367 format!("failed to read supervisor status: {err}"),
7368 )
7369 })?;
7370 if !status.enabled || !status.process_alive {
7371 not_running.insert(module.module_id().to_owned());
7372 }
7373 statuses.insert(module.module_id().to_owned(), status);
7374 }
7375 let mut process_resources = BTreeMap::new();
7378 for module in &supervised {
7379 process_resources.insert(
7380 module.module_id().to_owned(),
7381 crate::subtree_resources::read_process(module.clone()).await,
7382 );
7383 }
7384 let mut image_readings = BTreeMap::new();
7387 for module in &supervised {
7388 image_readings.insert(
7389 module.module_id().to_owned(),
7390 module.running_image_agreement().await,
7391 );
7392 }
7393 let readings = self.resource_tracker.list(supervised.clone());
7394 let mut modules = Vec::new();
7395 for module in supervised {
7396 let status = statuses
7397 .remove(module.module_id())
7398 .expect("each supervised module has a status snapshot");
7399 let (configured, _) = module.configuration().map_err(|err| {
7400 RouterError::backend(
7401 0,
7402 frame.header.corr,
7403 format!("failed to read module configuration: {err}"),
7404 )
7405 })?;
7406 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
7407 RouterError::backend(
7408 0,
7409 frame.header.corr,
7410 format!("failed to read retry time: {err}"),
7411 )
7412 })?;
7413 let image = image_readings
7414 .remove(module.module_id())
7415 .expect("each supervised module has an image snapshot");
7416 let resources = process_resources.remove(module.module_id());
7417 let mut verdict =
7418 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
7419 verdict.spec = Some(launch_spec_agreement(
7420 &configured,
7421 status.process_alive,
7422 status.launch_spec_digest.as_ref(),
7423 ));
7424 let pending_reload = Some(verdict);
7425 let configured_enabled = match &approved {
7426 ApprovedDaemonView::Approved {
7427 config: approved, ..
7428 } => approved
7429 .modules
7430 .iter()
7431 .find(|module| module.module_id == status.module_id)
7432 .map(|module| module.enabled),
7433 ApprovedDaemonView::NotStamped => self
7434 .capability_evaluator
7435 .configured_enabled(&status.module_id),
7436 ApprovedDaemonView::Unavailable => None,
7437 };
7438 let admission_refused = status
7439 .admission_refused
7440 .clone()
7441 .filter(|_| status.enabled && !status.process_alive)
7442 .filter(|_| {
7443 publication
7444 .as_ref()
7445 .and_then(|view| view.registration_declaration(&status.module_id))
7446 .as_deref()
7447 == Some("undeclared")
7448 });
7449 modules.push(
7452 SupervisorEntry::new(
7453 status.module_id,
7454 status.state.to_string(),
7455 status.enabled,
7456 status.live,
7457 status.health.status,
7458 )
7459 .with_configured_enabled(configured_enabled)
7460 .with_admission_refused(admission_refused)
7461 .with_next_retry_at_ms(next_retry_at_ms)
7462 .with_launch_nonce_env(Some(!cfg!(unix)))
7463 .with_launch_nonce_source(status.launch_nonce_source)
7464 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
7465 .with_protocol(status.protocol)
7466 .with_pending_reload(pending_reload)
7467 .with_last_probe_ms(status.health.last_probe_ms)
7468 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
7469 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
7470 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
7471 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
7472 .with_restart_count(Some(status.restart_count))
7473 .with_max_restarts(Some(status.max_restarts))
7474 .with_lifetime_restarts(Some(status.lifetime_restarts))
7475 .with_spawn_generation(Some(status.spawn_generation))
7476 .with_restart_window_secs(Some(status.restart_window.as_secs()))
7477 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
7478 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
7479 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
7480 .with_resources(resources),
7481 );
7482 }
7483 for module in &mut modules {
7484 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
7485 module.tree = tree.clone();
7486 module.cpu_last_hour = hour.clone();
7487 }
7488 module.tree_unavailable = readings
7489 .reasons
7490 .get(&module.module_id)
7491 .map(|reason| (*reason).to_owned());
7492 if module.tree.is_none() {
7493 module.tree_unavailable = Some(
7494 if not_running.contains(&module.module_id) {
7495 "not_running"
7496 } else if let Some(reason) = readings.reasons.get(&module.module_id) {
7497 reason
7498 } else {
7499 "not_yet_measured"
7500 }
7501 .to_owned(),
7502 );
7503 }
7504 }
7505 let response = ClientControlResponse::SupervisorList {
7506 generation,
7507 modules,
7508 daemon: readings.daemon,
7509 agent_commands: readings.agent_commands,
7510 };
7511 Ok(vec![control_response_body_frame(
7512 &frame,
7513 &response,
7514 "ClientControlResponse::SupervisorList",
7515 )?])
7516 }
7517
7518 fn handle_supervisor_stderr_tail(
7519 &self,
7520 frame: Frame,
7521 module_id: String,
7522 max_lines: Option<u32>,
7523 max_bytes: Option<u32>,
7524 ) -> Result<Vec<Frame>, RouterError> {
7525 let Some(module) = self.supervisor.get(&module_id) else {
7526 return Ok(vec![control_error_frame(
7527 &frame,
7528 "unknown_module",
7529 format!("module_id '{module_id}' is not supervised"),
7530 )?]);
7531 };
7532
7533 let snapshot = module.stderr_tail(
7534 max_lines.map(|value| value as usize),
7535 max_bytes.map(|value| value as usize),
7536 );
7537
7538 let response = ClientControlResponse::SupervisorStderrTail {
7539 module_id,
7540 tail: StderrTail {
7541 capture: match snapshot.capture {
7542 CaptureState::Captured => StderrCaptureState::Captured,
7543 CaptureState::Incomplete { reason } => {
7544 StderrCaptureState::Incomplete { reason }
7545 }
7546 CaptureState::NotCaptured { reason } => {
7547 StderrCaptureState::NotCaptured { reason }
7548 }
7549 },
7550 entries: snapshot
7551 .entries
7552 .into_iter()
7553 .map(|entry| match entry {
7554 TailEntry::Line {
7555 text,
7556 truncated,
7557 at_ms,
7558 } => StderrTailEntry::Line {
7559 text,
7560 truncated,
7561 at_ms,
7562 },
7563 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
7564 })
7565 .collect(),
7566 dropped_lines: snapshot.dropped_lines,
7567 },
7568 };
7569 Ok(vec![control_response_body_frame(
7570 &frame,
7571 &response,
7572 "ClientControlResponse::SupervisorStderrTail",
7573 )?])
7574 }
7575
7576 async fn observe_control_wait<F: std::future::Future>(
7577 &self,
7578 ctx: &RouteCtx,
7579 target: Option<&str>,
7580 kind: &'static str,
7581 future: F,
7582 ) -> F::Output {
7583 self.counters
7584 .observe_reader_wait(
7585 || {
7586 let module = self
7587 .forwarding
7588 .module_id_for_connection(ctx.connection_id)
7589 .ok()
7590 .flatten();
7591 let principal = if module.is_some() { "module" } else { "direct" };
7592 (
7593 self.resolve_reader_wait_target(kind, target)
7594 .or(module)
7595 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
7596 principal.into(),
7597 )
7598 },
7599 kind,
7600 future,
7601 )
7602 .await
7603 }
7604
7605 pub(crate) fn resolve_reader_wait_target(
7608 &self,
7609 op: &str,
7610 selector: Option<&str>,
7611 ) -> Option<String> {
7612 let selector = selector?;
7613 match op {
7614 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
7615 .supervisor
7616 .get(selector)
7617 .map(|module| module.module_id().to_owned()),
7618 ops::SUPERVISOR_HEALTH_PROBE => self
7619 .registry
7620 .get_module(selector)
7621 .ok()
7622 .flatten()
7623 .map(|registration| registration.manifest.module_id),
7624 _ => None,
7625 }
7626 }
7627
7628 async fn handle_supervisor_terminals(
7629 &self,
7630 ctx: &RouteCtx,
7631 frame: Frame,
7632 module_id: String,
7633 ) -> Result<Vec<Frame>, RouterError> {
7634 let Some(module) = self.supervisor.get(&module_id) else {
7635 return Ok(vec![control_error_frame(
7636 &frame,
7637 "unknown_module",
7638 format!("module_id '{module_id}' is not supervised"),
7639 )?]);
7640 };
7641
7642 let terminals = self
7645 .observe_control_wait(
7646 ctx,
7647 Some(module.module_id()),
7648 ops::SUPERVISOR_TERMINALS,
7649 module.read_durable_terminal_history(),
7650 )
7651 .await
7652 .map_err(|error| {
7653 RouterError::backend(
7654 0,
7655 frame.header.corr,
7656 format!("failed to read terminal history: {error}"),
7657 )
7658 })?;
7659 let response = ClientControlResponse::SupervisorTerminals {
7660 module_id,
7661 terminals,
7662 };
7663 Ok(vec![control_response_body_frame(
7664 &frame,
7665 &response,
7666 "ClientControlResponse::SupervisorTerminals",
7667 )?])
7668 }
7669
7670 fn handle_supervisor_routes(
7671 &self,
7672 frame: Frame,
7673 module_id: Option<String>,
7674 ) -> Result<Vec<Frame>, RouterError> {
7675 let modules = self
7676 .forwarding
7677 .route_census(module_id.as_deref())
7678 .map_err(RouterError::Forwarding)?
7679 .into_iter()
7680 .map(|(module_id, routes)| SupervisorRouteModule {
7681 module_id,
7682 routes: routes
7683 .into_iter()
7684 .map(|route| SupervisorRoute {
7685 consumer: match route.principal {
7686 Principal::Reserved { module_id } => {
7687 SupervisorRouteConsumer::Reserved { module_id }
7688 }
7689 Principal::Direct | Principal::Unverified => {
7690 SupervisorRouteConsumer::Direct {
7691 connection_id: route.goodbye_target.connection_id.get(),
7692 }
7693 }
7694 },
7695 age_ms: Instant::now()
7696 .saturating_duration_since(route.bound_at)
7697 .as_millis()
7698 .try_into()
7699 .unwrap_or(u64::MAX),
7700 draining: route.draining,
7701 drain_reason: route.drain_reason,
7702 })
7703 .collect(),
7704 })
7705 .collect();
7706 let response = ClientControlResponse::SupervisorRoutes { modules };
7707 Ok(vec![control_response_body_frame(
7708 &frame,
7709 &response,
7710 "ClientControlResponse::SupervisorRoutes",
7711 )?])
7712 }
7713
7714 fn running_registration_method(
7715 &self,
7716 module: &str,
7717 ) -> Option<subc_control::RegistrationMethod> {
7718 let registration = self.registry.get_module(module).ok()??;
7719 let connection = registration.connection_id;
7720 let channel = self.supervisor.registration_spawn(connection).is_some();
7721 let nonce = if channel {
7722 None
7723 } else {
7724 self.hello_launch_nonces
7725 .lock()
7726 .ok()?
7727 .nonce(connection)
7728 .map(str::to_owned)
7729 };
7730 if !channel && nonce.is_none() {
7731 return None;
7732 }
7733 self.supervisor
7734 .registration_snapshot_is_current_live(
7735 &self.registry,
7736 module,
7737 connection,
7738 nonce.as_deref(),
7739 )
7740 .then_some(if channel {
7741 subc_control::RegistrationMethod::Channel
7742 } else {
7743 subc_control::RegistrationMethod::Nonce
7744 })
7745 }
7746
7747 async fn handle_supervisor_provenance(
7748 &self,
7749 ctx: &RouteCtx,
7750 frame: Frame,
7751 module_id: Option<String>,
7752 ) -> Result<Vec<Frame>, RouterError> {
7753 let targeted = module_id.is_some();
7754 let selected = if let Some(module_id) = module_id {
7755 let Some(module) = self.supervisor.get(&module_id) else {
7756 return Ok(vec![control_error_frame(
7757 &frame,
7758 "unknown_module",
7759 format!("module_id '{module_id}' is not supervised"),
7760 )?]);
7761 };
7762 vec![module]
7763 } else {
7764 self.supervisor.list()
7765 };
7766
7767 let target = targeted.then(|| selected[0].module_id());
7768 let declarations = if let Some(owner) = &self.config_runtime {
7769 owner
7770 .on_blocking(|owner| owner.registration_declarations())
7771 .await
7772 .ok()
7773 } else {
7774 None
7775 };
7776 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
7777 let mut modules = Vec::with_capacity(selected.len());
7778 let mut extensions = Vec::with_capacity(selected.len());
7779 for module in &selected {
7780 let (status, observed_image) = module
7781 .status_and_running_image_agreement()
7782 .await
7783 .map_err(|err| {
7784 RouterError::backend(
7785 0,
7786 frame.header.corr,
7787 format!("failed to read supervisor status: {err}"),
7788 )
7789 })?;
7790 let module_declared = self
7791 .registry
7792 .get_module(&status.module_id)
7793 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7794 .and_then(|registration| registration.manifest.provenance)
7795 .map(|build| ModuleDeclaredProvenance::Reported { build })
7796 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
7797 #[cfg(test)]
7798 let running_image = match &self.provenance_probe_override {
7799 Some(result) => result.clone(),
7800 None => observed_image,
7801 };
7802 #[cfg(not(test))]
7803 let running_image = observed_image;
7804 let registration = self.running_registration_method(&status.module_id);
7805 let mut extension = subc_control::SupervisorModuleProvenanceExt::new()
7806 .with_registration(registration);
7807 extension.registration_declaration = declarations
7808 .as_ref()
7809 .and_then(|entries| entries.get(&status.module_id))
7810 .cloned();
7811 if status.process_alive && status.registration_declaration.is_some() {
7812 extension.registration_declaration = status.registration_declaration.clone();
7813 }
7814 extensions.push(extension);
7815 modules.push(SupervisorModuleProvenance {
7816 module_id: status.module_id,
7817 module_declared,
7818 daemon_observed: SupervisorObservedProcess {
7819 launch_nonce_source: status.launch_nonce_source,
7820 pid: status.pid,
7821 spawned_at_ms: status.spawned_at_ms,
7822 spawned_from: status.spawned_from,
7823 running_image,
7824 },
7825 });
7826 }
7827 let daemon = SupervisorDaemonProvenance {
7828 daemon_build: self.daemon_provenance.build.clone(),
7829 daemon_observed: DaemonObservedProcess {
7830 pid: self.daemon_provenance.pid,
7831 started_at_ms: self
7832 .daemon_provenance
7833 .start_clock
7834 .map(|clock| clock.started_at_ms())
7835 .or(self.daemon_provenance.started_at_ms),
7836 running_image: self
7837 .daemon_provenance
7838 .probe
7839 .observe(
7840 self.daemon_provenance.pid,
7841 self.daemon_provenance.executable_path.as_deref(),
7842 self.daemon_provenance.executable_identity.clone(),
7843 self.daemon_provenance.process_start_time,
7844 )
7845 .await,
7846 },
7847 };
7848 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
7849 let mut response = serde_json::to_value(response)
7850 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
7851 for (entry, extension) in response["modules"]
7852 .as_array_mut()
7853 .expect("provenance modules are an array")
7854 .iter_mut()
7855 .zip(extensions)
7856 {
7857 extension.write_to(entry).map_err(|error| {
7858 RouterError::backend(0, frame.header.corr, error.to_string())
7859 })?;
7860 }
7861 Ok(vec![control_response_body_frame(
7862 &frame,
7863 &response,
7864 "ClientControlResponse::SupervisorProvenance",
7865 )?])
7866 })
7867 .await
7868 }
7869
7870 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7871 self.refresh_capability_requirements();
7872 let publication = self
7873 .config_runtime
7874 .as_ref()
7875 .and_then(|owner| owner.polled_snapshot().ok());
7876 let generation = self
7877 .registry
7878 .generation()
7879 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7880 let modules = self
7881 .supervisor
7882 .list()
7883 .into_iter()
7884 .map(|module| {
7885 let status = module.status_for_control("health").map_err(|err| {
7886 RouterError::backend(
7887 0,
7888 frame.header.corr,
7889 format!("failed to read supervisor health: {err}"),
7890 )
7891 })?;
7892 let module_id = status.module_id;
7893 let capability_detail = self
7894 .capability_evaluator
7895 .required_problem_detail(&module_id);
7896 let declaration_warning = self.registry.get_module(&module_id).ok().flatten()
7897 .filter(|registration| registration.manifest.registration_channel == Some(false))
7898 .filter(|_| publication.as_ref().and_then(|view| view.registration_declaration(&module_id))
7899 .as_deref().map(|next| if status.process_alive { status.registration_declaration.as_deref().unwrap_or(next) } else { next }) == Some("declared"))
7900 .map(|_| format!("{module_id}: HELLO registration_channel is false despite the approved channel declaration"));
7901 let capability_detail = append_capability_problem_detail(capability_detail, declaration_warning);
7902 Ok(SupervisorHealthEntry {
7903 module_id,
7904 status: status.health.status,
7905 detail: append_capability_problem_detail(
7906 status.health.detail,
7907 capability_detail,
7908 ),
7909 metrics: status.health.metrics,
7910 consecutive_failures: status.health.consecutive_failures,
7911 late_answer_count: status.health.late_answer_count,
7912 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
7913 last_action: status.health.last_action,
7914 last_action_ms: status.health.last_action_ms,
7915 last_probe_ms: status.health.last_probe_ms,
7916 })
7917 })
7918 .collect::<Result<Vec<_>, RouterError>>()?;
7919 let response = ClientControlResponse::SupervisorHealth {
7920 generation,
7921 modules,
7922 };
7923 Ok(vec![control_response_body_frame(
7924 &frame,
7925 &response,
7926 "ClientControlResponse::SupervisorHealth",
7927 )?])
7928 }
7929
7930 async fn handle_supervisor_restart(
7931 &self,
7932 frame: Frame,
7933 module_id: String,
7934 drain_timeout_ms: Option<u64>,
7935 requester: Principal,
7936 ) -> Result<Vec<Frame>, RouterError> {
7937 let operation_lock = self.supervisor.operation_lock();
7938 let _operation_guard = operation_lock.lock().await;
7939 let Some(module) = self.supervisor.get(&module_id) else {
7940 return Ok(vec![control_error_frame(
7941 &frame,
7942 "unknown_module",
7943 format!("module_id '{module_id}' is not supervised"),
7944 )?]);
7945 };
7946
7947 self.route_outages.mark_operator_action(&module_id);
7948 if let Err(err) = module
7949 .restart_requested(drain_timeout_ms, Some(requester))
7950 .await
7951 {
7952 self.route_outages
7953 .operator_action_ended_unrefused(&module_id);
7954 let (code, message) = match err {
7955 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
7956 crate::supervise::SuperviseError::Disabled { .. } => {
7957 ("module_disabled", err.to_string())
7958 }
7959 crate::supervise::SuperviseError::SwapInProgress { .. } => {
7960 ("swap_in_progress", err.to_string())
7961 }
7962 _ => (
7963 "target_unavailable",
7964 format!("failed to restart module_id '{module_id}': {err}"),
7965 ),
7966 };
7967 return Ok(vec![control_error_frame(&frame, code, message)?]);
7968 }
7969
7970 let response = ClientControlResponse::SupervisorAck {
7971 module_id,
7972 applied: true,
7973 };
7974 Ok(vec![control_response_body_frame(
7975 &frame,
7976 &response,
7977 "ClientControlResponse::SupervisorAck",
7978 )?])
7979 }
7980
7981 async fn handle_supervisor_swap(
7985 &self,
7986 frame: Frame,
7987 module_id: String,
7988 ready_timeout_ms: Option<u64>,
7989 requester: Principal,
7990 ) -> Result<Vec<Frame>, RouterError> {
7991 let module = {
7998 let operation_lock = self.supervisor.operation_lock();
7999 let _operation_guard = operation_lock.lock().await;
8000 self.supervisor.get(&module_id)
8001 };
8002 let Some(module) = module else {
8003 return Ok(vec![control_error_frame(
8004 &frame,
8005 "unknown_module",
8006 format!("module_id '{module_id}' is not supervised"),
8007 )?]);
8008 };
8009
8010 self.route_outages.mark_operator_action(&module_id);
8011 if let Err(err) = module
8012 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
8013 .await
8014 {
8015 self.route_outages
8016 .operator_action_ended_unrefused(&module_id);
8017 use crate::supervise::SuperviseError;
8018 let message = err.to_string();
8019 let error = match err {
8020 SuperviseError::ConfigHeld { .. } => ErrorBody::new("config_gated", message),
8021 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
8022 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
8023 code: "swap_refused".to_string(),
8024 message,
8025 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
8026 },
8027 SuperviseError::SwapFailed {
8028 arm,
8029 candidate_exit,
8030 ..
8031 } => ErrorBody {
8032 code: "swap_failed".to_string(),
8033 message,
8034 detail: Some(serde_json::json!({
8035 "arm": arm.as_str(),
8036 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
8037 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
8038 })),
8039 },
8040 _ => ErrorBody::new(
8041 "target_unavailable",
8042 format!("failed to swap module_id '{module_id}': {message}"),
8043 ),
8044 };
8045 return Ok(vec![control_error_body_frame(&frame, error)?]);
8046 }
8047 self.route_outages
8051 .operator_action_ended_unrefused(&module_id);
8052
8053 let response = ClientControlResponse::SupervisorAck {
8054 module_id,
8055 applied: true,
8056 };
8057 Ok(vec![control_response_body_frame(
8058 &frame,
8059 &response,
8060 "ClientControlResponse::SupervisorAck",
8061 )?])
8062 }
8063
8064 async fn handle_supervisor_reload(
8065 &self,
8066 frame: Frame,
8067 module_id: String,
8068 requester: Principal,
8069 ) -> Result<Vec<Frame>, RouterError> {
8070 let operation_lock = self.supervisor.operation_lock();
8071 let _operation_guard = operation_lock.lock().await;
8072 let Some(module) = self.supervisor.get(&module_id) else {
8073 return Ok(vec![control_error_frame(
8074 &frame,
8075 "unknown_module",
8076 format!("module_id '{module_id}' is not supervised"),
8077 )?]);
8078 };
8079
8080 self.route_outages.mark_operator_action(&module_id);
8081 if let Err(err) = module.reload_requested(Some(requester)).await {
8082 self.route_outages
8083 .operator_action_ended_unrefused(&module_id);
8084 let (code, message) = match err {
8085 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8086 crate::supervise::SuperviseError::Disabled { .. } => {
8087 ("module_disabled", err.to_string())
8088 }
8089 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8090 ("swap_in_progress", err.to_string())
8091 }
8092 _ => (
8093 "reload_failed",
8094 format!("failed to reload module_id '{module_id}': {err}"),
8095 ),
8096 };
8097 return Ok(vec![control_error_frame(&frame, code, message)?]);
8098 }
8099
8100 let response = ClientControlResponse::SupervisorAck {
8101 module_id,
8102 applied: true,
8103 };
8104 Ok(vec![control_response_body_frame(
8105 &frame,
8106 &response,
8107 "ClientControlResponse::SupervisorAck",
8108 )?])
8109 }
8110
8111 async fn handle_supervisor_rescan(
8115 &self,
8116 frame: Frame,
8117 mode: RescanMode,
8118 requester: Principal,
8119 ) -> Result<Vec<Frame>, RouterError> {
8120 let preview = mode.previews();
8121 let mut approved_preview = None;
8122 if matches!(mode, RescanMode::Preview) {
8123 if let Some(owner) = &self.config_runtime {
8124 match owner
8125 .on_blocking(|owner| {
8126 owner.observe_file_proposal()?;
8127 owner.approved_daemon_configuration()
8128 })
8129 .await
8130 {
8131 Ok(config) => approved_preview = config,
8132 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8133 }
8134 }
8135 }
8136 let Some(context) = self.rescan.clone() else {
8137 return Ok(vec![control_error_frame(
8138 &frame,
8139 "rescan_unavailable",
8140 "the daemon was not started with a reloadable config path".to_string(),
8141 )?]);
8142 };
8143
8144 let operation_lock = self.supervisor.operation_lock();
8145 let _operation_guard = operation_lock.lock().await;
8146 let loaded = match self.load_rescan_config(&mode, &context) {
8150 Ok(config) => config,
8151 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
8152 };
8153 let Some((config, activity_config)) = loaded else {
8161 return Ok(vec![control_error_frame(
8162 &frame,
8163 "invalid_daemon_config",
8164 format!(
8165 "daemon config not found at {}; refusing to rescan (an absent config would \
8166 retire every supervised module)",
8167 context.config_path.display()
8168 ),
8169 )?]);
8170 };
8171 let (
8172 configured_port,
8173 storage_config,
8174 admission_facts_carrier_module_id,
8175 admission_facts_targets,
8176 scope_authority_owners,
8177 federated_scope_owner,
8178 privileged_operator_prompt,
8179 modules,
8180 reserved_capabilities,
8181 ) = (
8182 config.port,
8183 config.storage,
8184 config.admission_facts_carrier_module_id,
8185 config.admission_facts_targets,
8186 config.scope_authority_owners,
8187 config.federated_scope_owner,
8188 config.privileged_operator_prompt,
8189 config.modules,
8190 config.reserved_capabilities,
8191 );
8192
8193 let mut restart_required = Vec::new();
8202 for section in RestartRequiredSection::ALL {
8203 let changed = match section {
8204 RestartRequiredSection::Port => configured_port != context.configured_port,
8205 RestartRequiredSection::Storage => storage_config != context.storage_config,
8206 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
8207 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
8208 }
8209 RestartRequiredSection::AdmissionFactsTargets => {
8210 admission_facts_targets != context.admission_facts_targets
8211 }
8212 RestartRequiredSection::ScopeAuthorityOwners => {
8213 scope_authority_owners != context.scope_authority_owners
8214 }
8215 RestartRequiredSection::FederatedScopeOwner => {
8216 federated_scope_owner != context.federated_scope_owner
8217 }
8218 RestartRequiredSection::PrivilegedOperatorPrompt => {
8219 privileged_operator_prompt != context.privileged_operator_prompt
8220 }
8221 };
8222 if changed {
8223 restart_required.push(section.label().to_string());
8224 }
8225 }
8226 if !restart_required.is_empty() {
8227 warn!(
8228 config_path = %context.config_path.display(),
8229 sections = %restart_required.join(", "),
8230 "daemon config changed outside the modules section; restart the daemon to apply those changes"
8231 );
8232 }
8233
8234 for configured in &modules {
8235 if let Err(err) = validate_spec(&configured.module_spec()) {
8236 return Ok(vec![control_error_frame(
8237 &frame,
8238 "invalid_daemon_config",
8239 format!("supervisor rescan rejected daemon config: {err}"),
8240 )?]);
8241 }
8242 }
8243
8244 let configured_capabilities = modules
8245 .iter()
8246 .map(|module| (module.module_id.clone(), module.enabled))
8247 .collect::<Vec<_>>();
8248 let preview_capability_warnings = if preview {
8249 let (_, registrations) = self.runtime_capability_snapshot()?;
8250 let current_modules = self
8251 .supervisor
8252 .list()
8253 .into_iter()
8254 .map(|module| module.module_id().to_string())
8255 .collect::<BTreeSet<_>>();
8256 let resulting_modules = configured_capabilities.clone();
8257 let removed = current_modules
8258 .into_iter()
8259 .filter(|module_id| {
8260 !resulting_modules
8261 .iter()
8262 .any(|(configured_id, _)| configured_id == module_id)
8263 })
8264 .collect::<Vec<_>>();
8265 let disabled = match &mode {
8268 RescanMode::Candidate(_) => resulting_modules
8269 .iter()
8270 .filter(|(_, enabled)| !enabled)
8271 .map(|(module_id, _)| module_id.clone())
8272 .collect::<Vec<_>>(),
8273 _ => Vec::new(),
8274 };
8275 self.capability_evaluator.preview_warnings(
8276 resulting_modules,
8277 &removed,
8278 &disabled,
8279 ®istrations,
8280 )
8281 } else {
8282 Vec::new()
8283 };
8284 let reconciled = match (&mode, approved_preview.as_ref()) {
8285 (RescanMode::Preview, Some(approved)) => Ok((
8286 config_gate::preview_approved_daemon(approved, &modules),
8287 Vec::new(),
8288 )),
8289 (RescanMode::Candidate(candidate), _) => self
8290 .preview_candidate(&context, modules, candidate)
8291 .map(|result| (result, Vec::new())),
8292 _ => {
8293 self.reconcile_supervised_modules(
8294 &context.supervisor,
8295 modules,
8296 preview,
8297 requester.clone(),
8298 )
8299 .await
8300 }
8301 };
8302 let (result, changed) = match reconciled {
8303 Ok(result) => result,
8304 Err(message) => {
8305 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
8306 }
8307 };
8308 if !preview {
8309 if let Some(ledger) = &self.activity {
8310 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
8311 ledger.config_changed(&result.added, &result.removed, &changed, requester);
8312 }
8313 if let Err(code) = self.configure_activity(activity_config).await {
8314 warn!(
8315 code,
8316 "activity retention could not be applied during rescan"
8317 );
8318 }
8319 self.capability_evaluator
8320 .configure(configured_capabilities, reserved_capabilities);
8321 self.capability_evaluator.wake_deadline_loop();
8322 self.refresh_capability_requirements();
8323 }
8324 let mut result = result;
8325 result.restart_required = restart_required;
8326 result.capability_warnings = preview_capability_warnings;
8327 let response = ClientControlResponse::SupervisorRescan { result };
8328 Ok(vec![control_response_body_frame(
8329 &frame,
8330 &response,
8331 "ClientControlResponse::SupervisorRescan",
8332 )?])
8333 }
8334
8335 async fn handle_supervisor_release_reserved(
8336 &self,
8337 frame: Frame,
8338 module_id: String,
8339 ) -> Result<Vec<Frame>, RouterError> {
8340 let Some(context) = self.rescan.clone() else {
8341 return Ok(vec![control_error_frame(
8342 &frame,
8343 "release_unavailable",
8344 "reserved-id release requires a daemon started with a reloadable config path",
8345 )?]);
8346 };
8347 let operation_lock = self.supervisor.operation_lock();
8348 let _operation_guard = operation_lock.lock().await;
8349 let approved = if let Some(owner) = &self.config_runtime {
8350 match owner
8353 .on_blocking(|owner| {
8354 owner.observe_file_proposal()?;
8355 owner.approved_daemon_configuration()
8356 })
8357 .await
8358 {
8359 Ok(config) => config,
8360 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8361 }
8362 } else {
8363 None
8364 };
8365 let loaded = if let Some(approved) = approved {
8366 Ok(Some(approved))
8367 } else {
8368 crate::daemon_config::load(&context.config_path)
8369 };
8370 let loaded = match loaded {
8371 Ok(Some(config)) => config,
8372 Ok(None) => {
8373 return Ok(vec![control_error_frame(
8374 &frame,
8375 "invalid_daemon_config",
8376 format!(
8377 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
8378 context.config_path.display()
8379 ),
8380 )?])
8381 }
8382 Err(err) => {
8383 return Ok(vec![control_error_frame(
8384 &frame,
8385 "invalid_daemon_config",
8386 format!("unable to verify reserved-id release against daemon config: {err}"),
8387 )?])
8388 }
8389 };
8390 if loaded
8391 .modules
8392 .iter()
8393 .any(|configured| configured.module_id == module_id)
8394 {
8395 return Ok(vec![control_error_frame(
8396 &frame,
8397 "reserved_module_configured",
8398 format!(
8399 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
8400 ),
8401 )?]);
8402 }
8403 if !self.supervisor.release_retained_reserved_gate(&module_id) {
8404 return Ok(vec![control_error_frame(
8405 &frame,
8406 "reserved_gate_not_retained",
8407 format!(
8408 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
8409 ),
8410 )?]);
8411 }
8412
8413 let response = ClientControlResponse::SupervisorAck {
8414 module_id,
8415 applied: true,
8416 };
8417 Ok(vec![control_response_body_frame(
8418 &frame,
8419 &response,
8420 "ClientControlResponse::SupervisorAck",
8421 )?])
8422 }
8423
8424 fn plan_rescan(
8428 &self,
8429 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8430 ) -> Result<RescanPlan, String> {
8431 let mut current = BTreeMap::new();
8432 for module in self.supervisor.list() {
8433 let (spec, health) = module.configuration().map_err(|err| {
8434 format!(
8435 "failed to read configuration for module_id '{}': {err}",
8436 module.module_id()
8437 )
8438 })?;
8439 let enabled = module
8440 .status()
8441 .map_err(|err| {
8442 format!(
8443 "failed to read status for module_id '{}': {err}",
8444 module.module_id()
8445 )
8446 })?
8447 .enabled;
8448 current.insert(
8449 module.module_id().to_string(),
8450 (module, spec, health, enabled),
8451 );
8452 }
8453 let configured = configured_modules
8454 .into_iter()
8455 .map(|module| (module.module_id.clone(), module))
8456 .collect::<BTreeMap<_, _>>();
8457
8458 let added = configured
8459 .keys()
8460 .filter(|module_id| !current.contains_key(*module_id))
8461 .cloned()
8462 .collect::<Vec<_>>();
8463 let removed = current
8464 .keys()
8465 .filter(|module_id| !configured.contains_key(*module_id))
8466 .cloned()
8467 .collect::<Vec<_>>();
8468 let mut changed_pending_reload = Vec::new();
8469 let mut configuration_changes = BTreeSet::new();
8470 let mut enabled_changes = BTreeSet::new();
8471 let mut unchanged = 0_u32;
8472
8473 for (module_id, configured_module) in &configured {
8474 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
8475 else {
8476 continue;
8477 };
8478 let launch_changed = *current_spec != configured_module.module_spec();
8482 let configuration_changed =
8483 launch_changed || *current_health != configured_module.health;
8484 let enabled_changed = *current_enabled != configured_module.enabled
8487 || self
8488 .capability_evaluator
8489 .configured_enabled(module_id)
8490 .is_some_and(|enabled| enabled != configured_module.enabled);
8491 if configuration_changed {
8492 configuration_changes.insert(module_id.clone());
8493 }
8494 if launch_changed {
8495 changed_pending_reload.push(module_id.clone());
8496 }
8497 if enabled_changed {
8498 enabled_changes.insert(module_id.clone());
8499 }
8500 if !configuration_changed && !enabled_changed {
8501 unchanged = unchanged.saturating_add(1);
8502 }
8503 }
8504 Ok(RescanPlan {
8505 current,
8506 configured,
8507 added,
8508 removed,
8509 changed_pending_reload,
8510 configuration_changes,
8511 enabled_changes,
8512 unchanged,
8513 })
8514 }
8515
8516 async fn reconcile_supervised_modules(
8525 &self,
8526 supervisor: &Supervisor,
8527 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8528 preview: bool,
8529 requester: Principal,
8530 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
8531 let plan = self.plan_rescan(configured_modules)?;
8532
8533 if preview {
8539 return Ok((plan.preview_result(), Vec::new()));
8540 }
8541 let changed = plan
8542 .configuration_changes
8543 .union(&plan.enabled_changes)
8544 .cloned()
8545 .collect::<Vec<_>>();
8546 let RescanPlan {
8547 current,
8548 configured,
8549 added,
8550 removed,
8551 changed_pending_reload,
8552 configuration_changes,
8553 enabled_changes,
8554 unchanged,
8555 } = plan;
8556 if !(added.is_empty()
8561 && removed.is_empty()
8562 && configuration_changes.is_empty()
8563 && enabled_changes.is_empty())
8564 {
8565 if let Some(context) = &self.rescan {
8566 context
8567 .generation
8568 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8569 }
8570 }
8571
8572 for module_id in &removed {
8573 let module = ¤t
8574 .get(module_id)
8575 .expect("removed module came from current supervisor state")
8576 .0;
8577 module
8578 .retire_requested(Some(requester.clone()))
8579 .await
8580 .map_err(|err| {
8581 format!("failed to retire module_id '{module_id}' during rescan: {err}")
8582 })?;
8583 self.supervisor.record_rescan_removal(module_id);
8617 self.supervisor.retire(module_id);
8618 self.route_outages.forget(module_id);
8619 }
8620
8621 for module_id in configured.keys() {
8622 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
8623 continue;
8624 };
8625 let configured_module = configured
8626 .get(module_id)
8627 .expect("configured module id came from configured map");
8628 if configuration_changes.contains(module_id) {
8629 module
8630 .update_configuration(
8631 configured_module.module_spec(),
8632 configured_module.health.clone(),
8633 configured_module.drain_timeout_ms,
8634 )
8635 .await
8636 .map_err(|err| {
8637 format!(
8638 "failed to update module_id '{module_id}' configuration during rescan: {err}"
8639 )
8640 })?;
8641 }
8642 if *current_enabled != configured_module.enabled {
8645 self.route_outages.mark_operator_action(module_id);
8648 module
8649 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
8650 .await
8651 .map_err(|err| {
8652 self.route_outages.operator_action_ended_unrefused(module_id);
8653 format!(
8654 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
8655 configured_module.enabled
8656 )
8657 })?;
8658 }
8659 }
8660
8661 for module_id in &added {
8662 let configured_module = configured
8663 .get(module_id)
8664 .expect("added module id came from configured map");
8665 supervisor
8666 .supervise_configured_requested_async(
8667 configured_module.module_spec(),
8668 configured_module.enabled,
8669 configured_module.health.clone(),
8670 configured_module.drain_timeout_ms,
8671 configured_module.restart,
8672 configured_module.register_timeout,
8673 Some(requester.clone()),
8674 )
8675 .await
8676 .map_err(|err| {
8677 format!("failed to add module_id '{module_id}' during rescan: {err}")
8678 })?;
8679 }
8680
8681 Ok((
8682 SupervisorRescanResult {
8683 added,
8684 removed,
8685 changed_pending_reload,
8686 enabled_changes: enabled_changes.iter().cloned().collect(),
8687 unchanged,
8688 preview: false,
8689 restart_required: Vec::new(),
8692 capability_warnings: Vec::new(),
8693 module_actions: Vec::new(),
8694 earlier_pending: Vec::new(),
8695 expect: None,
8696 },
8697 changed,
8698 ))
8699 }
8700
8701 async fn handle_supervisor_set_enabled(
8702 &self,
8703 frame: Frame,
8704 module_id: String,
8705 enabled: bool,
8706 requester: Principal,
8707 ) -> Result<Vec<Frame>, RouterError> {
8708 let operation_lock = self.supervisor.operation_lock();
8709 let _operation_guard = operation_lock.lock().await;
8710 let Some(module) = self.supervisor.get(&module_id) else {
8711 return Ok(vec![control_error_frame(
8712 &frame,
8713 "unknown_module",
8714 format!("module_id '{module_id}' is not supervised"),
8715 )?]);
8716 };
8717
8718 self.route_outages.mark_operator_action(&module_id);
8721 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
8722 Ok(applied) => applied,
8723 Err(err) => {
8724 self.route_outages
8725 .operator_action_ended_unrefused(&module_id);
8726 return Ok(vec![control_error_frame(
8727 &frame,
8728 if err.is_config_gate_error() {
8729 "config_gated"
8730 } else {
8731 "target_unavailable"
8732 },
8733 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
8734 )?]);
8735 }
8736 };
8737 if !applied {
8738 self.route_outages
8741 .operator_action_ended_unrefused(&module_id);
8742 }
8743
8744 self.capability_evaluator.wake_deadline_loop();
8745 self.refresh_capability_requirements();
8746 let response = ClientControlResponse::SupervisorAck { module_id, applied };
8747 Ok(vec![control_response_body_frame(
8748 &frame,
8749 &response,
8750 "ClientControlResponse::SupervisorAck",
8751 )?])
8752 }
8753
8754 async fn handle_supervisor_health_probe(
8755 &self,
8756 ctx: &RouteCtx,
8757 frame: Frame,
8758 module_id: String,
8759 ) -> Result<Vec<Frame>, RouterError> {
8760 self.refresh_capability_requirements();
8761 let Some(registration) = self
8762 .registry
8763 .get_module(&module_id)
8764 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
8765 else {
8766 return Ok(vec![control_error_frame(
8767 &frame,
8768 "unknown_module",
8769 format!("module_id '{module_id}' is not registered"),
8770 )?]);
8771 };
8772
8773 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
8786 {
8787 return Ok(vec![control_error_frame(
8788 &frame,
8789 "health_not_advertised",
8790 format!("module_id '{module_id}' did not advertise health.check"),
8791 )?]);
8792 }
8793
8794 self.observe_control_wait(
8795 ctx,
8796 Some(®istration.manifest.module_id),
8797 ops::SUPERVISOR_HEALTH_PROBE,
8798 async {
8799 let deadline = Instant::now() + self.health_probe_timeout;
8800 let pending = match self.forwarding.begin_module_control_rpc_for(
8801 &module_id,
8802 MODULE_CONTROL_OP_HEALTH_CHECK,
8803 deadline,
8804 ) {
8805 Ok(pending) => pending,
8806 Err(err) => {
8807 return Ok(vec![control_error_frame(
8808 &frame,
8809 forwarding_error_code(&err),
8810 err.to_string(),
8811 )?])
8812 }
8813 };
8814
8815 let PendingModuleControlRpc {
8816 endpoint,
8817 module_sink,
8818 negotiated_ver,
8819 corr: probe_corr,
8820 receiver,
8821 } = pending;
8822 let mut guard =
8823 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
8824 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
8825 .map_err(|err| {
8826 RouterError::backend(
8827 0,
8828 frame.header.corr,
8829 format!("failed to encode health.check request: {err}"),
8830 )
8831 })?;
8832 let probe_frame = Frame::build_with_version(
8833 negotiated_ver,
8834 FrameType::Request,
8835 control_flags(),
8836 0,
8837 0,
8838 probe_corr,
8839 probe_body,
8840 )
8841 .map_err(RouterError::FrameBuild)?;
8842
8843 match timeout_at(deadline, module_sink.send(probe_frame)).await {
8846 Ok(Ok(())) => {}
8847 Ok(Err(err)) => {
8848 return Ok(vec![control_error_frame(
8849 &frame,
8850 "target_unavailable",
8851 err.to_string(),
8852 )?]);
8853 }
8854 Err(_) => {
8855 return Ok(vec![control_error_frame(
8856 &frame,
8857 "module_timeout",
8858 format!(
8859 "module_id '{module_id}' did not answer health.check within {:?}",
8860 self.health_probe_timeout
8861 ),
8862 )?]);
8863 }
8864 }
8865
8866 match timeout_at(deadline, receiver).await {
8867 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => {
8868 Ok(vec![control_error_frame(
8869 &frame,
8870 "invalid_control_body",
8871 "health.check returned an unexpected config response",
8872 )?])
8873 }
8874 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
8875 guard.disarm();
8876 let Some(report) = response.health_report() else {
8877 return Ok(vec![control_error_frame(
8878 &frame,
8879 "invalid_control_body",
8880 "health.check RPC returned a non-health response",
8881 )?]);
8882 };
8883 let HealthReport {
8888 status,
8889 detail,
8890 metrics,
8891 } = report;
8892 let capability_detail = self
8893 .capability_evaluator
8894 .required_problem_detail(&module_id);
8895 let response = ClientControlResponse::SupervisorHealthProbe {
8896 module_id,
8897 status,
8898 detail: append_capability_problem_detail(detail, capability_detail),
8899 metrics,
8900 };
8901 Ok(vec![control_response_body_frame(
8902 &frame,
8903 &response,
8904 "ClientControlResponse::SupervisorHealthProbe",
8905 )?])
8906 }
8907 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
8908 guard.disarm();
8909 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
8914 }
8915 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
8916 guard.disarm();
8917 Ok(vec![control_error_frame(
8918 &frame,
8919 "target_unavailable",
8920 message,
8921 )?])
8922 }
8923 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
8924 guard.disarm();
8925 Ok(vec![control_error_frame(
8926 &frame,
8927 "invalid_control_body",
8928 message,
8929 )?])
8930 }
8931 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
8932 guard.disarm();
8933 Ok(vec![control_error_frame(
8934 &frame,
8935 "invalid_control_body",
8936 format!("expected module-control op '{expected}', got '{actual}'"),
8937 )?])
8938 }
8939 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
8940 guard.disarm();
8941 Ok(vec![control_error_frame(
8942 &frame,
8943 "module_timeout",
8944 format!(
8945 "module_id '{module_id}' answered health.check after {:?}",
8946 self.health_probe_timeout
8947 ),
8948 )?])
8949 }
8950 Ok(Err(_)) => Ok(vec![control_error_frame(
8951 &frame,
8952 "target_unavailable",
8953 "health.check waiter was canceled before the module responded",
8954 )?]),
8955 Err(_) => Ok(vec![control_error_frame(
8956 &frame,
8957 "module_timeout",
8958 format!(
8959 "module_id '{module_id}' did not answer health.check within {:?}",
8960 self.health_probe_timeout
8961 ),
8962 )?]),
8963 }
8964 },
8965 )
8966 .await
8967 }
8968
8969 fn supervisor_status(
8970 &self,
8971 module_id: &str,
8972 corr: u64,
8973 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
8974 self.supervisor
8975 .get(module_id)
8976 .map(|module| {
8977 let warming = module.is_warming_for_control("status").map_err(|err| {
8978 RouterError::backend(
8979 0,
8980 corr,
8981 format!(
8982 "failed to read supervisor warming state for module_id '{module_id}': {err}"
8983 ),
8984 )
8985 })?;
8986 module.status_for_control("status").map_err(|err| {
8987 RouterError::backend(
8988 0,
8989 corr,
8990 format!(
8991 "failed to read supervisor status for module_id '{module_id}': {err}"
8992 ),
8993 )
8994 }).map(|status| (status, warming))
8995 })
8996 .transpose()
8997 }
8998
8999 fn guard_module_control_op(
9000 &self,
9001 frame: &Frame,
9002 module_id: &str,
9003 op: &str,
9004 ) -> Result<Option<Frame>, RouterError> {
9005 if self.module_grants_op(module_id, op, frame.header.corr)? {
9006 return Ok(None);
9007 }
9008
9009 Ok(Some(control_error_frame(
9010 frame,
9011 "op_not_allowed",
9012 format!("module_id '{module_id}' did not grant control op '{op}'"),
9013 )?))
9014 }
9015
9016 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
9017 let Some(registration) = self
9018 .registry
9019 .get_module(module_id)
9020 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
9021 else {
9022 return Ok(false);
9023 };
9024 Ok(module_registration_grants_op(®istration.control_ops, op))
9025 }
9026
9027 fn handle_status_update(
9028 &self,
9029 endpoint: ModuleEndpointId,
9030 frame: Frame,
9031 ) -> Result<Vec<Frame>, RouterError> {
9032 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
9033 Ok(update) => update,
9034 Err(err) => {
9035 if is_known_module_push_op(&frame.body) {
9040 return Ok(vec![control_error_frame(
9041 &frame,
9042 "invalid_control_body",
9043 format!("malformed module control push body: {err}"),
9044 )?]);
9045 }
9046 return Ok(Vec::new());
9047 }
9048 };
9049
9050 match update {
9051 ModuleControlPush::RouteStatus {
9052 route_channel,
9053 route_epoch,
9054 status,
9055 } => {
9056 self.forwarding
9057 .cache_status(endpoint, route_channel, route_epoch, status)
9058 .map_err(RouterError::Forwarding)?;
9059 }
9060 }
9061 Ok(Vec::new())
9062 }
9063
9064 fn handle_route_poll(
9065 &self,
9066 ctx: &RouteCtx,
9067 frame: Frame,
9068 route_channel: u16,
9069 route_epoch: u32,
9070 kind: PollKind,
9071 ) -> Result<Vec<Frame>, RouterError> {
9072 let snapshot = self
9073 .forwarding
9074 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
9075 .map_err(RouterError::Forwarding)?;
9076 let response = match (kind, snapshot) {
9077 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
9078 ClientControlResponse::RoutePoll {
9079 route_channel,
9080 route_epoch,
9081 status,
9082 live: None,
9083 }
9084 }
9085 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9086 route_channel,
9087 route_epoch,
9088 status: None,
9089 live: None,
9090 },
9091 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
9092 let live = self
9118 .process_liveness
9119 .as_ref()
9120 .and_then(|source| source.process_live(&module_id))
9121 .unwrap_or(true);
9122 ClientControlResponse::RoutePoll {
9123 route_channel,
9124 route_epoch,
9125 status: None,
9126 live: Some(live),
9127 }
9128 }
9129 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9130 route_channel,
9131 route_epoch,
9132 status: None,
9133 live: Some(false),
9134 },
9135 };
9136
9137 Ok(vec![control_response_body_frame(
9138 &frame,
9139 &response,
9140 "ClientControlResponse::RoutePoll",
9141 )?])
9142 }
9143
9144 pub(crate) fn observe_module_control_completion(
9145 &self,
9146 completion: ModuleControlRpcCompletion,
9147 ) -> bool {
9148 match completion {
9149 ModuleControlRpcCompletion::Unknown => false,
9150 ModuleControlRpcCompletion::Settled => true,
9151 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
9152 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
9153 info!(
9154 module_id = %module_id,
9155 latency_ms,
9156 "late health.check answer proves the module is alive"
9157 );
9158 match self
9159 .supervisor
9160 .record_late_health_answer(&module_id, latency_ms)
9161 {
9162 Ok(true) => {}
9163 Ok(false) => debug!(
9164 module_id = %module_id,
9165 latency_ms,
9166 "late health.check answer has no active supervisor snapshot"
9167 ),
9168 Err(err) => warn!(
9169 module_id = %module_id,
9170 latency_ms,
9171 error = %err,
9172 "failed to record late health.check answer"
9173 ),
9174 }
9175 true
9176 }
9177 }
9178 }
9179
9180 fn refuse_to_end_module_connection_for_a_client(
9201 &self,
9202 module_connection_id: ConnectionId,
9203 corr: u64,
9204 err: ForwardingError,
9205 ) -> Result<(), RouterError> {
9206 if let ForwardingError::ConnectionClosing { connection_id } = err {
9207 if connection_id != module_connection_id {
9208 warn!(
9209 module_connection_id = module_connection_id.get(),
9210 client_connection_id = connection_id.get(),
9211 corr,
9212 "dropping a route.bind response for a closing client; the module connection keeps serving"
9213 );
9214 return Ok(());
9215 }
9216 }
9217 Err(RouterError::Forwarding(err))
9218 }
9219
9220 fn handle_module_relay_response(
9221 &self,
9222 connection_id: ConnectionId,
9223 frame: Frame,
9224 ) -> Result<Vec<Frame>, RouterError> {
9225 let mut secondary_error = None;
9226 let outcome = match frame.header.ty {
9227 FrameType::Response => match module_control_response_probe(&frame.body) {
9228 Ok(probe) if probe.op == "route.bind" => {
9229 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9230 Ok(ModuleControlResponse::RouteBindAck {}) => {
9231 RouteBindRelayOutcome::Accepted
9232 }
9233 Ok(other) => {
9234 let message =
9235 format!("route.bind response carried unexpected body: {other:?}");
9236 secondary_error = Some(control_error_frame(
9237 &frame,
9238 "invalid_control_body",
9239 message.clone(),
9240 )?);
9241 RouteBindRelayOutcome::ModuleGone(message)
9242 }
9243 Err(err) => {
9244 let message = format!("malformed route.bind response body: {err}");
9245 secondary_error = Some(control_error_frame(
9246 &frame,
9247 "invalid_control_body",
9248 message.clone(),
9249 )?);
9250 RouteBindRelayOutcome::ModuleGone(message)
9251 }
9252 }
9253 }
9254 Ok(probe) => {
9255 let outcome = if matches!(probe.op.as_str(), "config.review" | "consent/v1") {
9256 ModuleControlRpcOutcome::RawResponse(frame.body.clone())
9257 } else {
9258 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9259 Ok(response) => ModuleControlRpcOutcome::Response(response),
9260 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9261 "malformed {} response body: {err}",
9262 probe.op
9263 )),
9264 }
9265 };
9266 let completion = self
9267 .forwarding
9268 .complete_module_control_rpc(
9269 connection_id,
9270 frame.header.corr,
9271 Some(&probe.op),
9272 outcome,
9273 )
9274 .map_err(RouterError::Forwarding)?;
9275 if !self.observe_module_control_completion(completion) {
9276 debug!(
9277 connection_id = connection_id.get(),
9278 corr = frame.header.corr,
9279 op = %probe.op,
9280 "dropping late or unknown module-control RPC response"
9281 );
9282 }
9283 return Ok(Vec::new());
9284 }
9285 Err(err) => {
9286 if let Some(expected_op) = self
9287 .forwarding
9288 .pending_module_control_op(connection_id, frame.header.corr)
9289 .map_err(RouterError::Forwarding)?
9290 {
9291 let completion = self
9292 .forwarding
9293 .complete_module_control_rpc(
9294 connection_id,
9295 frame.header.corr,
9296 None,
9297 ModuleControlRpcOutcome::MalformedResponse(format!(
9298 "malformed {expected_op} response body: {err}"
9299 )),
9300 )
9301 .map_err(RouterError::Forwarding)?;
9302 if !self.observe_module_control_completion(completion) {
9303 debug!(
9304 connection_id = connection_id.get(),
9305 corr = frame.header.corr,
9306 "dropping late malformed module-control RPC response"
9307 );
9308 }
9309 return Ok(Vec::new());
9310 }
9311 let message = format!("malformed route.bind response body: {err}");
9312 secondary_error = Some(control_error_frame(
9313 &frame,
9314 "invalid_control_body",
9315 message.clone(),
9316 )?);
9317 RouteBindRelayOutcome::ModuleGone(message)
9318 }
9319 },
9320 FrameType::Error => {
9321 if self
9322 .forwarding
9323 .pending_module_control_op(connection_id, frame.header.corr)
9324 .map_err(RouterError::Forwarding)?
9325 .is_some()
9326 {
9327 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
9328 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
9329 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9330 "malformed module-control ERROR body: {err}"
9331 )),
9332 };
9333 let completion = self
9334 .forwarding
9335 .complete_module_control_rpc(
9336 connection_id,
9337 frame.header.corr,
9338 None,
9339 outcome,
9340 )
9341 .map_err(RouterError::Forwarding)?;
9342 if !self.observe_module_control_completion(completion) {
9343 debug!(
9344 connection_id = connection_id.get(),
9345 corr = frame.header.corr,
9346 "dropping late or unknown module-control RPC error"
9347 );
9348 }
9349 return Ok(Vec::new());
9350 }
9351 match serde_json::from_slice::<ErrorBody>(&frame.body) {
9352 Ok(body) => RouteBindRelayOutcome::Rejected(body),
9353 Err(err) => {
9354 let message = format!("malformed route.bind ERROR body: {err}");
9355 secondary_error = Some(control_error_frame(
9356 &frame,
9357 "invalid_control_body",
9358 message.clone(),
9359 )?);
9360 RouteBindRelayOutcome::ModuleGone(message)
9361 }
9362 }
9363 }
9364 ty => {
9365 return Ok(vec![control_error_frame(
9366 &frame,
9367 "unsupported_control_frame",
9368 format!("unsupported module channel-0 frame {ty:?}"),
9369 )?])
9370 }
9371 };
9372
9373 let settled =
9374 self.forwarding
9375 .complete_pending_relay(connection_id, frame.header.corr, outcome);
9376 let completion = match settled {
9377 Ok(completion) => completion,
9378 Err(err) => {
9379 self.refuse_to_end_module_connection_for_a_client(
9380 connection_id,
9381 frame.header.corr,
9382 err,
9383 )?;
9384 return Ok(secondary_error.into_iter().collect());
9385 }
9386 };
9387 if let Some(target) = completion.abandoned.as_ref() {
9388 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
9389 }
9390 if !completion.settled {
9391 debug!(
9392 connection_id = connection_id.get(),
9393 corr = frame.header.corr,
9394 frame_type = ?frame.header.ty,
9395 "dropping late or unknown route.bind relay response"
9396 );
9397 }
9398 Ok(secondary_error.into_iter().collect())
9399 }
9400
9401 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
9402 debug!(connection_id = connection_id.get(), "handling GOODBYE");
9403 self.cleanup_connection_with_end_reason(
9407 connection_id,
9408 RegistrationEndReason::ExplicitGoodbye,
9409 )
9410 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
9411 Ok(Vec::new())
9412 }
9413}
9414
9415impl Default for ControlHandler {
9416 fn default() -> Self {
9417 Self::new(Arc::new(Registry::default()))
9418 }
9419}
9420
9421impl crate::supervise::SwapPromotionObserver for ControlHandler {
9422 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
9423 self.apply_registration_capabilities(registration);
9424 }
9425}
9426
9427fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
9428 CapabilityRequirementStatus {
9429 consumer: status.consumer,
9430 capability: status.capability,
9431 need: match status.need {
9432 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
9433 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
9434 },
9435 verdict: status.verdict.as_str().to_string(),
9436 episode_seq: status.episode_seq,
9437 config_satisfiable: status.config_satisfiable,
9438 runtime_available: status.runtime_available,
9439 detail: status.detail,
9440 }
9441}
9442
9443fn append_capability_problem_detail(
9444 detail: Option<String>,
9445 capability_detail: Option<String>,
9446) -> Option<String> {
9447 match (detail, capability_detail) {
9448 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
9449 (Some(detail), None) => Some(detail),
9450 (None, Some(capability_detail)) => Some(capability_detail),
9451 (None, None) => None,
9452 }
9453}
9454
9455fn subc_ops() -> Vec<String> {
9456 SUBC_CONTROL_OPS
9457 .iter()
9458 .map(|op| (*op).to_string())
9459 .collect()
9460}
9461
9462fn module_subc_ops() -> Vec<String> {
9463 SUBC_CONTROL_OPS
9464 .iter()
9465 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
9466 .map(|op| (*op).to_string())
9467 .collect()
9468}
9469
9470#[cfg(test)]
9471fn module_baseline_control_ops() -> Vec<String> {
9472 MODULE_BASELINE_CONTROL_OPS
9473 .iter()
9474 .map(|op| (*op).to_string())
9475 .collect()
9476}
9477
9478fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
9479 let mut seen = HashSet::new();
9480 let mut effective = Vec::new();
9481 for op in MODULE_BASELINE_CONTROL_OPS {
9482 if seen.insert((*op).to_string()) {
9483 effective.push((*op).to_string());
9484 }
9485 }
9486 for op in declared.unwrap_or_default() {
9487 if seen.insert(op.clone()) {
9488 effective.push(op);
9489 }
9490 }
9491 effective
9492}
9493
9494fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
9495 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
9496}
9497
9498fn target_module_id(target: &RouteTarget) -> &str {
9499 match target {
9500 RouteTarget::ToolProvider { module_id }
9501 | RouteTarget::ManagementSurface { module_id }
9502 | RouteTarget::InternalService { module_id, .. } => module_id,
9503 }
9504}
9505
9506fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
9507 roles.iter().any(|role| match (target, role) {
9508 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
9509 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
9510 (
9511 RouteTarget::InternalService { service_id, .. },
9512 ProviderRole::InternalService {
9513 service_id: provided,
9514 ..
9515 },
9516 ) => service_id == provided,
9517 _ => false,
9518 })
9519}
9520
9521fn is_routable_role(role: &ProviderRole) -> bool {
9522 matches!(
9523 role,
9524 ProviderRole::ToolProvider { .. }
9525 | ProviderRole::ManagementSurface { .. }
9526 | ProviderRole::InternalService { .. }
9527 )
9528}
9529
9530#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9531enum ControlRequestBodyError {
9532 UnknownOp,
9533 InvalidBody,
9534}
9535
9536#[derive(Debug, Deserialize)]
9537struct ControlOpProbe {
9538 op: String,
9539}
9540
9541fn module_control_response_probe(body: &[u8]) -> Result<ControlOpProbe, serde_json::Error> {
9542 serde_json::from_slice(body).or_else(|original| {
9543 #[derive(Deserialize)]
9544 struct ConsentProbe {
9545 protocol: String,
9546 }
9547 match serde_json::from_slice::<ConsentProbe>(body) {
9548 Ok(probe) if probe.protocol == subc_protocol::consent::CONSENT_V1 => {
9549 Ok(ControlOpProbe { op: probe.protocol })
9550 }
9551 _ => Err(original),
9552 }
9553 })
9554}
9555
9556const MODULE_PUSH_OPS: &[&str] = &["route.status"];
9559
9560fn is_known_module_push_op(body: &[u8]) -> bool {
9561 serde_json::from_slice::<ControlOpProbe>(body)
9562 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
9563 .unwrap_or(false)
9564}
9565
9566fn is_known_module_request_op(body: &[u8]) -> bool {
9567 serde_json::from_slice::<ControlOpProbe>(body)
9568 .map(|probe| is_module_to_subc_op(&probe.op))
9569 .unwrap_or(false)
9570}
9571
9572fn is_module_to_subc_op(op: &str) -> bool {
9573 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
9574}
9575
9576fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
9577 debug!(
9578 op = %op,
9579 connection_id = connection_id.get(),
9580 corr,
9581 "control dispatch"
9582 );
9583}
9584
9585fn log_slow_control_dispatch(
9586 dispatch_started_at: Option<StdInstant>,
9587 op: &'static str,
9588 connection_id: ConnectionId,
9589 corr: u64,
9590 route_open_timings: Option<&RouteOpenTimings>,
9591) {
9592 let Some(dispatch_started_at) = dispatch_started_at else {
9593 return;
9594 };
9595 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
9597 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
9598 if let Some(timings) = route_open_timings {
9599 warn!(
9600 op = %op,
9601 connection_id = connection_id.get(),
9602 corr,
9603 elapsed_ms = elapsed.as_millis() as u64,
9604 module_id = ?timings.module_id,
9605 principal = %timings.principal,
9606 admission_ms = timings.admission.as_millis() as u64,
9607 project_root_ms = timings.project_root.as_millis() as u64,
9608 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
9609 bind_send_ms = timings.bind_send.as_millis() as u64,
9610 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
9611 "slow control dispatch"
9612 );
9613 return;
9614 }
9615 warn!(
9616 op = %op,
9617 connection_id = connection_id.get(),
9618 corr,
9619 elapsed_ms = elapsed.as_millis() as u64,
9620 "slow control dispatch"
9621 );
9622 }
9623}
9624
9625#[cfg(test)]
9627type TestCallerLookup =
9628 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
9629
9630fn privileged_operation(request: &impl PrivilegedRequest) -> Option<PrivilegedOperation<'_>> {
9632 request.privileged_operation()
9633}
9634
9635trait PrivilegedRequest {
9636 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>>;
9637}
9638
9639impl PrivilegedRequest for ClientControlRequest {
9640 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>> {
9641 let request = self;
9642 match request {
9643 ClientControlRequest::SupervisorRestart { module_id, .. } => {
9644 Some(PrivilegedOperation::Restart { module_id })
9645 }
9646 ClientControlRequest::SupervisorSwap { module_id, .. } => {
9647 Some(PrivilegedOperation::Swap { module_id })
9648 }
9649 ClientControlRequest::SupervisorReload { module_id } => {
9650 Some(PrivilegedOperation::Reload { module_id })
9651 }
9652 ClientControlRequest::SupervisorRescan { preview } => {
9653 if *preview {
9654 None
9655 } else {
9656 Some(PrivilegedOperation::Rescan)
9657 }
9658 }
9659 ClientControlRequest::SupervisorRescanChecked {
9664 preview,
9665 candidate,
9666 expect,
9667 } => (!*preview && candidate.is_none() && expect.is_some())
9668 .then_some(PrivilegedOperation::Rescan),
9669 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
9670 Some(PrivilegedOperation::SetEnabled {
9671 module_id,
9672 enabled: *enabled,
9673 })
9674 }
9675 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
9676 Some(PrivilegedOperation::ReleaseReserved { module_id })
9677 }
9678 ClientControlRequest::ServerDescribe {}
9679 | ClientControlRequest::DaemonShutdownIntent { .. }
9682 | ClientControlRequest::CatalogList { .. }
9683 | ClientControlRequest::RouteOpen { .. }
9684 | ClientControlRequest::RoutePoll { .. }
9685 | ClientControlRequest::SupervisorList {}
9686 | ClientControlRequest::SupervisorHealth {}
9687 | ClientControlRequest::SupervisorHealthProbe { .. }
9688 | ClientControlRequest::SupervisorRoutes { .. }
9689 | ClientControlRequest::SupervisorProvenance { .. }
9690 | ClientControlRequest::SupervisorStderrTail { .. }
9691 | ClientControlRequest::SupervisorTerminals { .. }
9692 | ClientControlRequest::SupervisorSpawnSnapshot {}
9693 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
9694 }
9695 }
9696}
9697
9698fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
9699 match request {
9700 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
9701 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
9702 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
9703 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
9704 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
9705 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
9706 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
9707 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
9708 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
9709 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
9710 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
9711 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
9712 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
9713 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
9714 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
9715 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
9716 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
9717 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
9718 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
9719 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
9720 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
9721 }
9722}
9723
9724struct RouteOpenCaller {
9732 principal: String,
9736 scope: String,
9739 harness: String,
9742}
9743
9744impl RouteOpenCaller {
9745 fn new(
9746 principal: Option<&Principal>,
9747 scope: Option<&ScopeSelector>,
9748 identity: &BindIdentity,
9749 ) -> Self {
9750 Self {
9751 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
9752 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
9753 harness: bounded_log_text(&identity.harness),
9754 }
9755 }
9756
9757 fn unknown() -> Self {
9760 Self {
9761 principal: "unknown".to_string(),
9762 scope: "unknown".to_string(),
9763 harness: "unknown".to_string(),
9764 }
9765 }
9766}
9767
9768const MAX_LOGGED_TEXT_CHARS: usize = 64;
9771
9772fn bounded_log_text(value: &str) -> String {
9776 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
9777 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
9778 bounded.push('…');
9779 }
9780 format!("{bounded:?}")
9781}
9782
9783fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
9789 use subc_protocol::scope::ScopeKind;
9790 let kind = match stamp.kind {
9791 ScopeKind::Head => "head",
9792 ScopeKind::Worker => "worker",
9793 ScopeKind::Ephemeral => "ephemeral",
9794 ScopeKind::Federated => "federated",
9795 _ => "unknown",
9796 };
9797 format!(
9798 "{kind}/{}/{}@{}",
9799 principal_label(&stamp.owner),
9800 bounded_log_text(&stamp.scope_ref),
9801 stamp.scope_epoch
9802 )
9803}
9804
9805fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
9812 let epoch = selector
9813 .scope_epoch
9814 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
9815 format!(
9816 "requested/{}/{}@{epoch}",
9817 bounded_log_text(&principal_label(&selector.owner)),
9818 bounded_log_text(&selector.scope_ref),
9819 )
9820}
9821
9822fn principal_label(principal: &Principal) -> String {
9823 match principal {
9824 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
9825 Principal::Direct => "direct".to_string(),
9826 other => format!("{other:?}"),
9827 }
9828}
9829
9830fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
9831 match request {
9832 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
9833 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
9834 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
9835 }
9836 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
9837 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
9838 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
9839 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
9840 ModuleControlRequestFromModule::RouteAuthorize { .. } => {
9841 subc_protocol::scope::ROUTE_AUTHORIZE_OP
9842 }
9843 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
9844 _ => "unsupported",
9845 }
9846}
9847
9848fn parse_client_control_request(
9849 body: &[u8],
9850) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
9851 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
9852 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
9853 Ok(probe)
9854 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
9855 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
9856 {
9857 ControlRequestBodyError::InvalidBody
9858 }
9859 Ok(_) => ControlRequestBodyError::UnknownOp,
9860 Err(_) => ControlRequestBodyError::InvalidBody,
9861 };
9862 (err, classification)
9863 })
9864}
9865
9866fn parse_module_control_request_from_module(
9867 body: &[u8],
9868) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
9869 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
9870 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
9871 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
9872 Ok(_) => ControlRequestBodyError::UnknownOp,
9873 Err(_) => ControlRequestBodyError::InvalidBody,
9874 };
9875 (err, classification)
9876 })
9877}
9878
9879#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
9880enum ProviderRoleKind {
9881 ToolProvider,
9882 PipelineStage,
9883 ManagementSurface,
9884 InternalService,
9885}
9886
9887fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
9888 match role {
9889 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
9890 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
9891 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
9892 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
9893 }
9894}
9895
9896fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
9897 roles.iter().map(provider_role_kind).collect()
9898}
9899
9900const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
9903
9904#[derive(Debug, Default, PartialEq, Eq)]
9906struct ScopeOutcomeCounts {
9907 created: usize,
9908 replaced: usize,
9909 updated: usize,
9910 unchanged: usize,
9911 refused: usize,
9912}
9913
9914impl ScopeOutcomeCounts {
9915 fn of(results: &[ScopeRecordResult]) -> Self {
9916 let mut counts = Self::default();
9917 for result in results {
9918 let slot = match result.outcome {
9919 ScopeRecordOutcome::Created => &mut counts.created,
9920 ScopeRecordOutcome::Replaced => &mut counts.replaced,
9921 ScopeRecordOutcome::Updated => &mut counts.updated,
9922 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
9923 ScopeRecordOutcome::Refused => &mut counts.refused,
9924 };
9925 *slot += 1;
9926 }
9927 counts
9928 }
9929}
9930
9931#[cfg(test)]
9932mod scope_outcome_count_tests {
9933 use super::*;
9934
9935 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
9936 ScopeRecordResult {
9937 scope_ref: "r".to_string(),
9938 scope_epoch: 1,
9939 outcome,
9940 code: None,
9941 message: None,
9942 version: None,
9943 parent_state: None,
9944 }
9945 }
9946
9947 #[test]
9950 fn every_outcome_is_counted_in_its_own_field() {
9951 let results = [
9952 result(ScopeRecordOutcome::Created),
9953 result(ScopeRecordOutcome::Created),
9954 result(ScopeRecordOutcome::Replaced),
9955 result(ScopeRecordOutcome::Updated),
9956 result(ScopeRecordOutcome::Unchanged),
9957 result(ScopeRecordOutcome::Refused),
9958 result(ScopeRecordOutcome::Refused),
9959 result(ScopeRecordOutcome::Refused),
9960 ];
9961 assert_eq!(
9962 ScopeOutcomeCounts::of(&results),
9963 ScopeOutcomeCounts {
9964 created: 2,
9965 replaced: 1,
9966 updated: 1,
9967 unchanged: 1,
9968 refused: 3,
9969 }
9970 );
9971 }
9972}
9973
9974fn capability_census_trigger(
9978 old: Option<&CapabilityDeclarations>,
9979 new: Option<&CapabilityDeclarations>,
9980) -> bool {
9981 let old_provides = old
9982 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
9983 .unwrap_or_default();
9984 let old_denies = old
9985 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
9986 .unwrap_or_default();
9987 let new = new.cloned().unwrap_or(CapabilityDeclarations {
9988 provides: Vec::new(),
9989 requires: Vec::new(),
9990 must_never_reach: Vec::new(),
9991 });
9992
9993 new.provides
9994 .iter()
9995 .any(|capability| !old_provides.contains(capability))
9996 || new
9997 .must_never_reach
9998 .iter()
9999 .any(|capability| !old_denies.contains(capability))
10000}
10001
10002fn catalog_update_frozen_field_message(
10003 registered: &ModuleManifest,
10004 provides: &[ProviderRole],
10005) -> Option<String> {
10006 let old_has_provides = !registered.provides.is_empty();
10007 let new_has_provides = !provides.is_empty();
10008 if old_has_provides != new_has_provides {
10009 return Some(format!(
10010 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
10011 registered.module_id
10012 ));
10013 }
10014
10015 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
10016 return Some(format!(
10017 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
10018 registered.module_id
10019 ));
10020 }
10021
10022 let registered_concurrency = manifest_concurrency(registered);
10023 let mut candidate = registered.clone();
10024 candidate.provides = provides.to_vec();
10025 let candidate_concurrency = manifest_concurrency(&candidate);
10026 if candidate_concurrency != registered_concurrency {
10027 return Some(format!(
10028 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
10029 registered.module_id, registered_concurrency, candidate_concurrency
10030 ));
10031 }
10032
10033 None
10036}
10037
10038fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
10039 manifest.provides.iter().any(is_routable_role)
10040}
10041
10042fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
10048 manifest
10049 .provides
10050 .iter()
10051 .find_map(|provider| match provider {
10052 ProviderRole::ToolProvider { concurrency, .. }
10053 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
10054 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
10055 })
10056 .unwrap_or(Concurrency::ModuleManaged)
10057}
10058
10059fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
10066 let has_management_surface = manifest
10067 .provides
10068 .iter()
10069 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
10070 if !has_management_surface {
10071 return false;
10072 }
10073 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
10074 return false;
10075 };
10076 let Some(provides) = raw
10077 .get("manifest")
10078 .and_then(|manifest| manifest.get("provides"))
10079 .and_then(serde_json::Value::as_array)
10080 else {
10081 return false;
10082 };
10083 provides.iter().any(|role| {
10088 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
10089 && role.get("concurrency").is_none()
10090 })
10091}
10092
10093fn negotiate_version(peer_version: u8) -> Result<u8, String> {
10094 if peer_version != PROTOCOL_VERSION {
10095 return Err(format!(
10096 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
10097 ));
10098 }
10099 Ok(PROTOCOL_VERSION)
10100}
10101
10102fn pong(frame: &Frame) -> Result<Frame, RouterError> {
10103 Frame::build_with_version(
10104 response_version(frame),
10105 FrameType::Pong,
10106 frame.header.flags,
10107 0,
10108 0,
10109 frame.header.corr,
10110 Vec::new(),
10111 )
10112 .map_err(RouterError::FrameBuild)
10113}
10114
10115pub(crate) fn control_error_frame(
10116 frame: &Frame,
10117 code: &'static str,
10118 message: impl Into<String>,
10119) -> Result<Frame, RouterError> {
10120 control_error_body_frame(
10121 frame,
10122 ErrorBody {
10123 code: code.to_string(),
10124 message: message.into(),
10125 detail: None,
10126 },
10127 )
10128}
10129
10130pub(crate) fn control_error_body_frame(
10138 frame: &Frame,
10139 error: ErrorBody,
10140) -> Result<Frame, RouterError> {
10141 control_error_frame_with_flags(frame, error, daemon_error_flags())
10142}
10143
10144fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
10150 control_error_frame_with_flags(frame, error, control_flags())
10151}
10152
10153fn control_error_frame_with_flags(
10154 frame: &Frame,
10155 error: ErrorBody,
10156 flags: Flags,
10157) -> Result<Frame, RouterError> {
10158 let body = serde_json::to_vec(&error).map_err(|err| {
10159 RouterError::backend(
10160 0,
10161 frame.header.corr,
10162 format!("failed to encode control ERROR: {err}"),
10163 )
10164 })?;
10165
10166 Frame::build_with_version(
10167 response_version(frame),
10168 FrameType::Error,
10169 flags,
10170 0,
10171 0,
10172 frame.header.corr,
10173 body,
10174 )
10175 .map_err(RouterError::FrameBuild)
10176}
10177
10178pub(crate) fn control_response_body_frame<T: Serialize>(
10179 frame: &Frame,
10180 reply: &T,
10181 label: &'static str,
10182) -> Result<Frame, RouterError> {
10183 let body = serde_json::to_vec(reply).map_err(|err| {
10184 RouterError::backend(
10185 0,
10186 frame.header.corr,
10187 format!("failed to encode {label}: {err}"),
10188 )
10189 })?;
10190
10191 Frame::build_with_version(
10192 response_version(frame),
10193 FrameType::Response,
10194 control_flags(),
10195 0,
10196 0,
10197 frame.header.corr,
10198 body,
10199 )
10200 .map_err(RouterError::FrameBuild)
10201}
10202
10203fn forwarding_error_code(err: &ForwardingError) -> &'static str {
10233 match err {
10234 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
10235 ForwardingError::NoModuleConnection => "target_unavailable",
10236 ForwardingError::ModuleReloading { .. } => "module_reloading",
10237 ForwardingError::ClientRouteChannelExhausted { .. }
10238 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
10239 ForwardingError::StaleModuleEndpoint
10240 | ForwardingError::UnknownReservation { .. }
10241 | ForwardingError::ConnectionClosing { .. }
10242 | ForwardingError::ClientEgressClosed { .. }
10243 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
10244 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
10247 ForwardingError::RelayCorrelationExhausted
10248 | ForwardingError::RouteOpenBuild(_)
10249 | ForwardingError::Poisoned => "forwarding_error",
10250 }
10251}
10252
10253fn response_version(frame: &Frame) -> u8 {
10254 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
10255 frame.header.ver
10256 } else {
10257 PROTOCOL_VERSION
10258 }
10259}
10260
10261fn control_flags() -> Flags {
10262 Flags::new(false, Priority::Passive, false)
10263}
10264
10265fn send_goodbye_target_best_effort(
10270 counters: &DaemonCounters,
10271 target: &GoodbyeTarget,
10272 context: &'static str,
10273) {
10274 let Ok(frame) = Frame::build_with_version(
10275 target.negotiated_ver,
10276 FrameType::Goodbye,
10277 control_flags(),
10278 target.channel,
10279 target.epoch,
10280 0,
10281 Vec::new(),
10282 ) else {
10283 return;
10284 };
10285 crate::forwarding::send_module_route_goodbye(
10286 counters,
10287 &target.sink,
10288 frame,
10289 target.module_id.as_deref(),
10290 context,
10291 );
10292}
10293
10294pub(crate) fn send_route_control_pushes(
10295 forwarding: &ForwardingTable,
10296 routes: Vec<EndpointRoute>,
10297 push: ClientControlPush,
10298) {
10299 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
10300 for route in routes {
10301 let target = route.goodbye_target;
10302 if let Some((existing, channels)) = targets
10303 .iter_mut()
10304 .find(|(existing, _)| existing.connection_id == target.connection_id)
10305 {
10306 debug_assert_eq!(
10307 existing.negotiated_ver, target.negotiated_ver,
10308 "one connection cannot negotiate multiple frame versions"
10309 );
10310 if !channels.contains(&target.channel) {
10311 channels.push(target.channel);
10312 }
10313 continue;
10314 }
10315 let channel = target.channel;
10316 targets.push((target, vec![channel]));
10317 }
10318 for (target, mut channels) in targets {
10319 channels.sort_unstable();
10320 let mut push = push.clone();
10321 match &mut push {
10322 ClientControlPush::RouteClosing {
10323 channels: covered, ..
10324 }
10325 | ClientControlPush::RouteClosed {
10326 channels: covered, ..
10327 } => *covered = channels,
10328 }
10329 let body = match serde_json::to_vec(&push) {
10330 Ok(body) => body,
10331 Err(err) => {
10332 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
10333 continue;
10334 }
10335 };
10336 let frame = match Frame::build_with_version(
10337 target.negotiated_ver,
10338 FrameType::Push,
10339 control_flags(),
10340 0,
10341 0,
10342 0,
10343 body.clone(),
10344 ) {
10345 Ok(frame) => frame,
10346 Err(err) => {
10347 warn!(
10348 route_channel = target.channel,
10349 error = %err,
10350 "failed to build route lifecycle control PUSH frame"
10351 );
10352 continue;
10353 }
10354 };
10355 if let Err(err) = target.sink.try_send(frame) {
10356 if target.close_on_delivery_failure() {
10357 warn!(
10358 target_connection_id = target.connection_id.get(),
10359 route_channel = target.channel,
10360 error = %err,
10361 "route lifecycle control PUSH was not delivered to client; closing target connection"
10362 );
10363 let _ = forwarding.escalate_client_delivery_failure(
10364 target.connection_id,
10365 target.channel,
10366 target.epoch,
10367 CloseReason::new(
10368 "route_lifecycle_push_delivery_failed",
10369 format!(
10370 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
10371 target.channel
10372 ),
10373 ),
10374 crate::forwarding::UndeliveredFrame {
10375 module_id: target.module_id.as_deref(),
10376 sink: &target.sink,
10377 },
10378 );
10379 }
10380 }
10381 }
10382}
10383
10384#[cfg(test)]
10385mod tests {
10386 include!("control/agent_commands_tests.rs");
10387 include!("control/resource_revision_tests.rs");
10388 include!("control/route_open_recovery_tests.rs");
10389 #[tokio::test]
10390 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
10391 let registry = Arc::new(Registry::default());
10392 let handler = ControlHandler::new(registry);
10393 assert_eq!(
10394 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10395 None
10396 );
10397 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
10398 hello_via_sink(
10399 &handler,
10400 &ctx,
10401 &mut rx,
10402 hello_frame("daemon", PROTOCOL_VERSION, 1),
10403 )
10404 .await;
10405 assert_eq!(
10406 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10407 Some("daemon".into())
10408 );
10409 assert_eq!(
10410 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
10411 None
10412 );
10413 assert_eq!(
10414 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
10415 None,
10416 "a registered module is not necessarily supervised"
10417 );
10418 assert_eq!(
10419 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
10420 None,
10421 "aggregate provenance has no single target"
10422 );
10423 assert_eq!(
10424 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
10425 None
10426 );
10427 }
10428
10429 mod peer_manifest {
10430 include!("peer_manifest_tests.rs");
10431 }
10432 mod activity_read {
10433 include!("activity_read_tests.rs");
10434 }
10435
10436 mod lifecycle_activity {
10437 include!("control_lifecycle_tests.rs");
10438 }
10439 #[cfg(unix)]
10440 #[tokio::test]
10441 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
10442 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
10443 let path = dir.join("subc.jsonc");
10444 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
10445 "program":"/bin/sleep","args":["60"],"protocol":"none",
10446 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
10447 }}}).to_string()).unwrap();
10448 let mut configured = crate::daemon_config::load(&path)
10449 .unwrap()
10450 .unwrap()
10451 .modules
10452 .pop()
10453 .unwrap();
10454 let registry = std::sync::Arc::new(crate::Registry::default());
10455 let handle = crate::SupervisorHandle::new();
10456 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
10457 .with_handle(handle.clone());
10458 let module = supervisor
10459 .supervise_configured_with_health(
10460 configured.module_spec(),
10461 true,
10462 configured.health.clone(),
10463 None,
10464 configured.restart,
10465 )
10466 .unwrap();
10467 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
10468 let before = module.status().unwrap().pid;
10469 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
10470 configured.health.cadence = std::time::Duration::from_secs(3600);
10471 let health_only = handler
10472 .reconcile_supervised_modules(
10473 &supervisor,
10474 vec![configured.clone()],
10475 false,
10476 Principal::Direct,
10477 )
10478 .await
10479 .unwrap();
10480 assert!(
10481 health_only.0.changed_pending_reload.is_empty(),
10482 "health policy is already applied live"
10483 );
10484 assert_eq!(module.status().unwrap().pid, before);
10485 assert_eq!(
10486 module.configuration().unwrap().1.http,
10487 configured.health.http
10488 );
10489 configured.args = vec!["61".into()];
10490 let launch = handler
10491 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
10492 .await
10493 .unwrap();
10494 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
10495 assert_eq!(
10496 module.status().unwrap().pid,
10497 before,
10498 "a launch edit is stored until reload"
10499 );
10500 module.drain().await.unwrap();
10501 }
10502 use cortexkit_test_support::ScratchDir;
10503 use std::{
10504 collections::BTreeMap,
10505 fmt,
10506 path::PathBuf,
10507 sync::{Arc, Mutex},
10508 time::Duration,
10509 };
10510
10511 use serde_json::{json, Value};
10512 use subc_protocol::{
10513 manifest::{
10514 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
10515 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
10516 },
10517 session::HealthStatus,
10518 FrameType,
10519 };
10520
10521 use super::*;
10522 use crate::{
10523 forwarding::{DataRoute, DataRouteState},
10524 registry::ChannelState,
10525 router::FrameSink,
10526 stderr_tail::DEFAULT_MAX_LINE_BYTES,
10527 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
10528 RouteCtx, Router,
10529 };
10530 use tokio::{
10531 sync::mpsc,
10532 time::{sleep, Instant},
10533 };
10534 use tracing::{
10535 field::{Field, Visit},
10536 Event, Subscriber,
10537 };
10538 use tracing_subscriber::{layer::Context, prelude::*, Layer};
10539
10540 fn fake_aft_stub_path() -> PathBuf {
10559 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
10560 assert!(
10561 path.exists(),
10562 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
10563 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
10564 path.display()
10565 );
10566 path
10567 }
10568
10569 fn client_retries(code: &str) -> bool {
10574 subc_protocol::error_codes::is_retryable_route_open(code)
10575 }
10576
10577 #[test]
10588 fn retryability_of_forwarding_codes_matches_the_failure() {
10589 let transient = [
10592 ForwardingError::NoModuleConnection,
10593 ForwardingError::ModuleReloading {
10594 module_id: "m".into(),
10595 },
10596 ForwardingError::StaleModuleEndpoint,
10597 ForwardingError::UnknownReservation {
10598 client_channel: 1,
10599 module_channel: 1,
10600 },
10601 ForwardingError::ConnectionClosing {
10602 connection_id: ConnectionId::new(1),
10603 },
10604 ForwardingError::ClientEgressClosed {
10605 connection_id: ConnectionId::new(1),
10606 },
10607 ForwardingError::ModuleEgressUnavailable {
10608 connection_id: ConnectionId::new(1),
10609 },
10610 ];
10611 for err in transient {
10612 let code = forwarding_error_code(&err);
10613 assert!(
10614 client_retries(code),
10615 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
10616 );
10617 }
10618
10619 let permanent = [
10624 ForwardingError::ConnectionRoleConflict {
10625 connection_id: ConnectionId::new(1),
10626 },
10627 ForwardingError::ClientRouteChannelExhausted {
10628 connection_id: ConnectionId::new(1),
10629 },
10630 ForwardingError::ModuleRouteChannelExhausted {
10631 endpoint: ModuleEndpointId {
10632 connection_id: ConnectionId::new(1),
10633 generation: 1,
10634 },
10635 },
10636 ForwardingError::RelayCorrelationExhausted,
10637 ForwardingError::RouteOpenBuild("x".into()),
10638 ForwardingError::Poisoned,
10639 ];
10640 for err in permanent {
10641 let code = forwarding_error_code(&err);
10642 assert!(
10643 !client_retries(code),
10644 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
10645 );
10646 }
10647 }
10648
10649 #[tokio::test]
10677 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
10678 let handler = ControlHandler::default();
10679 let frame =
10680 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
10681
10682 let stamped = handler
10686 .route_open_principal(ConnectionId::new(1), &frame, None)
10687 .unwrap()
10688 .unwrap();
10689 assert_eq!(
10690 stamped,
10691 Principal::Direct,
10692 "a caller that proved nothing must not be stamped as a supervised module"
10693 );
10694
10695 let forged = handler
10700 .route_open_principal(
10701 ConnectionId::new(1),
10702 &frame,
10703 Some(ConsumerIdentity {
10704 module_id: "aft".to_string(),
10705 launch_nonce: "not-a-real-nonce".to_string(),
10706 }),
10707 )
10708 .unwrap();
10709 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
10710 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
10711 }
10712
10713 #[test]
10730 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
10731 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
10732 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
10733 let ClientControlRequest::RouteOpen {
10734 consumer_identity, ..
10735 } = parsed
10736 else {
10737 panic!("route.open body must parse as RouteOpen");
10738 };
10739 assert_eq!(
10740 consumer_identity,
10741 Some(ConsumerIdentity {
10742 module_id: "aft".to_string(),
10743 launch_nonce: "n".to_string(),
10744 }),
10745 "the wire field name must reach the value route_open_principal reads"
10746 );
10747 }
10748
10749 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
10750 ModuleManifest::builder(module_id, "0.1.0")
10751 .protocol_ver(protocol_ver)
10752 .provides(vec![ProviderRole::ToolProvider {
10753 tools: vec![Tool {
10754 name: "read".to_string(),
10755 description: None,
10756 execution_mode: ExecutionMode::Pure,
10757 schema: json!({"type": "object"}),
10758 }],
10759 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
10760 concurrency: Concurrency::ModuleManaged,
10761 emits_push: true,
10762 sub_supervises: true,
10763 }])
10764 .build()
10765 }
10766
10767 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
10768 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
10769 }
10770
10771 fn hello_frame_with_control_ops(
10772 module_id: &str,
10773 protocol_ver: u8,
10774 corr: u64,
10775 control_ops: Option<Vec<String>>,
10776 ) -> Frame {
10777 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
10778 }
10779
10780 fn hello_frame_with_nonce(
10781 module_id: &str,
10782 protocol_ver: u8,
10783 corr: u64,
10784 launch_nonce: Option<&str>,
10785 ) -> Frame {
10786 hello_frame_full(
10787 module_id,
10788 protocol_ver,
10789 corr,
10790 None,
10791 launch_nonce.map(ToOwned::to_owned),
10792 )
10793 }
10794
10795 fn hello_frame_full(
10796 module_id: &str,
10797 protocol_ver: u8,
10798 corr: u64,
10799 control_ops: Option<Vec<String>>,
10800 launch_nonce: Option<String>,
10801 ) -> Frame {
10802 let body = serde_json::to_vec(&ModuleHelloBody {
10803 manifest: manifest(module_id, protocol_ver),
10804 protocol_ver,
10805 control_ops,
10806 launch_nonce,
10807 })
10808 .unwrap();
10809 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
10810 }
10811
10812 fn hello_frame_with_rich_ui(
10813 module_id: &str,
10814 protocol_ver: u8,
10815 corr: u64,
10816 launch_nonce: Option<&str>,
10817 rich_ui: Value,
10818 ) -> Frame {
10819 let mut body = serde_json::to_value(ModuleHelloBody {
10820 manifest: manifest(module_id, protocol_ver),
10821 protocol_ver,
10822 control_ops: None,
10823 launch_nonce: launch_nonce.map(str::to_string),
10824 })
10825 .unwrap();
10826 body["manifest"]["rich_ui"] = rich_ui;
10827 Frame::build(
10828 FrameType::Hello,
10829 control_flags(),
10830 0,
10831 0,
10832 corr,
10833 serde_json::to_vec(&body).unwrap(),
10834 )
10835 .unwrap()
10836 }
10837
10838 fn non_routable_hello_frame_with_control_ops(
10839 module_id: &str,
10840 corr: u64,
10841 control_ops: Option<Vec<String>>,
10842 ) -> Frame {
10843 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
10844 manifest.provides.clear();
10845 let body = serde_json::to_vec(&ModuleHelloBody {
10846 manifest,
10847 protocol_ver: PROTOCOL_VERSION,
10848 control_ops,
10849 launch_nonce: None,
10850 })
10851 .unwrap();
10852 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
10853 }
10854
10855 fn capability_grammar_hello_frame(
10856 capabilities: Value,
10857 runtime_computed: Option<Value>,
10858 corr: u64,
10859 ) -> Frame {
10860 let mut body = serde_json::to_value(ModuleHelloBody {
10861 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
10862 protocol_ver: PROTOCOL_VERSION,
10863 control_ops: None,
10864 launch_nonce: None,
10865 })
10866 .expect("HELLO body serializes");
10867 body["manifest"]["capabilities"] = capabilities;
10868 if let Some(runtime_computed) = runtime_computed {
10869 body["runtime_computed"] = runtime_computed;
10870 }
10871 Frame::build(
10872 FrameType::Hello,
10873 control_flags(),
10874 0,
10875 0,
10876 corr,
10877 serde_json::to_vec(&body).expect("HELLO body reserializes"),
10878 )
10879 .expect("HELLO frame builds")
10880 }
10881
10882 fn channel_request(channel: u16, corr: u64) -> Frame {
10883 Frame::build(
10884 FrameType::Request,
10885 Flags::new(true, Priority::Interactive, false),
10886 channel,
10887 0,
10888 corr,
10889 b"opaque".to_vec(),
10890 )
10891 .unwrap()
10892 }
10893
10894 fn route_ctx(
10895 connection_id: ConnectionId,
10896 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
10897 let (tx, rx) = mpsc::channel(8);
10898 (
10899 RouteCtx {
10900 connection_id,
10901 egress: FrameSink::new(tx),
10902 },
10903 rx,
10904 )
10905 }
10906
10907 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
10908 serde_json::from_slice(&frame.body).unwrap()
10909 }
10910
10911 async fn hello_via_sink(
10916 handler: &ControlHandler,
10917 ctx: &RouteCtx,
10918 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10919 hello: Frame,
10920 ) -> Frame {
10921 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
10922 assert!(
10923 replies.is_empty(),
10924 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
10925 );
10926 let ack = rx
10927 .try_recv()
10928 .expect("HELLO_ACK is queued on the module sink")
10929 .frame;
10930 assert_eq!(ack.header.ty, FrameType::HelloAck);
10931 ack
10932 }
10933
10934 fn parse_error(frame: &Frame) -> Value {
10935 serde_json::from_slice(&frame.body).unwrap()
10936 }
10937
10938 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
10939 serde_json::from_slice(&frame.body).unwrap()
10940 }
10941
10942 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
10943 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
10944 route_channel,
10945 route_epoch: 0,
10946 kind,
10947 })
10948 .unwrap();
10949 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10950 }
10951
10952 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
10953 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
10954 module_id: module_id.to_string(),
10955 })
10956 .unwrap();
10957 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10958 }
10959
10960 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
10961 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
10962 }
10963
10964 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
10966 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
10967 let mut frame = route_open_frame(corr, module_id, root);
10968 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
10969 body["identity"]["project_root"] = serde_json::json!(path.as_path());
10970 frame.body = serde_json::to_vec(&body).unwrap();
10971 frame
10972 }
10973
10974 fn route_open_frame_with_consumer_capabilities(
10975 corr: u64,
10976 module_id: &str,
10977 _project_root: ScratchDir,
10978 consumer_capabilities: Option<Vec<String>>,
10979 ) -> Frame {
10980 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
10981 busy_refusals: None,
10982 target: RouteTarget::ToolProvider {
10983 module_id: module_id.to_string(),
10984 },
10985 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
10986 consumer_identity: None,
10987 consumer_capabilities,
10988 role_versions: None,
10989 admission_facts: None,
10990 scope: None,
10991 })
10992 .unwrap();
10993 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10994 }
10995
10996 fn route_open_frame_with_role_versions(
10997 corr: u64,
10998 module_id: &str,
10999 _project_root: ScratchDir,
11000 role_versions: Option<BTreeMap<String, String>>,
11001 ) -> Frame {
11002 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11003 busy_refusals: None,
11004 target: RouteTarget::ToolProvider {
11005 module_id: module_id.to_string(),
11006 },
11007 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11008 consumer_identity: None,
11009 consumer_capabilities: None,
11010 role_versions,
11011 admission_facts: None,
11012 scope: None,
11013 })
11014 .unwrap();
11015 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11016 }
11017
11018 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
11019 entries
11020 .iter()
11021 .map(|(role, version)| (role.to_string(), version.to_string()))
11022 .collect()
11023 }
11024
11025 fn route_open_frame_with_admission_facts(
11026 corr: u64,
11027 module_id: &str,
11028 _project_root: ScratchDir,
11029 consumer_identity: Option<subc_control::ConsumerIdentity>,
11030 facts: Option<Value>,
11031 ) -> Frame {
11032 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11033 busy_refusals: None,
11034 target: RouteTarget::ToolProvider {
11035 module_id: module_id.to_string(),
11036 },
11037 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11038 consumer_identity,
11039 consumer_capabilities: None,
11040 role_versions: None,
11041 admission_facts: facts,
11042 scope: None,
11043 })
11044 .unwrap();
11045 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11046 }
11047
11048 #[derive(Clone, Default)]
11049 struct EventCapture {
11050 events: Arc<Mutex<Vec<CapturedEvent>>>,
11051 }
11052
11053 #[derive(Clone, Debug)]
11054 struct CapturedEvent {
11055 target: String,
11056 level: tracing::Level,
11057 fields: BTreeMap<String, String>,
11058 }
11059
11060 impl EventCapture {
11061 fn events(&self) -> Vec<CapturedEvent> {
11062 self.events.lock().unwrap().clone()
11063 }
11064 }
11065
11066 impl<S> Layer<S> for EventCapture
11067 where
11068 S: Subscriber,
11069 {
11070 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
11071 let mut visitor = EventFieldVisitor::default();
11072 event.record(&mut visitor);
11073 self.events.lock().unwrap().push(CapturedEvent {
11074 target: event.metadata().target().to_string(),
11075 level: *event.metadata().level(),
11076 fields: visitor.fields,
11077 });
11078 }
11079 }
11080
11081 #[derive(Default)]
11082 struct EventFieldVisitor {
11083 fields: BTreeMap<String, String>,
11084 }
11085
11086 impl Visit for EventFieldVisitor {
11087 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
11088 self.fields
11089 .insert(field.name().to_string(), format!("{value:?}"));
11090 }
11091 }
11092
11093 fn health_response(corr: u64, status: HealthStatus) -> Frame {
11094 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
11095 status,
11096 detail: Some("warming".to_string()),
11097 metrics: Some(json!({"queue_depth": 3})),
11098 })
11099 .unwrap();
11100 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11101 }
11102
11103 fn route_bind_ack(corr: u64) -> Frame {
11104 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
11105 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11106 }
11107
11108 fn unique_project_root(label: &str) -> ScratchDir {
11109 ScratchDir::new(label)
11110 }
11111
11112 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
11113 match parse_route_poll(frame) {
11114 ClientControlResponse::RoutePoll {
11115 status: None,
11116 live: Some(live),
11117 ..
11118 } => assert_eq!(live, expected_live),
11119 other => panic!("unexpected route.poll response: {other:?}"),
11120 }
11121 }
11122
11123 fn bind_liveness_route(
11124 registry: &Registry,
11125 forwarding: &ForwardingTable,
11126 module_id: &str,
11127 ) -> (RouteCtx, u16, u32) {
11128 let module_connection = ConnectionId::new(101);
11129 let client_connection = ConnectionId::new(202);
11130 let registration = registry
11131 .register_with_control_ops(
11132 manifest(module_id, PROTOCOL_VERSION),
11133 PROTOCOL_VERSION,
11134 module_connection,
11135 module_baseline_control_ops(),
11136 )
11137 .unwrap();
11138 let (module_tx, _module_rx) = mpsc::channel(8);
11139 let endpoint = forwarding
11140 .register_module_connection(
11141 module_connection,
11142 module_id.to_string(),
11143 PROTOCOL_VERSION,
11144 manifest_concurrency(®istration.manifest),
11145 FrameSink::new(module_tx),
11146 )
11147 .unwrap();
11148 let (client_ctx, _client_rx) = route_ctx(client_connection);
11149 let pending = forwarding
11150 .begin_route_bind_relay_for_test(
11151 client_connection,
11152 client_ctx.egress.clone(),
11153 1,
11154 module_id,
11155 )
11156 .unwrap();
11157 assert_eq!(pending.endpoint, endpoint);
11158 let route_channel = pending.client_channel;
11159 let route_epoch = pending.client_epoch;
11160 forwarding
11161 .complete_pending_relay(
11162 module_connection,
11163 pending.corr,
11164 RouteBindRelayOutcome::Accepted,
11165 )
11166 .unwrap();
11167 (client_ctx, route_channel, route_epoch)
11168 }
11169
11170 struct FakeProcessLiveness {
11171 live: Option<bool>,
11172 }
11173
11174 impl ModuleProcessLiveness for FakeProcessLiveness {
11175 fn process_live(&self, _module_id: &str) -> Option<bool> {
11176 self.live
11177 }
11178 }
11179
11180 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11181 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
11182 {
11183 let registry = Arc::new(Registry::default());
11184 let supervisor_handle = SupervisorHandle::new();
11185 let supervisor = Supervisor::new_for_test(
11186 Arc::clone(®istry),
11187 RestartPolicy::new(1, Duration::from_millis(10)),
11188 )
11189 .with_handle(supervisor_handle.clone());
11190 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
11191 let module = supervisor
11192 .spawn(ModuleSpec {
11193 carry_containment: false,
11194 module_id: "stderr-tail-wire".to_string(),
11195 program: fake_aft_stub_path(),
11196 args: Vec::new(),
11197 env: vec![
11198 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
11199 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
11200 ],
11201 reserved: false,
11202 reserved_prefixes: Vec::new(),
11203 protocol: ModuleProtocol::Subc,
11204 overlap: Default::default(),
11205 })
11206 .unwrap();
11207
11208 let deadline = Instant::now() + Duration::from_secs(5);
11209 loop {
11210 let tail = module.stderr_tail(None, None);
11211 if tail
11212 .entries
11213 .iter()
11214 .any(|entry| matches!(entry, TailEntry::ProcessStart))
11215 && tail.entries.iter().any(|entry| {
11216 matches!(
11217 entry,
11218 TailEntry::Line {
11219 truncated: true,
11220 ..
11221 }
11222 )
11223 })
11224 {
11225 break;
11226 }
11227 assert!(
11228 Instant::now() < deadline,
11229 "module did not produce a truncated line and restart boundary: {tail:?}"
11230 );
11231 sleep(Duration::from_millis(10)).await;
11232 }
11233
11234 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11235 let request = ClientControlRequest::SupervisorStderrTail {
11236 module_id: "stderr-tail-wire".to_string(),
11237 max_lines: None,
11238 max_bytes: None,
11239 };
11240 let frame = Frame::build(
11241 FrameType::Request,
11242 control_flags(),
11243 0,
11244 0,
11245 1,
11246 serde_json::to_vec(&request).unwrap(),
11247 )
11248 .unwrap();
11249 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11250 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11251 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
11252 serde_json::from_slice(&responses[0].body).unwrap()
11253 else {
11254 panic!("expected supervisor.stderr_tail response");
11255 };
11256
11257 assert!(
11258 tail.entries
11259 .iter()
11260 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
11261 "the control response lost the restart boundary"
11262 );
11263 let Some(StderrTailEntry::Line {
11264 text,
11265 truncated,
11266 at_ms,
11267 }) = tail.entries.iter().find(|entry| {
11268 matches!(
11269 entry,
11270 StderrTailEntry::Line {
11271 truncated: true,
11272 ..
11273 }
11274 )
11275 })
11276 else {
11277 panic!("the control response lost the truncated line");
11278 };
11279 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
11280 assert!(*truncated);
11281 assert!(
11282 at_ms.is_some(),
11283 "the control response lost the line's capture time"
11284 );
11285 }
11286
11287 #[tokio::test(flavor = "current_thread")]
11292 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
11293 let dir = ScratchDir::new("terminals-off-worker");
11294 let journal_path = dir.join("terminals.jsonl");
11295 let registry = Arc::new(Registry::default());
11296 let supervisor_handle = SupervisorHandle::new();
11297 let supervisor =
11298 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11299 .with_handle(supervisor_handle.clone())
11300 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
11301 let module = supervisor
11302 .spawn(ModuleSpec {
11303 carry_containment: false,
11304 module_id: "terminal-off-worker".to_string(),
11305 program: fake_aft_stub_path(),
11306 args: Vec::new(),
11307 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11308 reserved: false,
11309 reserved_prefixes: Vec::new(),
11310 protocol: ModuleProtocol::Subc,
11311 overlap: Default::default(),
11312 })
11313 .unwrap();
11314 let deadline = Instant::now() + Duration::from_secs(5);
11315 while module.terminal_history().entries.len() != 2 {
11316 assert!(Instant::now() < deadline, "module did not record two exits");
11317 sleep(Duration::from_millis(10)).await;
11318 }
11319
11320 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
11321 let handler =
11322 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
11323 let frame = Frame::build(
11324 FrameType::Request,
11325 control_flags(),
11326 0,
11327 0,
11328 1,
11329 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
11330 module_id: "terminal-off-worker".to_string(),
11331 })
11332 .unwrap(),
11333 )
11334 .unwrap();
11335 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11336 let spawned_at = std::time::Instant::now();
11337 let read = tokio::spawn({
11338 let handler = Arc::clone(&handler);
11339 async move { handler.handle_control_frame(&ctx, frame).await }
11340 });
11341 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
11344 .await
11345 .unwrap()
11346 .expect("the history read reached its pause");
11347 let elapsed = spawned_at.elapsed();
11348 assert!(
11349 elapsed < Duration::from_secs(2) && !read.is_finished(),
11350 "this task could not run while the history read was paused \
11351 (resumed after {elapsed:?}, read finished: {})",
11352 read.is_finished()
11353 );
11354
11355 drop(release);
11356 let responses = read.await.unwrap().unwrap();
11357 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11358 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
11359 panic!("expected supervisor.terminals response");
11360 };
11361 assert_eq!(terminals.entries.len(), 2);
11362 assert_eq!(terminals.journal_skipped_lines, 0);
11363 assert_eq!(terminals.journal_read_errors, 0);
11364 }
11365
11366 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11367 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
11368 let registry = Arc::new(Registry::default());
11369 let supervisor_handle = SupervisorHandle::new();
11370 let supervisor =
11371 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11372 .with_handle(supervisor_handle.clone());
11373 let module = supervisor
11374 .spawn(ModuleSpec {
11375 carry_containment: false,
11376 module_id: "terminal-golden".to_string(),
11377 program: fake_aft_stub_path(),
11378 args: Vec::new(),
11379 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11380 reserved: false,
11381 reserved_prefixes: Vec::new(),
11382 protocol: ModuleProtocol::Subc,
11383 overlap: Default::default(),
11384 })
11385 .unwrap();
11386
11387 let deadline = Instant::now() + Duration::from_secs(5);
11388 while module.terminal_history().entries.len() != 2 {
11389 assert!(
11390 Instant::now() < deadline,
11391 "module did not retain two terminal exits: {:?}",
11392 module.terminal_history()
11393 );
11394 sleep(Duration::from_millis(10)).await;
11395 }
11396
11397 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11398 let request = ClientControlRequest::SupervisorTerminals {
11399 module_id: "terminal-golden".to_string(),
11400 };
11401 let frame = Frame::build(
11402 FrameType::Request,
11403 control_flags(),
11404 0,
11405 0,
11406 1,
11407 serde_json::to_vec(&request).unwrap(),
11408 )
11409 .unwrap();
11410 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11411 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11412 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11413 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
11414 panic!("expected supervisor.terminals response");
11415 };
11416 assert_eq!(terminals.entries.len(), 2);
11417 assert_eq!(terminals.dropped, 0);
11418
11419 let mut rendered = serde_json::to_value(response).unwrap();
11420 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
11423 for (index, entry) in rendered["entries"]
11424 .as_array_mut()
11425 .expect("terminal response entries array")
11426 .iter_mut()
11427 .enumerate()
11428 {
11429 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
11430 }
11431
11432 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11433 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
11434 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
11435 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11436 std::fs::write(&golden_path, &serialized).unwrap();
11437 }
11438 let expected: Value =
11439 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
11440 assert_eq!(rendered, expected);
11441 }
11442
11443 #[test]
11444 fn hello_registers_manifest_and_returns_ack() {
11445 let registry = Arc::new(Registry::default());
11446 let handler = ControlHandler::new(Arc::clone(®istry));
11447 let conn = ConnectionId::new(1);
11448
11449 let responses = handler
11450 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
11451 .unwrap();
11452
11453 assert_eq!(responses.len(), 1);
11454 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11455 assert_eq!(responses[0].header.channel, 0);
11456 assert_eq!(responses[0].header.corr, 7);
11457 let ack = parse_ack(&responses[0]);
11458 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11459 assert!(ack
11460 .subc_capabilities
11461 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
11462 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
11463 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
11464 assert!(ack
11465 .subc_ops
11466 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
11467 assert!(ack
11468 .subc_ops
11469 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
11470 assert!(ack
11471 .subc_ops
11472 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
11473
11474 let registration = registry.get_module("aft").unwrap().unwrap();
11475 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
11476 assert_eq!(registration.state, ChannelState::Active);
11477 assert_eq!(registration.connection_id, conn);
11478 assert_eq!(registration.control_ops, module_baseline_control_ops());
11479 assert!(registration.manifest.display_name.is_none());
11480 }
11481
11482 #[test]
11483 fn hello_display_name_refuses_invalid_manifest_by_field() {
11484 let registry = Arc::new(Registry::default());
11485 let handler = ControlHandler::new(Arc::clone(®istry));
11486 for (index, invalid) in [
11487 json!(""),
11488 json!("a".repeat(41)),
11489 json!(" Routing"),
11490 json!("a\nb"),
11491 json!("a\u{202e}b"),
11492 json!(7),
11493 ]
11494 .into_iter()
11495 .enumerate()
11496 {
11497 let module_id = format!("bad-display-name-{index}");
11498 let mut body = serde_json::to_value(ModuleHelloBody {
11499 manifest: manifest(&module_id, PROTOCOL_VERSION),
11500 protocol_ver: PROTOCOL_VERSION,
11501 control_ops: None,
11502 launch_nonce: None,
11503 })
11504 .unwrap();
11505 body["manifest"]["display_name"] = invalid;
11506 let hello = Frame::build(
11507 FrameType::Hello,
11508 control_flags(),
11509 0,
11510 0,
11511 7,
11512 serde_json::to_vec(&body).unwrap(),
11513 )
11514 .unwrap();
11515 let response = handler
11516 .handle_control(ConnectionId::new(100 + index as u64), hello)
11517 .unwrap();
11518 assert_eq!(response[0].header.ty, FrameType::Error);
11519 let error = parse_error(&response[0]);
11520 assert_eq!(error["code"], "invalid_manifest");
11521 assert!(error["message"].as_str().unwrap().contains("display_name"));
11522 assert!(registry.get_module(&module_id).unwrap().is_none());
11523 }
11524 }
11525
11526 #[test]
11527 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
11528 let invalid_identifiers = [
11529 ("case_change", "credentials-Provider/v1"),
11530 ("leading_zero", "credentials-provider/v01"),
11531 ("trailing_hyphen", "credentials-provider-/v1"),
11532 ("consecutive_hyphens", "credentials--provider/v1"),
11533 ("uppercase", "Credentials-provider/v1"),
11534 ("missing_v", "credentials-provider/1"),
11535 ("whitespace", "credentials provider/v1"),
11536 ("zero_version", "credentials-provider/v0"),
11537 ("out_of_range_version", "credentials-provider/v4294967296"),
11538 (
11539 "overlength_name",
11540 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
11541 ),
11542 ];
11543 let mut cases = invalid_identifiers
11544 .into_iter()
11545 .map(|(name, identifier)| {
11546 (
11547 format!("identifier_{name}"),
11548 "capabilities.provides[0]".to_string(),
11549 identifier.to_string(),
11550 json!({ "provides": [identifier] }),
11551 None,
11552 )
11553 })
11554 .collect::<Vec<_>>();
11555 cases.extend([
11556 (
11557 "unknown_need".to_string(),
11558 "capabilities.requires[0].need".to_string(),
11559 "deferred".to_string(),
11560 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
11561 None,
11562 ),
11563 (
11564 "duplicate_provides".to_string(),
11565 "capabilities.provides[1]".to_string(),
11566 "credentials-provider/v1".to_string(),
11567 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
11568 None,
11569 ),
11570 (
11571 "duplicate_must_never_reach".to_string(),
11572 "capabilities.must_never_reach[1]".to_string(),
11573 "credentials-provider/v1".to_string(),
11574 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
11575 None,
11576 ),
11577 (
11578 "duplicate_requires_same_need".to_string(),
11579 "capabilities.requires[1]".to_string(),
11580 "credentials-provider/v1".to_string(),
11581 json!({ "requires": [
11582 { "capability": "credentials-provider/v1", "need": "required" },
11583 { "capability": "credentials-provider/v1", "need": "required" }
11584 ] }),
11585 None,
11586 ),
11587 (
11588 "duplicate_requires_conflicting_need".to_string(),
11589 "capabilities.requires[1]".to_string(),
11590 "credentials-provider/v1".to_string(),
11591 json!({ "requires": [
11592 { "capability": "credentials-provider/v1", "need": "required" },
11593 { "capability": "credentials-provider/v1", "need": "optional" }
11594 ] }),
11595 None,
11596 ),
11597 (
11598 "capabilities_root_pointer".to_string(),
11599 "runtime_computed[0]".to_string(),
11600 "/capabilities".to_string(),
11601 json!({}),
11602 Some(json!(["/capabilities"])),
11603 ),
11604 (
11605 "capabilities_descendant_pointer".to_string(),
11606 "runtime_computed[0]".to_string(),
11607 "/capabilities/provides".to_string(),
11608 json!({}),
11609 Some(json!(["/capabilities/provides"])),
11610 ),
11611 (
11612 "malformed_pointer_without_leading_slash".to_string(),
11613 "runtime_computed[0]".to_string(),
11614 "capabilities".to_string(),
11615 json!({}),
11616 Some(json!(["capabilities"])),
11617 ),
11618 (
11619 "malformed_pointer_escape".to_string(),
11620 "runtime_computed[0]".to_string(),
11621 "/roles/~2/tools".to_string(),
11622 json!({}),
11623 Some(json!(["/roles/~2/tools"])),
11624 ),
11625 (
11626 "unknown_capabilities_field".to_string(),
11627 "capabilities.future".to_string(),
11628 "<array>".to_string(),
11629 json!({ "future": [] }),
11630 None,
11631 ),
11632 ]);
11633
11634 for (index, (name, field, value, capabilities, runtime_computed)) in
11635 cases.into_iter().enumerate()
11636 {
11637 let registry = Arc::new(Registry::default());
11638 let handler = ControlHandler::new(Arc::clone(®istry));
11639 let response = handler
11640 .handle_control(
11641 ConnectionId::new((index + 1) as u64),
11642 capability_grammar_hello_frame(
11643 capabilities,
11644 runtime_computed,
11645 index as u64 + 1,
11646 ),
11647 )
11648 .expect("invalid HELLO returns a refusal");
11649
11650 assert_eq!(response.len(), 1, "{name} must emit one refusal");
11651 let error = parse_error(&response[0]);
11652 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
11653 let message = error["message"]
11654 .as_str()
11655 .expect("error message is a string");
11656 assert!(
11657 message.contains(&field),
11658 "{name}: field missing from {message}"
11659 );
11660 assert!(
11661 message.contains(&value),
11662 "{name}: value missing from {message}"
11663 );
11664 assert_eq!(
11665 registry
11666 .active_registration_count()
11667 .expect("registry reads"),
11668 0,
11669 "{name}: refused HELLO must not create a catalog entry"
11670 );
11671 }
11672 }
11673
11674 #[test]
11675 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
11676 let registry = Arc::new(Registry::default());
11677 let handler = ControlHandler::new(Arc::clone(®istry));
11678 let capabilities = json!({
11679 "provides": ["credentials-provider/v1"],
11680 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
11681 "must_never_reach": ["federation-transport/v1"]
11682 });
11683 let response = handler
11684 .handle_control(
11685 ConnectionId::new(99),
11686 capability_grammar_hello_frame(
11687 capabilities.clone(),
11688 Some(json!(["/roles/0/tools"])),
11689 99,
11690 ),
11691 )
11692 .expect("valid HELLO registers");
11693 assert_eq!(response[0].header.ty, FrameType::HelloAck);
11694
11695 let request = Frame::build(
11696 FrameType::Request,
11697 control_flags(),
11698 0,
11699 0,
11700 100,
11701 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11702 .expect("catalog request serializes"),
11703 )
11704 .expect("catalog request frame builds");
11705 let response = handler
11706 .handle_catalog_list(request, None)
11707 .expect("catalog list succeeds");
11708 let ClientControlResponse::CatalogList { modules, .. } =
11709 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
11710 else {
11711 panic!("catalog request must return catalog.list");
11712 };
11713 assert_eq!(modules.len(), 1);
11714 assert_eq!(
11715 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
11716 capabilities
11717 );
11718 }
11719
11720 #[test]
11721 fn catalog_list_mirrors_management_operation_description() {
11722 let registry = Arc::new(Registry::default());
11723 let handler = ControlHandler::new(Arc::clone(®istry));
11724 let description = "List managed records and return their identifiers and metadata.";
11725 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
11726 manifest.provides = vec![ProviderRole::ManagementSurface {
11727 operations: vec![ManagementOperation {
11728 name: "records.list".to_string(),
11729 kind: ManagementOperationKind::Query,
11730 description: Some(description.to_string()),
11731 }],
11732 config_schema: json!({"type": "object"}),
11733 observability: vec![ObservabilitySurface {
11734 name: "records.stats".to_string(),
11735 kind: ObservabilityKind::Snapshot,
11736 }],
11737 identity_scope: vec![IdentityScope::Project],
11738 concurrency: Concurrency::ModuleManaged,
11739 }];
11740 registry
11741 .register_with_control_ops(
11742 manifest,
11743 PROTOCOL_VERSION,
11744 ConnectionId::new(99),
11745 Vec::new(),
11746 )
11747 .expect("described management manifest registers");
11748
11749 let request = Frame::build(
11750 FrameType::Request,
11751 control_flags(),
11752 0,
11753 0,
11754 100,
11755 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11756 .expect("catalog request serializes"),
11757 )
11758 .expect("catalog request frame builds");
11759 let response = handler
11760 .handle_catalog_list(request, None)
11761 .expect("catalog list succeeds");
11762 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11763 assert_eq!(
11764 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
11765 "catalog.list must preserve the declared operation description verbatim"
11766 );
11767 }
11768
11769 #[test]
11770 fn catalog_list_mirrors_registered_operation_delivery() {
11771 let registry = Arc::new(Registry::default());
11772 let handler = ControlHandler::new(Arc::clone(®istry));
11773 let delivery =
11774 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
11775 let mut body = serde_json::to_value(ModuleHelloBody {
11776 manifest: manifest("watch-management", PROTOCOL_VERSION),
11777 protocol_ver: PROTOCOL_VERSION,
11778 control_ops: None,
11779 launch_nonce: None,
11780 })
11781 .unwrap();
11782 body["manifest"]["provides"] = json!([{
11783 "role": "management_surface",
11784 "operations": [
11785 {"name": "status", "kind": "query"},
11786 {"name": "future", "kind": "query"},
11787 {"name": "reset", "kind": "mutate"}
11788 ],
11789 "config_schema": {}, "observability": [], "identity_scope": []
11790 }]);
11791 body["manifest"]["operation_delivery"] = delivery.clone();
11792 let hello = Frame::build(
11793 FrameType::Hello,
11794 control_flags(),
11795 0,
11796 0,
11797 99,
11798 serde_json::to_vec(&body).unwrap(),
11799 )
11800 .unwrap();
11801 let response = handler
11802 .handle_control(ConnectionId::new(99), hello)
11803 .unwrap();
11804 assert_eq!(response[0].header.ty, FrameType::HelloAck);
11805
11806 let (_, registrations) = registry.list_modules().unwrap();
11807 let registered = ®istrations[0].manifest;
11808 assert!(registered.is_watch_operation("status"));
11809 assert!(!registered.is_watch_operation("future"));
11810 assert!(!registered.is_watch_operation("reset"));
11811 assert!(!registered.is_watch_operation("missing"));
11812
11813 let request = Frame::build(
11814 FrameType::Request,
11815 control_flags(),
11816 0,
11817 0,
11818 100,
11819 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11820 )
11821 .unwrap();
11822 let response = handler.handle_catalog_list(request, None).unwrap();
11823 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
11824 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
11825 let ClientControlResponse::CatalogList { modules, .. } =
11826 serde_json::from_slice(&response[0].body).unwrap()
11827 else {
11828 panic!("expected catalog.list");
11829 };
11830 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
11831 }
11832
11833 #[test]
11834 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
11835 let registry = Arc::new(Registry::default());
11836 let supervisor = SupervisorHandle::new();
11837 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
11838 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11839 let descriptor = json!({
11840 "schema": "ckios.richui.manifest/1",
11841 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
11842 "bytes": 4096,
11843 "chunks": 4
11844 });
11845 let connection = ConnectionId::new(201);
11846 let hello = handler
11847 .handle_control(
11848 connection,
11849 hello_frame_with_rich_ui(
11850 "vault",
11851 PROTOCOL_VERSION,
11852 201,
11853 Some("the-real-nonce"),
11854 descriptor.clone(),
11855 ),
11856 )
11857 .unwrap();
11858 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11859
11860 let request =
11861 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
11862 let provides = registry
11863 .get_module_by_connection(connection)
11864 .unwrap()
11865 .unwrap()
11866 .manifest
11867 .provides;
11868 handler
11869 .handle_catalog_update(connection, request, provides, None, Some(false))
11870 .unwrap();
11871
11872 let request = Frame::build(
11873 FrameType::Request,
11874 control_flags(),
11875 0,
11876 0,
11877 203,
11878 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11879 )
11880 .unwrap();
11881 let response = handler.handle_catalog_list(request, None).unwrap();
11882 let ClientControlResponse::CatalogList { modules, .. } =
11883 serde_json::from_slice(&response[0].body).unwrap()
11884 else {
11885 panic!("expected catalog.list");
11886 };
11887 assert_eq!(
11888 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
11889 descriptor
11890 );
11891 assert!(modules[0].nonce_verified);
11892 }
11893
11894 fn hello_with_role_declarations(declarations: Value) -> Frame {
11895 let mut body = serde_json::to_value(ModuleHelloBody {
11896 manifest: manifest("role-test", PROTOCOL_VERSION),
11897 protocol_ver: PROTOCOL_VERSION,
11898 control_ops: None,
11899 launch_nonce: None,
11900 })
11901 .unwrap();
11902 body["manifest"]["role_declarations"] = declarations;
11903 Frame::build(
11904 FrameType::Hello,
11905 control_flags(),
11906 0,
11907 0,
11908 301,
11909 serde_json::to_vec(&body).unwrap(),
11910 )
11911 .unwrap()
11912 }
11913
11914 fn update_role_declarations_frame(declarations: Value) -> Frame {
11915 Frame::build(
11916 FrameType::Request,
11917 control_flags(),
11918 0,
11919 0,
11920 303,
11921 serde_json::to_vec(&json!({
11922 "op": "catalog.update_role_declarations",
11923 "role_declarations": declarations,
11924 }))
11925 .unwrap(),
11926 )
11927 .unwrap()
11928 }
11929
11930 #[tokio::test]
11931 async fn role_declarations_update_replaces_then_clears_catalog() {
11932 let registry = Arc::new(Registry::default());
11933 let handler = ControlHandler::new(Arc::clone(®istry));
11934 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
11935 let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
11936 hello_via_sink(
11937 &handler,
11938 &ctx,
11939 &mut rx,
11940 hello_with_role_declarations(initial.clone()),
11941 )
11942 .await;
11943 assert_eq!(
11944 role_declarations_catalog(&handler).0["role_declarations"],
11945 initial
11946 );
11947 let before = registry.get_module("role-test").unwrap().unwrap();
11948 for declarations in [
11949 json!({"media-provider/v1": {"engines": ["new"]}}),
11950 json!({}),
11951 ] {
11952 let response = handler
11953 .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
11954 .await
11955 .unwrap();
11956 assert_eq!(response[0].header.ty, FrameType::Response);
11957 assert_eq!(
11958 serde_json::from_slice::<Value>(&response[0].body).unwrap(),
11959 json!({"op": "catalog.update_role_declarations"})
11960 );
11961 let (raw, entry) = role_declarations_catalog(&handler);
11962 assert_eq!(
11963 serde_json::to_value(entry.role_declarations).unwrap(),
11964 declarations
11965 );
11966 if declarations.as_object().unwrap().is_empty() {
11967 assert!(raw.get("role_declarations").is_none());
11968 } else {
11969 assert_eq!(raw["role_declarations"], declarations);
11970 }
11971 let after = registry.get_module("role-test").unwrap().unwrap();
11972 let mut expected = before.manifest.clone();
11973 expected.role_declarations = serde_json::from_value(declarations).unwrap();
11974 assert_eq!(after.manifest, expected);
11975 assert_eq!(after.ready, before.ready);
11976 }
11977 }
11978
11979 #[tokio::test]
11980 async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
11981 let registry = Arc::new(Registry::default());
11982 let handler = ControlHandler::new(registry);
11983 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
11984 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
11985 hello_via_sink(
11986 &handler,
11987 &ctx,
11988 &mut rx,
11989 hello_with_role_declarations(initial.clone()),
11990 )
11991 .await;
11992 for (invalid, reason) in [
11993 (json!(null), "role_declarations must be an object"),
11994 (json!([]), "role_declarations must be an object"),
11995 (
11996 json!({"bad key": {}}),
11997 "role_declarations key must use capability-name grammar",
11998 ),
11999 (
12000 json!({"media-provider/v1": []}),
12001 "role_declarations values must be objects",
12002 ),
12003 (
12004 json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
12005 "role_declarations exceeds 16384 compact JSON bytes",
12006 ),
12007 ] {
12008 let response = handler
12009 .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
12010 .await
12011 .unwrap();
12012 let error = parse_error(&response[0]);
12013 assert_eq!(error["code"], "invalid_role_declarations");
12014 assert_eq!(error["message"], reason);
12015 assert_eq!(
12016 role_declarations_catalog(&handler).0["role_declarations"],
12017 initial
12018 );
12019 }
12020 }
12021
12022 #[tokio::test]
12023 async fn role_declarations_update_is_owned_by_the_registered_connection() {
12024 let registry = Arc::new(Registry::default());
12025 let handler = ControlHandler::new(Arc::clone(®istry));
12026 let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
12027 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12028 hello_via_sink(
12029 &handler,
12030 &owner,
12031 &mut owner_rx,
12032 hello_with_role_declarations(initial.clone()),
12033 )
12034 .await;
12035 let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
12036 let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
12037 let response = handler
12038 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12039 .await
12040 .unwrap();
12041 assert_eq!(parse_error(&response[0])["code"], "not_registered");
12042 hello_via_sink(
12043 &handler,
12044 &other,
12045 &mut other_rx,
12046 hello_frame("other", PROTOCOL_VERSION, 304),
12047 )
12048 .await;
12049 let response = handler
12050 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12051 .await
12052 .unwrap();
12053 assert_eq!(response[0].header.ty, FrameType::Response);
12054 assert_eq!(
12055 serde_json::to_value(
12056 registry
12057 .get_module("role-test")
12058 .unwrap()
12059 .unwrap()
12060 .manifest
12061 .role_declarations
12062 )
12063 .unwrap(),
12064 initial
12065 );
12066 assert_eq!(
12067 serde_json::to_value(
12068 registry
12069 .get_module("other")
12070 .unwrap()
12071 .unwrap()
12072 .manifest
12073 .role_declarations
12074 )
12075 .unwrap(),
12076 replacement
12077 );
12078 }
12079
12080 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
12081 let request = Frame::build(
12082 FrameType::Request,
12083 control_flags(),
12084 0,
12085 0,
12086 302,
12087 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12088 )
12089 .unwrap();
12090 let response = handler.handle_catalog_list(request, None).unwrap();
12091 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
12092 let ClientControlResponse::CatalogList { mut modules, .. } =
12093 serde_json::from_slice(&response[0].body).unwrap()
12094 else {
12095 panic!("expected catalog.list");
12096 };
12097 (raw["modules"][0].clone(), modules.remove(0))
12098 }
12099
12100 #[test]
12101 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
12102 let registry = Arc::new(Registry::default());
12103 let handler = ControlHandler::new(Arc::clone(®istry));
12104 let declarations = json!({
12105 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
12106 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
12107 });
12108 let connection = ConnectionId::new(301);
12109 let response = handler
12110 .handle_control(
12111 connection,
12112 hello_with_role_declarations(declarations.clone()),
12113 )
12114 .unwrap();
12115 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12116 let registered = registry
12117 .get_module_by_connection(connection)
12118 .unwrap()
12119 .unwrap()
12120 .manifest;
12121 assert_eq!(
12125 serde_json::to_value(®istered.role_declarations).unwrap(),
12126 declarations
12127 );
12128 let (raw, entry) = role_declarations_catalog(&handler);
12129 assert_eq!(raw["role_declarations"], declarations);
12130 assert_eq!(entry.role_declarations, registered.role_declarations);
12131
12132 let request =
12133 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
12134 handler
12135 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
12136 .unwrap();
12137 let (raw, entry) = role_declarations_catalog(&handler);
12138 assert_eq!(raw["role_declarations"], declarations);
12139 assert_eq!(entry.role_declarations, registered.role_declarations);
12140 }
12141
12142 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
12143 let registry = Arc::new(Registry::default());
12144 let handler = ControlHandler::new(Arc::clone(®istry));
12145 let capture = EventCapture::default();
12146 let _subscriber =
12147 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12148 let response = handler
12149 .handle_control(
12150 ConnectionId::new(301),
12151 hello_with_role_declarations(declarations),
12152 )
12153 .unwrap();
12154 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12155 assert!(registry
12156 .get_module_by_connection(ConnectionId::new(301))
12157 .unwrap()
12158 .unwrap()
12159 .manifest
12160 .role_declarations
12161 .is_empty());
12162 let (raw, entry) = role_declarations_catalog(&handler);
12163 assert!(raw.get("role_declarations").is_none());
12164 assert!(entry.role_declarations.is_empty());
12165 let warnings: Vec<_> = capture
12166 .events()
12167 .into_iter()
12168 .filter(|event| {
12169 event.fields.get("message").map(String::as_str)
12170 == Some("dropping whole role_declarations map from HELLO")
12171 })
12172 .collect();
12173 assert_eq!(warnings.len(), 1);
12174 assert_eq!(warnings[0].level, tracing::Level::WARN);
12175 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
12176 assert_eq!(warnings[0].fields["reason"], reason);
12177 }
12178
12179 #[test]
12180 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
12181 for value in [
12182 json!(null),
12183 json!(false),
12184 json!(0),
12185 json!("opaque"),
12186 json!([]),
12187 ] {
12188 assert_hello_drops_role_declarations_with_warning(
12189 json!({"routing/v1": {}, "broken/v1": value}),
12190 "role_declarations values must be objects",
12191 );
12192 }
12193 assert_hello_drops_role_declarations_with_warning(
12194 json!([]),
12195 "role_declarations must be an object",
12196 );
12197 }
12198
12199 #[test]
12200 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
12201 assert_hello_drops_role_declarations_with_warning(
12202 json!({"routing/v1": {}, "bad key": {}}),
12203 "role_declarations key must use capability-name grammar",
12204 );
12205 }
12206
12207 #[test]
12208 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
12209 assert_hello_drops_role_declarations_with_warning(
12210 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
12211 "role_declarations exceeds 16384 compact JSON bytes",
12212 );
12213 }
12214
12215 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
12216 let mut body = serde_json::to_value(ModuleHelloBody {
12217 manifest: manifest("flow-ops", PROTOCOL_VERSION),
12218 protocol_ver: PROTOCOL_VERSION,
12219 control_ops: None,
12220 launch_nonce: None,
12221 })
12222 .unwrap();
12223 body["manifest"]["provides"] = json!([{
12224 "role": "management_surface",
12225 "operations": [
12226 {"name": "diagnostics.findings", "kind": "query"},
12227 {"name": "reset", "kind": "mutate"}
12228 ],
12229 "config_schema": {}, "observability": [], "identity_scope": []
12230 }]);
12231 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
12232 if let Some(flow_operations) = flow_operations {
12233 body["manifest"]["flow_operations"] = flow_operations;
12234 }
12235 Frame::build(
12236 FrameType::Hello,
12237 control_flags(),
12238 0,
12239 0,
12240 401,
12241 serde_json::to_vec(&body).unwrap(),
12242 )
12243 .unwrap()
12244 }
12245
12246 #[test]
12247 fn catalog_list_mirrors_flow_operations_unchanged() {
12248 for (capability, flow_operations) in [
12251 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
12252 ("call-modes/v1", json!(["diagnostics.findings"])),
12253 ("flow-scopes/v1", json!([])),
12254 ] {
12255 let registry = Arc::new(Registry::default());
12256 let handler = ControlHandler::new(Arc::clone(®istry));
12257 let connection = ConnectionId::new(401);
12258 let response = handler
12259 .handle_control(
12260 connection,
12261 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
12262 )
12263 .unwrap();
12264 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12265 let registered = registry
12266 .get_module_by_connection(connection)
12267 .unwrap()
12268 .unwrap()
12269 .manifest;
12270 assert_eq!(
12271 serde_json::to_value(®istered.flow_operations).unwrap(),
12272 flow_operations
12273 );
12274 let (raw, entry) = role_declarations_catalog(&handler);
12275 assert_eq!(raw["flow_operations"], flow_operations);
12276 assert_eq!(entry.flow_operations, registered.flow_operations);
12277 }
12278
12279 let registry = Arc::new(Registry::default());
12280 let handler = ControlHandler::new(Arc::clone(®istry));
12281 let response = handler
12282 .handle_control(
12283 ConnectionId::new(401),
12284 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
12285 )
12286 .unwrap();
12287 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12288 let (raw, entry) = role_declarations_catalog(&handler);
12289 assert!(raw.get("flow_operations").is_none());
12290 assert_eq!(entry.flow_operations, None);
12291 }
12292
12293 #[test]
12294 fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
12295 let oversized: serde_json::Map<String, Value> = (0..257)
12296 .map(|n| (format!("tool:t{n}"), json!(true)))
12297 .collect();
12298 for (capabilities, map, valid) in [
12299 (
12300 json!(["scope-federated/v1"]),
12301 json!({"tool:read":true,"management:reset":true}),
12302 true,
12303 ),
12304 (json!(["scope-federated/v1"]), json!({}), true),
12305 (json!([]), json!({"tool:read":true}), false),
12306 (
12307 json!(["scope-federated/v1"]),
12308 json!({"tool:valid":true,"tool:bad":false}),
12309 false,
12310 ),
12311 (
12312 json!(["scope-federated/v1"]),
12313 json!({"tool:valid":true,"management:missing":true}),
12314 false,
12315 ),
12316 (
12317 json!(["scope-federated/v1"]),
12318 Value::Object(oversized),
12319 false,
12320 ),
12321 ] {
12322 let capture = EventCapture::default();
12323 let _subscriber = tracing::subscriber::set_default(
12324 tracing_subscriber::registry().with(capture.clone()),
12325 );
12326 let registry = Arc::new(Registry::default());
12327 let handler = ControlHandler::new(Arc::clone(®istry));
12328 let hello = hello_with_flow_operations(capabilities, None);
12329 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12330 body["manifest"]["operation_deferral"] = map.clone();
12331 let hello = Frame::build(
12332 FrameType::Hello,
12333 control_flags(),
12334 0,
12335 0,
12336 401,
12337 serde_json::to_vec(&body).unwrap(),
12338 )
12339 .unwrap();
12340 let response = handler
12341 .handle_control(ConnectionId::new(401), hello)
12342 .unwrap();
12343 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12344 let (raw, entry) = role_declarations_catalog(&handler);
12345 if valid {
12346 assert_eq!(raw["operation_deferral"], map);
12347 assert_eq!(
12348 serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
12349 map
12350 );
12351 } else {
12352 assert!(entry.operation_deferral.is_none());
12353 assert!(raw.get("operation_deferral").is_none());
12354 assert_eq!(
12355 capture
12356 .events()
12357 .iter()
12358 .filter(|event| event.fields.get("message").map(String::as_str)
12359 == Some("dropping operation_deferral from HELLO"))
12360 .count(),
12361 1
12362 );
12363 }
12364 }
12365 }
12366
12367 fn assert_hello_drops_flow_operations_with_warning(
12368 capabilities: Value,
12369 flow_operations: Value,
12370 reason: &str,
12371 ) {
12372 let registry = Arc::new(Registry::default());
12373 let handler = ControlHandler::new(Arc::clone(®istry));
12374 let capture = EventCapture::default();
12375 let _subscriber =
12376 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12377 let response = handler
12378 .handle_control(
12379 ConnectionId::new(401),
12380 hello_with_flow_operations(capabilities, Some(flow_operations)),
12381 )
12382 .unwrap();
12383 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12384 let registered = registry
12385 .get_module_by_connection(ConnectionId::new(401))
12386 .unwrap()
12387 .expect("the module still registers")
12388 .manifest;
12389 assert_eq!(registered.module_id, "flow-ops");
12390 assert_eq!(registered.flow_operations, None);
12391 let (raw, entry) = role_declarations_catalog(&handler);
12392 assert_eq!(raw["module_id"], "flow-ops");
12393 assert!(raw.get("flow_operations").is_none());
12394 assert_eq!(entry.flow_operations, None);
12395 let warnings: Vec<_> = capture
12396 .events()
12397 .into_iter()
12398 .filter(|event| {
12399 event.fields.get("message").map(String::as_str)
12400 == Some("dropping flow_operations from HELLO")
12401 })
12402 .collect();
12403 assert_eq!(warnings.len(), 1);
12404 assert_eq!(warnings[0].level, tracing::Level::WARN);
12405 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
12406 assert_eq!(warnings[0].fields["reason"], reason);
12407 }
12408
12409 #[test]
12410 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
12411 assert_hello_drops_flow_operations_with_warning(
12412 json!(["flow-scopes/v1"]),
12413 json!(["diagnostics.findings", "missing"]),
12414 "flow_operations names 'missing', which is not a declared management operation",
12415 );
12416 }
12417
12418 #[test]
12419 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
12420 assert_hello_drops_flow_operations_with_warning(
12421 json!([]),
12422 json!(["diagnostics.findings"]),
12423 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12424 );
12425 assert_hello_drops_flow_operations_with_warning(
12427 json!(["agent-run-scopes/v1"]),
12428 json!([]),
12429 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12430 );
12431 }
12432
12433 #[test]
12434 fn hello_malformed_flow_operations_is_dropped_with_warning() {
12435 assert_hello_drops_flow_operations_with_warning(
12436 json!(["flow-scopes/v1"]),
12437 json!(["reset", "reset"]),
12438 "flow_operations names 'reset' more than once",
12439 );
12440 assert_hello_drops_flow_operations_with_warning(
12441 json!(["flow-scopes/v1"]),
12442 json!(vec!["reset"; 257]),
12443 "flow_operations lists 257 names, more than 256",
12444 );
12445 assert_hello_drops_flow_operations_with_warning(
12446 json!(["flow-scopes/v1"]),
12447 json!("reset"),
12448 "flow_operations must be an array of operation names",
12449 );
12450 }
12451
12452 #[test]
12453 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
12454 let registry = Arc::new(Registry::default());
12455 let handler = ControlHandler::new(registry.clone());
12456 let descriptor = json!({
12457 "schema": "ckios.richui.manifest/1",
12458 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12459 "bytes": 128,
12460 "chunks": 2
12461 });
12462 let malformed = [
12463 ("schema", json!("")),
12464 ("schema", json!("a".repeat(65))),
12465 ("schema", json!("Bad schema")),
12466 (
12467 "manifest_digest",
12468 json!(format!("sha256:{}", "A".repeat(64))),
12469 ),
12470 ("manifest_digest", json!("sha256:abc")),
12471 ("bytes", json!(0)),
12472 ("bytes", json!(16_777_217)),
12473 ("bytes", json!("128")),
12474 ("chunks", json!(0)),
12475 ("chunks", json!(1025)),
12476 ("chunks", json!("2")),
12477 ];
12478 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
12479 let mut invalid = descriptor.clone();
12480 invalid[field] = invalid_value;
12481 let response = handler
12482 .handle_control(
12483 ConnectionId::new(210 + index as u64),
12484 hello_frame_with_rich_ui(
12485 &format!("bad-rich-ui-{index}"),
12486 PROTOCOL_VERSION,
12487 210 + index as u64,
12488 None,
12489 invalid,
12490 ),
12491 )
12492 .unwrap();
12493 assert_eq!(response[0].header.ty, FrameType::Error);
12494 let error = parse_error(&response[0]);
12495 assert_eq!(error["code"], "invalid_manifest");
12496 assert!(error["message"]
12497 .as_str()
12498 .unwrap()
12499 .contains(&format!("rich_ui.{field}")));
12500 assert!(registry
12501 .get_module(&format!("bad-rich-ui-{index}"))
12502 .unwrap()
12503 .is_none());
12504 }
12505 }
12506
12507 #[test]
12508 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
12509 let registry = Arc::new(Registry::default());
12510 let handler = ControlHandler::new(Arc::clone(®istry));
12511 let connection = ConnectionId::new(219);
12512 registry
12513 .register_with_control_ops(
12514 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
12515 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
12516 "bad schema",
12517 format!("sha256:{}", "a".repeat(64)),
12518 128,
12519 2,
12520 )))
12521 .build(),
12522 PROTOCOL_VERSION,
12523 connection,
12524 Vec::new(),
12525 )
12526 .unwrap();
12527 let request =
12528 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
12529 let response = handler
12530 .handle_catalog_update(connection, request, Vec::new(), None, None)
12531 .unwrap();
12532 assert_eq!(response[0].header.ty, FrameType::Error);
12533 let error = parse_error(&response[0]);
12534 assert_eq!(error["code"], "invalid_manifest");
12535 assert!(error["message"]
12536 .as_str()
12537 .unwrap()
12538 .contains("rich_ui.schema"));
12539 }
12540
12541 #[test]
12542 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
12543 let registry = Arc::new(Registry::default());
12544 let supervisor = SupervisorHandle::new();
12545 supervisor.open_swap("provider", "candidate-nonce".to_string());
12546 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12547 let connection = ConnectionId::new(218);
12548 let hello = handler
12549 .handle_control(
12550 connection,
12551 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
12552 )
12553 .unwrap();
12554 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12555 assert!(registry.list_modules().unwrap().1.is_empty());
12556
12557 registry.promote_candidate("provider").unwrap().unwrap();
12558 let request = Frame::build(
12559 FrameType::Request,
12560 control_flags(),
12561 0,
12562 0,
12563 219,
12564 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12565 )
12566 .unwrap();
12567 let response = handler.handle_catalog_list(request, None).unwrap();
12568 let ClientControlResponse::CatalogList { modules, .. } =
12569 serde_json::from_slice(&response[0].body).unwrap()
12570 else {
12571 panic!("expected catalog.list");
12572 };
12573 assert_eq!(modules.len(), 1);
12574 assert!(modules[0].nonce_verified);
12575 }
12576
12577 #[test]
12578 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
12579 for provenance_json in [
12580 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
12581 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
12582 ] {
12583 let registry = Arc::new(Registry::default());
12584 let handler = ControlHandler::new(registry);
12585 let hello = Frame::build(
12586 FrameType::Hello,
12587 control_flags(),
12588 0,
12589 0,
12590 220,
12591 serde_json::to_vec(&ModuleHelloBody {
12592 manifest: ModuleManifest::builder("built", "0.1.0")
12593 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
12594 .build(),
12595 protocol_ver: PROTOCOL_VERSION,
12596 control_ops: None,
12597 launch_nonce: None,
12598 })
12599 .unwrap(),
12600 )
12601 .unwrap();
12602 assert_eq!(
12603 handler
12604 .handle_control(ConnectionId::new(220), hello)
12605 .unwrap()[0]
12606 .header
12607 .ty,
12608 FrameType::HelloAck
12609 );
12610 let request = Frame::build(
12611 FrameType::Request,
12612 control_flags(),
12613 0,
12614 0,
12615 221,
12616 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12617 )
12618 .unwrap();
12619 let response = handler.handle_catalog_list(request, None).unwrap();
12620 let ClientControlResponse::CatalogList { modules, .. } =
12621 serde_json::from_slice(&response[0].body).unwrap()
12622 else {
12623 panic!("expected catalog.list");
12624 };
12625 assert_eq!(modules.len(), 1);
12626 assert_eq!(
12627 serde_json::to_vec(&modules[0]).unwrap(),
12628 format!(
12629 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
12630 )
12631 .as_bytes()
12632 );
12633 }
12634 }
12635
12636 #[test]
12637 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
12638 let registry = Arc::new(Registry::default());
12639 let handler = ControlHandler::new(Arc::clone(®istry));
12640 let hello = Frame::build(
12641 FrameType::Hello,
12642 control_flags(),
12643 0,
12644 0,
12645 220,
12646 serde_json::to_vec(&ModuleHelloBody {
12647 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
12648 protocol_ver: PROTOCOL_VERSION,
12649 control_ops: None,
12650 launch_nonce: Some("unverified".to_string()),
12651 })
12652 .unwrap(),
12653 )
12654 .unwrap();
12655 assert_eq!(
12656 handler
12657 .handle_control(ConnectionId::new(220), hello)
12658 .unwrap()[0]
12659 .header
12660 .ty,
12661 FrameType::HelloAck
12662 );
12663 let request = Frame::build(
12664 FrameType::Request,
12665 control_flags(),
12666 0,
12667 0,
12668 221,
12669 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12670 )
12671 .unwrap();
12672 let response = handler.handle_catalog_list(request, None).unwrap();
12673 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
12674 assert!(wire["modules"][0].get("provenance").is_none());
12675 let ClientControlResponse::CatalogList { modules, .. } =
12676 serde_json::from_slice(&response[0].body).unwrap()
12677 else {
12678 panic!("expected catalog.list");
12679 };
12680 assert_eq!(
12681 serde_json::to_vec(&modules[0]).unwrap(),
12682 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
12683 );
12684 assert!(!modules[0].nonce_verified);
12685 assert!(modules[0].provenance.is_none());
12686 }
12687
12688 #[test]
12689 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
12690 let registry = Arc::new(Registry::default());
12691 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
12692 [("vault".to_string(), true), ("squatter".to_string(), true)],
12693 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
12694 );
12695 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
12696 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
12697 provides: vec!["credentials-provider/v1".to_string()],
12698 requires: Vec::new(),
12699 must_never_reach: Vec::new(),
12700 });
12701 let frame = Frame::build(
12702 FrameType::Hello,
12703 control_flags(),
12704 0,
12705 0,
12706 77,
12707 serde_json::to_vec(&ModuleHelloBody {
12708 manifest: squatter,
12709 protocol_ver: PROTOCOL_VERSION,
12710 control_ops: None,
12711 launch_nonce: None,
12712 })
12713 .expect("HELLO serializes"),
12714 )
12715 .expect("HELLO frame builds");
12716 let response = handler
12717 .handle_control(ConnectionId::new(77), frame)
12718 .expect("reserved claim receives a typed refusal");
12719 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
12720 assert_eq!(
12721 registry
12722 .active_registration_count()
12723 .expect("registry reads"),
12724 0,
12725 "a reserved capability refusal must not leave a catalog entry"
12726 );
12727 }
12728
12729 #[test]
12730 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
12731 for settlement in ["timeout", "inconclusive", "drop"] {
12732 let breakers = RouteBindBreakers::default();
12733 let RouteBindAdmission::Admitted {
12734 guard: mut old,
12735 probe: false,
12736 } = breakers.admit("prov")
12737 else {
12738 panic!("ordinary relay admitted")
12739 };
12740 let RouteBindAdmission::Admitted {
12741 guard: mut opener, ..
12742 } = breakers.admit("prov")
12743 else {
12744 panic!("second relay admitted")
12745 };
12746 assert!(
12747 !opener
12748 .record_timeout(1, Duration::ZERO)
12749 .unwrap()
12750 .reopened_after_probe
12751 );
12752 let RouteBindAdmission::Admitted {
12753 guard: mut probe,
12754 probe: true,
12755 } = breakers.admit("prov")
12756 else {
12757 panic!("one cooldown probe admitted")
12758 };
12759 match settlement {
12760 "timeout" => assert!(
12761 !old.record_timeout(1, Duration::ZERO)
12762 .unwrap()
12763 .reopened_after_probe
12764 ),
12765 "inconclusive" => old.record_inconclusive(),
12766 "drop" => drop(old),
12767 _ => unreachable!(),
12768 }
12769 assert!(
12770 matches!(
12771 breakers.admit("prov"),
12772 RouteBindAdmission::Refused {
12773 probe_in_flight: true,
12774 ..
12775 }
12776 ),
12777 "{settlement} of a pre-open relay cannot release the real probe"
12778 );
12779 assert!(
12780 probe
12781 .record_timeout(1, Duration::ZERO)
12782 .unwrap()
12783 .reopened_after_probe
12784 );
12785 assert!(matches!(
12786 breakers.admit("prov"),
12787 RouteBindAdmission::Admitted { probe: true, .. }
12788 ));
12789 }
12790 let breakers = RouteBindBreakers::default();
12791 let admit = || match breakers.admit("prov") {
12792 RouteBindAdmission::Admitted { guard, .. } => guard,
12793 _ => panic!("relay admitted"),
12794 };
12795 admit().record_timeout(1, Duration::ZERO);
12796 let mut old_probe = admit();
12797 breakers.reset_for_new_module_connection("prov");
12798 admit().record_timeout(1, Duration::ZERO);
12799 let _new_probe = admit();
12800 old_probe.record_inconclusive();
12801 assert!(matches!(
12802 breakers.admit("prov"),
12803 RouteBindAdmission::Refused {
12804 probe_in_flight: true,
12805 ..
12806 }
12807 ));
12808 }
12809
12810 #[tokio::test]
12811 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
12812 for candidate in [false, true] {
12813 let registry = Arc::new(Registry::default());
12814 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
12815 [("vault".to_string(), true), ("squatter".to_string(), true)],
12816 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
12817 );
12818 let conn = ConnectionId::new(77);
12819 let (ctx, mut rx) = route_ctx(conn);
12820 let initial = capability_manifest("squatter", &[], &[]);
12821 if candidate {
12822 registry
12823 .register_candidate_with_control_ops(
12824 initial.clone(),
12825 PROTOCOL_VERSION,
12826 conn,
12827 module_baseline_control_ops(),
12828 )
12829 .unwrap();
12830 handler
12831 .forwarding
12832 .register_candidate_module_connection(
12833 conn,
12834 "squatter".to_string(),
12835 PROTOCOL_VERSION,
12836 manifest_concurrency(&initial),
12837 ctx.egress.clone(),
12838 )
12839 .unwrap();
12840 } else {
12841 hello_via_sink(
12842 &handler,
12843 &ctx,
12844 &mut rx,
12845 hello_frame_with_manifest(initial.clone(), 1),
12846 )
12847 .await;
12848 }
12849 let response = handler
12850 .handle_control_frame(
12851 &ctx,
12852 catalog_update_with_capabilities_frame(
12853 2,
12854 capability_manifest("squatter", &["credentials-provider/v1"], &[])
12855 .capabilities
12856 .unwrap(),
12857 ),
12858 )
12859 .await
12860 .unwrap();
12861 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
12862 assert_eq!(
12863 registry
12864 .get_module_by_connection(conn)
12865 .unwrap()
12866 .unwrap()
12867 .manifest,
12868 initial
12869 );
12870 }
12871 }
12872
12873 #[test]
12874 #[cfg(not(feature = "dev-desktop-approval"))]
12875 fn server_describe_default_build_advertises_no_config_capability() {
12876 let assert_default = |handler: &ControlHandler| {
12877 let request = Frame::build(
12878 FrameType::Request,
12879 control_flags(),
12880 0,
12881 0,
12882 79,
12883 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12884 )
12885 .unwrap();
12886 let responses = handler.handle_server_describe(request).unwrap();
12887 assert_eq!(responses.len(), 1);
12888 let response: Value = serde_json::from_slice(&responses[0].body).unwrap();
12889 assert_eq!(response["op"], "server.describe");
12890 let capabilities = response["capabilities"]
12891 .as_array()
12892 .expect("capability array");
12893 assert!(
12894 !capabilities.is_empty(),
12895 "exercise an actual describe response, not an empty fixture"
12896 );
12897 assert!(
12898 capabilities
12899 .iter()
12900 .all(|capability| !capability.as_str().unwrap().starts_with("config.")),
12901 "a default build must not announce configuration authority: {capabilities:?}"
12902 );
12903 };
12904 assert_default(&ControlHandler::new(Arc::new(Registry::default())));
12905 #[cfg(unix)]
12906 {
12907 let (_rig, _owner, _supervisor, handler, _spec) =
12911 crate::config_gate::runtime::RuntimeOwner::unactivated_supervisor_fixture_for_test(
12912 );
12913 assert_default(&handler);
12914 }
12915 }
12916
12917 #[test]
12918 fn server_describe_surfaces_required_capability_verdict_fields() {
12919 let registry = Arc::new(Registry::default());
12920 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
12921 [
12922 ("consumer".to_string(), true),
12923 ("provider".to_string(), false),
12924 ],
12925 BTreeMap::new(),
12926 );
12927 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
12928 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
12929 provides: Vec::new(),
12930 requires: vec![subc_protocol::manifest::CapabilityRequirement {
12931 capability: "credentials-provider/v1".to_string(),
12932 need: subc_protocol::manifest::CapabilityNeed::Required,
12933 }],
12934 must_never_reach: Vec::new(),
12935 });
12936 let hello = Frame::build(
12937 FrameType::Hello,
12938 control_flags(),
12939 0,
12940 0,
12941 78,
12942 serde_json::to_vec(&ModuleHelloBody {
12943 manifest: consumer,
12944 protocol_ver: PROTOCOL_VERSION,
12945 control_ops: None,
12946 launch_nonce: None,
12947 })
12948 .expect("HELLO serializes"),
12949 )
12950 .expect("HELLO frame builds");
12951 handler
12952 .handle_control(ConnectionId::new(78), hello)
12953 .expect("consumer registers");
12954 let describe = Frame::build(
12955 FrameType::Request,
12956 control_flags(),
12957 0,
12958 0,
12959 79,
12960 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
12961 .expect("request serializes"),
12962 )
12963 .expect("describe frame builds");
12964 let response = handler
12965 .handle_server_describe(describe)
12966 .expect("server.describe succeeds");
12967 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
12968 let requirement = &rendered["capability_requirements"][0];
12969 assert_eq!(requirement["consumer"], "consumer");
12970 assert_eq!(requirement["verdict"], "never_provided");
12971 assert_eq!(requirement["episode_seq"], 1);
12972 assert_eq!(requirement["config_satisfiable"], false);
12973 assert_eq!(requirement["runtime_available"], false);
12974 assert!(requirement["detail"]
12975 .as_str()
12976 .expect("detail string")
12977 .contains("credentials-provider/v1"));
12978 }
12979
12980 fn deregister_required_provider(
12986 daemon_shutdown_begun: bool,
12987 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
12988 const CAPABILITY: &str = "project-identity/v1";
12989 let registry = Arc::new(Registry::default());
12990 let daemon_shutdown = DaemonShutdownFlag::default();
12991 let handler = ControlHandler::new(Arc::clone(®istry))
12992 .with_capability_config(
12993 [
12994 ("consumer".to_string(), true),
12995 ("provider".to_string(), true),
12996 ],
12997 BTreeMap::new(),
12998 )
12999 .with_daemon_shutdown_flag(daemon_shutdown.clone());
13000 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13001 consumer.capabilities = Some(CapabilityDeclarations {
13002 provides: Vec::new(),
13003 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13004 capability: CAPABILITY.to_string(),
13005 need: subc_protocol::manifest::CapabilityNeed::Required,
13006 }],
13007 must_never_reach: Vec::new(),
13008 });
13009 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
13010 let provider_conn = ConnectionId::new(182);
13011 for (conn, manifest, corr) in [
13012 (ConnectionId::new(181), consumer, 181),
13013 (provider_conn, provider, 182),
13014 ] {
13015 let hello = handler
13016 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
13017 .expect("module registers");
13018 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13019 }
13020 assert_eq!(
13021 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13022 Some(CapabilityVerdict::Provided),
13023 "the registered provider must satisfy the requirement first"
13024 );
13025 if daemon_shutdown_begun {
13026 daemon_shutdown.set_for_test();
13027 }
13028
13029 let capture = EventCapture::default();
13030 let _subscriber =
13031 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13032 handler
13033 .cleanup_connection(provider_conn)
13034 .expect("provider deregisters");
13035 let events = capture
13036 .events()
13037 .into_iter()
13038 .filter(|event| {
13039 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
13040 })
13041 .collect();
13042 (
13043 events,
13044 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13045 )
13046 }
13047
13048 #[test]
13049 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
13050 let (events, verdict) = deregister_required_provider(true);
13051 assert_eq!(
13052 verdict,
13053 Some(CapabilityVerdict::NeverProvided),
13054 "shutdown must not change the verdict route.open refusals read"
13055 );
13056 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13057 let event = &events[0];
13058 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
13059 assert_eq!(
13060 event.fields.get("cause").map(String::as_str),
13061 Some("\"daemon_shutdown\""),
13062 "{event:?}"
13063 );
13064 assert_eq!(
13065 event.fields.get("verdict").map(String::as_str),
13066 Some("\"never_provided\""),
13067 "{event:?}"
13068 );
13069 }
13070
13071 #[test]
13072 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
13073 let (events, verdict) = deregister_required_provider(false);
13074 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
13075 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13076 let event = &events[0];
13077 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
13078 assert_eq!(event.fields.get("cause"), None, "{event:?}");
13079 assert_eq!(
13080 event.fields.get("verdict").map(String::as_str),
13081 Some("\"never_provided\""),
13082 "{event:?}"
13083 );
13084 }
13085
13086 #[test]
13087 fn catalog_list_omits_capabilities_for_legacy_manifest() {
13088 let registry = Arc::new(Registry::default());
13089 let handler = ControlHandler::new(Arc::clone(®istry));
13090 let hello = handler
13091 .handle_control(
13092 ConnectionId::new(101),
13093 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
13094 )
13095 .expect("legacy HELLO registers");
13096 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13097
13098 let request = Frame::build(
13099 FrameType::Request,
13100 control_flags(),
13101 0,
13102 0,
13103 102,
13104 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
13105 .expect("catalog request serializes"),
13106 )
13107 .expect("catalog request frame builds");
13108 let response = handler
13109 .handle_catalog_list(request, None)
13110 .expect("catalog list succeeds");
13111 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
13112 assert!(
13113 body["modules"][0].get("capabilities").is_none(),
13114 "legacy manifest must retain an absent capabilities field on catalog.list"
13115 );
13116 }
13117
13118 #[test]
13119 fn hello_ack_omits_storage_when_no_storage_config() {
13120 let registry = Arc::new(Registry::default());
13121 let handler = ControlHandler::new(Arc::clone(®istry));
13122 let responses = handler
13123 .handle_control(
13124 ConnectionId::new(1),
13125 hello_frame("aft", PROTOCOL_VERSION, 7),
13126 )
13127 .unwrap();
13128 let ack = parse_ack(&responses[0]);
13129 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
13130 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
13131 }
13132
13133 #[tokio::test]
13134 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
13135 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
13136 let registry = Arc::new(Registry::default());
13137 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
13138 let responses = handler
13139 .handle_control(
13140 ConnectionId::new(1),
13141 hello_frame("aft", PROTOCOL_VERSION, 7),
13142 )
13143 .unwrap();
13144 let ack = parse_ack(&responses[0]);
13145 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
13146
13147 let described = handler
13148 .handle_control_frame(
13149 &route_ctx(ConnectionId::new(2)).0,
13150 Frame::build(
13151 FrameType::Request,
13152 control_flags(),
13153 0,
13154 0,
13155 9,
13156 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13157 )
13158 .unwrap(),
13159 )
13160 .await
13161 .unwrap();
13162 let ClientControlResponse::ServerDescribe { machine_id, .. } =
13163 serde_json::from_slice(&described[0].body).unwrap()
13164 else {
13165 panic!("server.describe answered with another shape");
13166 };
13167 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
13168 }
13169
13170 #[test]
13171 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
13172 let registry = Arc::new(Registry::default());
13175 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
13176 crate::daemon_config::StorageConfig::Sqlite {
13177 data_home: std::path::PathBuf::from("/data"),
13178 },
13179 ));
13180
13181 let responses = handler
13182 .handle_control(
13183 ConnectionId::new(1),
13184 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
13185 )
13186 .unwrap();
13187 let ack = parse_ack(&responses[0]);
13188 assert_eq!(
13189 ack.storage,
13190 Some(serde_json::json!({
13191 "module_id": "alfonso-routing",
13192 "storage_namespace": "default",
13193 "isolation": { "kind": "module" },
13194 "backend": {
13195 "backend": "sqlite",
13196 "path": "/data/cortexkit/alfonso-routing/store.db"
13197 }
13198 })),
13199 "the delivered descriptor is the module's own sqlite store path"
13200 );
13201 }
13202
13203 #[test]
13204 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
13205 let registry = Arc::new(Registry::default());
13206 let handler = ControlHandler::new(Arc::clone(®istry));
13207 let conn = ConnectionId::new(1);
13208 let responses = handler
13209 .handle_control(
13210 conn,
13211 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13212 )
13213 .unwrap();
13214 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
13215 let registration = registry.get_module("aft").unwrap().unwrap();
13216 assert_eq!(registration.control_ops, module_baseline_control_ops());
13217
13218 let frame =
13219 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
13220 assert!(handler
13221 .guard_module_control_op(&frame, "aft", "route.bind")
13222 .unwrap()
13223 .is_none());
13224 let error = handler
13225 .guard_module_control_op(&frame, "aft", "test.synthetic")
13226 .unwrap()
13227 .expect("synthetic ungranted op should be rejected");
13228 assert_eq!(error.header.ty, FrameType::Error);
13229 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
13230 }
13231
13232 #[test]
13233 fn hello_control_ops_some_adds_optional_grants() {
13234 let registry = Arc::new(Registry::default());
13235 let handler = ControlHandler::new(Arc::clone(®istry));
13236 handler
13237 .handle_control(
13238 ConnectionId::new(1),
13239 hello_frame_with_control_ops(
13240 "aft",
13241 PROTOCOL_VERSION,
13242 7,
13243 Some(vec![
13244 "future.synthetic".to_string(),
13245 "route.bind".to_string(),
13246 ]),
13247 ),
13248 )
13249 .unwrap();
13250 let registration = registry.get_module("aft").unwrap().unwrap();
13251 assert_eq!(
13252 registration.control_ops,
13253 vec![
13254 "route.bind".to_string(),
13255 "route.status".to_string(),
13256 "future.synthetic".to_string(),
13257 ]
13258 );
13259 let frame =
13260 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
13261 assert!(handler
13262 .guard_module_control_op(&frame, "aft", "future.synthetic")
13263 .unwrap()
13264 .is_none());
13265 }
13266
13267 #[tokio::test]
13268 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
13269 let registry = Arc::new(Registry::default());
13270 let forwarding = Arc::new(ForwardingTable::default());
13271 let handler =
13272 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13273 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
13274 hello_via_sink(
13275 &handler,
13276 &module_ctx,
13277 &mut module_rx,
13278 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13279 )
13280 .await;
13281
13282 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
13283 let responses = handler
13284 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
13285 .await
13286 .unwrap();
13287 assert_eq!(responses.len(), 1);
13288 assert_eq!(responses[0].header.ty, FrameType::Error);
13289 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
13290 assert!(module_rx.try_recv().is_err());
13291 }
13292
13293 #[tokio::test]
13294 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
13295 let registry = Arc::new(Registry::default());
13296 let forwarding = Arc::new(ForwardingTable::default());
13297 let handler =
13298 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13299 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
13300 hello_via_sink(
13301 &handler,
13302 &module_ctx,
13303 &mut module_rx,
13304 hello_frame_with_control_ops(
13305 "aft",
13306 PROTOCOL_VERSION,
13307 7,
13308 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13309 ),
13310 )
13311 .await;
13312
13313 let project_root = unique_project_root("demux");
13314 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
13315 let route_handler = handler.clone();
13316 let route_task = tokio::spawn(async move {
13317 route_handler
13318 .handle_control_frame(
13319 &route_client_ctx,
13320 route_open_frame(100, "aft", project_root),
13321 )
13322 .await
13323 .unwrap()
13324 });
13325 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13326 .await
13327 .unwrap()
13328 .unwrap();
13329 assert!(matches!(
13330 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
13331 ModuleControlRequest::RouteBind { .. }
13332 ));
13333
13334 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
13335 let health_handler = handler.clone();
13336 let health_task = tokio::spawn(async move {
13337 health_handler
13338 .handle_control_frame(
13339 &health_client_ctx,
13340 supervisor_health_probe_frame(101, "aft"),
13341 )
13342 .await
13343 .unwrap()
13344 });
13345 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13346 .await
13347 .unwrap()
13348 .unwrap();
13349 assert_eq!(
13350 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13351 ModuleControlRequest::HealthCheck {}
13352 );
13353
13354 handler
13355 .handle_control_frame(
13356 &module_ctx,
13357 health_response(health_frame.header.corr, HealthStatus::Degraded),
13358 )
13359 .await
13360 .unwrap();
13361 let health_response = health_task.await.unwrap();
13362 assert_eq!(health_response.len(), 1);
13363 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13364 ClientControlResponse::SupervisorHealthProbe {
13365 module_id,
13366 status,
13367 detail,
13368 metrics,
13369 } => {
13370 assert_eq!(module_id, "aft");
13371 assert_eq!(status, HealthStatus::Degraded);
13372 assert_eq!(detail.as_deref(), Some("warming"));
13373 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
13374 }
13375 other => panic!("unexpected health response: {other:?}"),
13376 }
13377
13378 handler
13379 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13380 .await
13381 .unwrap();
13382 let route_response = route_task.await.unwrap();
13383 assert!(route_response.is_empty());
13384 let published = route_client_rx.recv().await.unwrap();
13385 assert!(matches!(
13386 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13387 ClientControlResponse::RouteOpen { .. }
13388 ));
13389 }
13390
13391 async fn relay_route_open(
13396 handler: &ControlHandler,
13397 client_connection: ConnectionId,
13398 client_egress: &FrameSink,
13399 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13400 corr: u64,
13401 module_id: &str,
13402 project_root_label: &str,
13403 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
13404 let ctx = RouteCtx {
13405 connection_id: client_connection,
13406 egress: client_egress.clone(),
13407 };
13408 let handler = handler.clone();
13409 let project_root = unique_project_root(project_root_label);
13410 let module_id = module_id.to_string();
13411 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
13412 let task = tokio::spawn(async move {
13413 let _guard = tracing::dispatcher::set_default(&dispatch);
13414 handler
13415 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
13416 .await
13417 .unwrap()
13418 });
13419 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13420 .await
13421 .expect("module receives the relayed route.bind")
13422 .expect("module egress is open");
13423 (task, bind.frame)
13424 }
13425
13426 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
13427 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
13428 ModuleControlRequest::RouteBind {
13429 route_channel,
13430 epoch,
13431 ..
13432 } => (route_channel, epoch),
13433 other => panic!("expected a route.bind request, got {other:?}"),
13434 }
13435 }
13436
13437 fn published_route(frame: &Frame) -> (u16, u32) {
13438 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
13439 ClientControlResponse::RouteOpen {
13440 route_channel,
13441 route_epoch,
13442 ..
13443 } => (route_channel, route_epoch),
13444 other => panic!("expected a route.open response, got {other:?}"),
13445 }
13446 }
13447
13448 #[tokio::test]
13468 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
13469 let registry = Arc::new(Registry::default());
13470 let forwarding = Arc::new(ForwardingTable::default());
13471 let handler =
13472 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13473
13474 let module_connection = ConnectionId::new(30);
13475 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13476 hello_via_sink(
13477 &handler,
13478 &module_ctx,
13479 &mut module_rx,
13480 hello_frame("aft", PROTOCOL_VERSION, 7),
13481 )
13482 .await;
13483
13484 let dying_client = ConnectionId::new(31);
13485 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
13486
13487 let (first_task, first_bind) = relay_route_open(
13490 &handler,
13491 dying_client,
13492 &dying_ctx.egress,
13493 &mut module_rx,
13494 100,
13495 "aft",
13496 "closing-first",
13497 )
13498 .await;
13499 handler
13500 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
13501 .await
13502 .unwrap();
13503 assert!(first_task.await.unwrap().is_empty());
13504 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
13505
13506 let (second_task, second_bind) = relay_route_open(
13508 &handler,
13509 dying_client,
13510 &dying_ctx.egress,
13511 &mut module_rx,
13512 101,
13513 "aft",
13514 "closing-second",
13515 )
13516 .await;
13517 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
13518
13519 assert!(forwarding
13522 .escalate_client_delivery_failure(
13523 dying_client,
13524 first_channel,
13525 first_epoch,
13526 CloseReason::new(
13527 "module_to_client_delivery_failed",
13528 "client egress refused a module frame",
13529 ),
13530 crate::forwarding::UndeliveredFrame {
13531 module_id: None,
13532 sink: &dying_ctx.egress,
13533 },
13534 )
13535 .unwrap());
13536 assert!(!dying_ctx.egress.is_closed());
13537
13538 let ack = handler
13540 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
13541 .await;
13542 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13543 if module_loop_error.is_some() {
13544 handler.cleanup_connection(module_connection).unwrap();
13548 }
13549 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13553 .await
13554 .ok()
13555 .flatten();
13556
13557 assert!(
13559 registry
13560 .get_module_by_connection(module_connection)
13561 .unwrap()
13562 .is_some(),
13563 "one client's closing connection ended the shared module connection: \
13564 {module_loop_error:?}"
13565 );
13566 let cotenant = ConnectionId::new(32);
13568 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13569 let (cotenant_task, cotenant_bind) = relay_route_open(
13570 &handler,
13571 cotenant,
13572 &cotenant_ctx.egress,
13573 &mut module_rx,
13574 102,
13575 "aft",
13576 "closing-cotenant",
13577 )
13578 .await;
13579 handler
13580 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
13581 .await
13582 .unwrap();
13583 assert!(cotenant_task.await.unwrap().is_empty());
13584 let (cotenant_channel, cotenant_epoch) =
13585 published_route(&cotenant_rx.recv().await.unwrap());
13586 assert!(matches!(
13587 forwarding
13588 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
13589 .unwrap(),
13590 DataRoute::Client(DataRouteState::Bound(_))
13591 ));
13592
13593 let goodbye = post_ack_module_frame
13596 .expect("module receives a GOODBYE for the abandoned route channel");
13597 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13598 assert_eq!(goodbye.header.channel, abandoned_channel);
13599 assert_eq!(goodbye.header.epoch, abandoned_epoch);
13600
13601 assert!(dying_rx.try_recv().is_err());
13605 let second_response = second_task.await.unwrap();
13606 assert_eq!(second_response.len(), 1);
13607 assert_eq!(
13608 parse_error(&second_response[0])["code"],
13609 "target_unavailable"
13610 );
13611 }
13612
13613 async fn catalog_snapshot(
13616 handler: &ControlHandler,
13617 client_ctx: &RouteCtx,
13618 corr: u64,
13619 ) -> (u64, Vec<(String, bool)>) {
13620 let body =
13621 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
13622 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
13623 let reply = handler
13624 .handle_control_frame(client_ctx, request)
13625 .await
13626 .unwrap()
13627 .pop()
13628 .unwrap();
13629 let ClientControlResponse::CatalogList {
13630 generation,
13631 modules,
13632 ..
13633 } = serde_json::from_slice(&reply.body).unwrap()
13634 else {
13635 panic!("not a catalog.list reply");
13636 };
13637 (
13638 generation,
13639 modules
13640 .into_iter()
13641 .map(|entry| (entry.module_id, entry.ready))
13642 .collect(),
13643 )
13644 }
13645
13646 fn hello_held_before_forwarding() -> (
13652 ControlHandler,
13653 Arc<std::sync::Barrier>,
13654 std::thread::JoinHandle<Vec<Frame>>,
13655 RouteCtx,
13656 mpsc::Receiver<crate::router::OutboundFrame>,
13657 ) {
13658 let hold = Arc::new(std::sync::Barrier::new(2));
13659 let handler = ControlHandler::with_forwarding(
13660 Arc::new(Registry::default()),
13661 Arc::new(ForwardingTable::default()),
13662 )
13663 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
13664 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
13665 let hello = {
13666 let handler = handler.clone();
13667 let module_ctx = module_ctx.clone();
13668 std::thread::spawn(move || {
13669 handler
13670 .handle_hello(
13671 module_ctx.connection_id,
13672 Some(module_ctx.egress.clone()),
13673 hello_frame("aft", PROTOCOL_VERSION, 7),
13674 )
13675 .unwrap()
13676 })
13677 };
13678 (handler, hold, hello, module_ctx, module_rx)
13679 }
13680
13681 fn lookup_during_hello_publication<T: Send + 'static>(
13687 registry: &Registry,
13688 hold: &std::sync::Barrier,
13689 lookup: impl FnOnce() -> T + Send + 'static,
13690 ) -> T {
13691 let attempted = registry.observe_next_lock_attempt();
13692 let reader = std::thread::spawn(lookup);
13693 if attempted.recv().unwrap() {
13694 let result = reader.join().unwrap();
13695 hold.wait();
13696 result
13697 } else {
13698 hold.wait();
13699 reader.join().unwrap()
13700 }
13701 }
13702
13703 #[test]
13704 fn catalog_read_after_hello_ack_waits_for_publication() {
13705 let registry = Arc::new(Registry::default());
13706 let hold = Arc::new(std::sync::Barrier::new(2));
13707 let mut handler = ControlHandler::with_forwarding(
13708 Arc::clone(®istry),
13709 Arc::new(ForwardingTable::default()),
13710 );
13711 handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
13712 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13713 let hello = {
13714 let handler = handler.clone();
13715 std::thread::spawn(move || {
13716 handler
13717 .handle_hello(
13718 module_ctx.connection_id,
13719 Some(module_ctx.egress),
13720 hello_frame("aft", PROTOCOL_VERSION, 7),
13721 )
13722 .unwrap()
13723 })
13724 };
13725 hold.wait();
13726 assert_eq!(
13727 module_rx.try_recv().unwrap().frame.header.ty,
13728 FrameType::HelloAck
13729 );
13730 let replies = lookup_during_hello_publication(®istry, &hold, move || {
13731 let request = Frame::build(
13734 FrameType::Request,
13735 control_flags(),
13736 0,
13737 0,
13738 300,
13739 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13740 )
13741 .unwrap();
13742 handler.handle_catalog_list(request, None).unwrap()
13743 });
13744 assert!(hello.join().unwrap().is_empty());
13745 let ClientControlResponse::CatalogList {
13746 generation,
13747 modules,
13748 ..
13749 } = serde_json::from_slice(&replies[0].body).unwrap()
13750 else {
13751 panic!("not a catalog.list reply");
13752 };
13753 assert_eq!(
13754 modules
13755 .iter()
13756 .map(|entry| entry.module_id.as_str())
13757 .collect::<Vec<_>>(),
13758 vec!["aft"]
13759 );
13760 assert_eq!(generation, 1);
13761 }
13762
13763 #[tokio::test]
13772 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
13773 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
13774 let client = ConnectionId::new(41);
13775 let (client_ctx, mut client_rx) = route_ctx(client);
13776
13777 hold.wait();
13778 assert!(handler.registry.get_module("aft").unwrap().is_some());
13780 assert!(!handler
13781 .forwarding
13782 .has_live_module_connection("aft")
13783 .unwrap());
13784 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
13785 assert!(
13786 listed.is_empty(),
13787 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
13788 );
13789 let refused = handler
13790 .handle_control_frame(
13791 &client_ctx,
13792 route_open_frame(301, "aft", unique_project_root("listed-window")),
13793 )
13794 .await
13795 .unwrap();
13796 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
13797 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
13798 let duplicate = handler
13799 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
13800 .await
13801 .unwrap();
13802 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13803
13804 hold.wait();
13805 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
13806 let ack = module_rx.try_recv().unwrap().frame;
13807 assert_eq!(ack.header.ty, FrameType::HelloAck);
13808 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
13809 assert_eq!(listed, vec![("aft".to_string(), true)]);
13810
13811 let (open_task, bind) = relay_route_open(
13812 &handler,
13813 client,
13814 &client_ctx.egress,
13815 &mut module_rx,
13816 303,
13817 "aft",
13818 "listed-after",
13819 )
13820 .await;
13821 assert!(handler
13822 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
13823 .await
13824 .unwrap()
13825 .is_empty());
13826 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
13827 published_route(&client_rx.try_recv().unwrap().frame);
13828 }
13829
13830 #[tokio::test]
13837 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
13838 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
13839 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13840 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
13841
13842 hold.wait();
13843 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
13844 assert!(listed.is_empty());
13845 assert_eq!(
13846 during, before,
13847 "the generation moved while the module was not listed"
13848 );
13849
13850 hold.wait();
13851 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
13852 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
13853 assert_eq!(listed, vec![("aft".to_string(), true)]);
13854 assert_eq!(
13855 after,
13856 before + 1,
13857 "the module appeared without exactly one generation bump"
13858 );
13859 }
13860
13861 #[tokio::test]
13866 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
13867 let registry = Arc::new(Registry::default());
13868 let handler = ControlHandler::with_forwarding(
13869 Arc::clone(®istry),
13870 Arc::new(ForwardingTable::default()),
13871 );
13872 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13873 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
13874 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
13875 drop(module_rx);
13876
13877 let replies = handler
13878 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
13879 .await
13880 .unwrap();
13881 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
13882 assert!(registry.get_module("aft").unwrap().is_none());
13883 assert!(registry
13884 .get_module_by_connection(module_ctx.connection_id)
13885 .unwrap()
13886 .is_none());
13887 assert_eq!(
13888 catalog_snapshot(&handler, &client_ctx, 301).await,
13889 (before, Vec::new())
13890 );
13891 }
13892
13893 async fn route_open_held_after_bind_send() -> (
13897 ControlHandler,
13898 Arc<tokio::sync::Barrier>,
13899 RouteCtx,
13900 mpsc::Receiver<crate::router::OutboundFrame>,
13901 ) {
13902 let hold = Arc::new(tokio::sync::Barrier::new(2));
13903 let handler = ControlHandler::with_forwarding(
13904 Arc::new(Registry::default()),
13905 Arc::new(ForwardingTable::default()),
13906 )
13907 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
13908 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13909 hello_via_sink(
13910 &handler,
13911 &module_ctx,
13912 &mut module_rx,
13913 hello_frame("aft", PROTOCOL_VERSION, 7),
13914 )
13915 .await;
13916 (handler, hold, module_ctx, module_rx)
13917 }
13918
13919 #[tokio::test]
13926 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
13927 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
13928 let client = ConnectionId::new(41);
13929 let (client_ctx, mut client_rx) = route_ctx(client);
13930
13931 let (open_task, bind) = relay_route_open(
13932 &handler,
13933 client,
13934 &client_ctx.egress,
13935 &mut module_rx,
13936 200,
13937 "aft",
13938 "acked-before-mark",
13939 )
13940 .await;
13941 let (module_channel, module_epoch) = route_bind_channel(&bind);
13942 hold.wait().await;
13944
13945 assert!(handler
13947 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
13948 .await
13949 .unwrap()
13950 .is_empty());
13951 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
13952
13953 hold.wait().await;
13954 assert!(
13955 open_task.await.unwrap().is_empty(),
13956 "route.open answers accepted"
13957 );
13958
13959 match module_rx.try_recv() {
13962 Err(mpsc::error::TryRecvError::Empty) => {}
13963 Ok(outbound) => panic!(
13964 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
13965 outbound.frame.header.ty,
13966 outbound.frame.header.channel,
13967 outbound.frame.header.epoch
13968 ),
13969 Err(err) => panic!("module egress closed: {err}"),
13970 }
13971
13972 let router = Router::with_control_handler(Arc::new(handler.clone()));
13975 let request = Frame::build(
13976 FrameType::Request,
13977 Flags::new(true, Priority::Interactive, false),
13978 client_channel,
13979 client_epoch,
13980 201,
13981 b"first request".to_vec(),
13982 )
13983 .unwrap();
13984 router
13985 .route_for_connection(&client_ctx, request)
13986 .await
13987 .unwrap();
13988 let delivered = module_rx.try_recv().unwrap().frame;
13989 assert_eq!(delivered.header.ty, FrameType::Request);
13990 assert_eq!(
13991 (delivered.header.channel, delivered.header.epoch),
13992 (module_channel, module_epoch)
13993 );
13994 assert_eq!(delivered.body, b"first request");
13995 }
13996
13997 #[tokio::test]
14003 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
14004 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14005 let client = ConnectionId::new(42);
14006 let (client_ctx, _client_rx) = route_ctx(client);
14007
14008 let (open_task, bind) = relay_route_open(
14009 &handler,
14010 client,
14011 &client_ctx.egress,
14012 &mut module_rx,
14013 300,
14014 "aft",
14015 "released-before-mark",
14016 )
14017 .await;
14018 let (module_channel, module_epoch) = route_bind_channel(&bind);
14019 hold.wait().await;
14020
14021 handler.cleanup_connection(client).unwrap();
14022 assert!(
14023 module_rx.try_recv().is_err(),
14024 "client teardown must not GOODBYE a bind route.open is still queueing"
14025 );
14026
14027 hold.wait().await;
14028 let response = open_task.await.unwrap();
14029 assert_eq!(response.len(), 1);
14030 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14031
14032 let goodbye = module_rx
14033 .try_recv()
14034 .expect("module is told to drop the binding the daemon abandoned")
14035 .frame;
14036 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14037 assert_eq!(
14038 (goodbye.header.channel, goodbye.header.epoch),
14039 (module_channel, module_epoch)
14040 );
14041 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14042 }
14043
14044 #[tokio::test]
14049 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
14050 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14051 let client = ConnectionId::new(43);
14052 let (client_ctx, client_rx) = route_ctx(client);
14053
14054 let (open_task, bind) = relay_route_open(
14055 &handler,
14056 client,
14057 &client_ctx.egress,
14058 &mut module_rx,
14059 400,
14060 "aft",
14061 "gone-client-before-mark",
14062 )
14063 .await;
14064 let (module_channel, module_epoch) = route_bind_channel(&bind);
14065 hold.wait().await;
14066
14067 drop(client_rx);
14068 assert!(client_ctx.egress.is_closed());
14069 handler
14070 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14071 .await
14072 .unwrap();
14073
14074 hold.wait().await;
14075 let response = open_task.await.unwrap();
14076 assert_eq!(response.len(), 1);
14077 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14078
14079 let goodbye = module_rx
14080 .try_recv()
14081 .expect("module is told to drop the binding it acked")
14082 .frame;
14083 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14084 assert_eq!(
14085 (goodbye.header.channel, goodbye.header.epoch),
14086 (module_channel, module_epoch)
14087 );
14088 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14089 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14090 }
14091
14092 #[test]
14099 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
14100 let handler = ControlHandler::default();
14101 let module_connection = ConnectionId::new(30);
14102 let client_connection = ConnectionId::new(31);
14103
14104 handler
14105 .refuse_to_end_module_connection_for_a_client(
14106 module_connection,
14107 77,
14108 ForwardingError::ConnectionClosing {
14109 connection_id: client_connection,
14110 },
14111 )
14112 .expect("a closing client must never end the module connection");
14113
14114 assert!(matches!(
14115 handler.refuse_to_end_module_connection_for_a_client(
14116 module_connection,
14117 78,
14118 ForwardingError::ConnectionClosing {
14119 connection_id: module_connection,
14120 },
14121 ),
14122 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
14123 connection_id
14124 })) if connection_id == module_connection
14125 ));
14126 assert!(matches!(
14127 handler.refuse_to_end_module_connection_for_a_client(
14128 module_connection,
14129 79,
14130 ForwardingError::Poisoned,
14131 ),
14132 Err(RouterError::Forwarding(ForwardingError::Poisoned))
14133 ));
14134 assert!(matches!(
14135 handler.refuse_to_end_module_connection_for_a_client(
14136 module_connection,
14137 80,
14138 ForwardingError::StaleModuleEndpoint,
14139 ),
14140 Err(RouterError::Forwarding(
14141 ForwardingError::StaleModuleEndpoint
14142 ))
14143 ));
14144 }
14145
14146 #[tokio::test]
14159 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
14160 let registry = Arc::new(Registry::default());
14161 let forwarding = Arc::new(ForwardingTable::default());
14162 let supervisor = SupervisorHandle::new();
14163 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14164 let handler =
14165 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14166 .with_supervisor(supervisor);
14167
14168 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14169 hello_via_sink(
14170 &handler,
14171 &target_ctx,
14172 &mut target_rx,
14173 hello_frame("target", PROTOCOL_VERSION, 1),
14174 )
14175 .await;
14176
14177 let wrong_nonce = handler
14181 .handle_control_frame(
14182 &route_ctx(ConnectionId::new(91)).0,
14183 route_open_frame_with_admission_facts(
14184 20,
14185 "target",
14186 unique_project_root("admission-facts"),
14187 Some(subc_control::ConsumerIdentity {
14188 module_id: "fed".to_string(),
14189 launch_nonce: "not-the-real-nonce".to_string(),
14190 }),
14191 None,
14192 ),
14193 )
14194 .await
14195 .unwrap();
14196 assert_eq!(
14197 parse_error(&wrong_nonce[0])["code"],
14198 "bad_consumer_identity",
14199 "a mismatched launch nonce must be refused, not stamped Reserved"
14200 );
14201
14202 let never_spawned = handler
14206 .handle_control_frame(
14207 &route_ctx(ConnectionId::new(92)).0,
14208 route_open_frame_with_admission_facts(
14209 21,
14210 "target",
14211 unique_project_root("admission-facts"),
14212 Some(subc_control::ConsumerIdentity {
14213 module_id: "never-spawned".to_string(),
14214 launch_nonce: "any-nonce".to_string(),
14215 }),
14216 None,
14217 ),
14218 )
14219 .await
14220 .unwrap();
14221 assert_eq!(
14222 parse_error(&never_spawned[0])["code"],
14223 "bad_consumer_identity",
14224 "an unspawned module_id must be refused rather than accepted for lack of a record"
14225 );
14226 }
14227
14228 #[tokio::test]
14251 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
14252 let registry = Arc::new(Registry::default());
14253 let forwarding = Arc::new(ForwardingTable::default());
14254 let supervisor = SupervisorHandle::new();
14255 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14256 let handler =
14257 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14258 .with_supervisor(supervisor);
14259
14260 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
14261 hello_via_sink(
14262 &handler,
14263 &target_ctx,
14264 &mut target_rx,
14265 hello_frame("target", PROTOCOL_VERSION, 1),
14266 )
14267 .await;
14268
14269 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
14270 let route_handler = handler.clone();
14271 let route_task = tokio::spawn(async move {
14272 route_handler
14273 .handle_control_frame(
14274 &client_ctx,
14275 route_open_frame_with_admission_facts(
14276 30,
14277 "target",
14278 unique_project_root("admission-facts"),
14279 Some(subc_control::ConsumerIdentity {
14280 module_id: "fed".to_string(),
14281 launch_nonce: "fed-nonce".to_string(),
14282 }),
14283 None,
14284 ),
14285 )
14286 .await
14287 .unwrap()
14288 });
14289
14290 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14297 .await
14298 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
14299 .expect("module control channel closed before route.bind");
14300 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14301 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14302 panic!("expected route.bind")
14303 };
14304 assert_eq!(
14305 principal,
14306 Some(Principal::Reserved {
14307 module_id: "fed".to_string()
14308 }),
14309 "a correctly attested consumer must be stamped Reserved for its own id"
14310 );
14311
14312 handler
14313 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14314 .await
14315 .unwrap();
14316 assert!(route_task.await.unwrap().is_empty());
14317 assert!(
14318 matches!(
14319 serde_json::from_slice::<ClientControlResponse>(
14320 &client_rx.recv().await.unwrap().body
14321 )
14322 .unwrap(),
14323 ClientControlResponse::RouteOpen { .. }
14324 ),
14325 "the route must actually open, not merely avoid an error"
14326 );
14327 }
14328
14329 #[tokio::test(start_paused = true)]
14330 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
14331 let registry = Arc::new(Registry::default());
14332 let forwarding = Arc::new(ForwardingTable::default());
14333 let supervisor = SupervisorHandle::new();
14334 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14335 let handler =
14336 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14337 .with_supervisor(supervisor);
14338
14339 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
14340 hello_via_sink(
14341 &handler,
14342 &target_ctx,
14343 &mut target_rx,
14344 hello_frame("target", PROTOCOL_VERSION, 1),
14345 )
14346 .await;
14347
14348 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
14349 let direct_handler = handler.clone();
14350 let direct_open = tokio::spawn(async move {
14351 direct_handler
14352 .handle_control_frame(
14353 &direct_ctx,
14354 route_open_frame(2, "target", unique_project_root("route-census-direct")),
14355 )
14356 .await
14357 .unwrap()
14358 });
14359 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14360 .await
14361 .expect("no direct route.bind within 5s")
14362 .expect("target control channel closed before direct route.bind");
14363 handler
14364 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
14365 .await
14366 .unwrap();
14367 assert!(direct_open.await.unwrap().is_empty());
14368 let _ = direct_rx.recv().await.unwrap();
14369
14370 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
14371 let reserved_handler = handler.clone();
14372 let reserved_open = tokio::spawn(async move {
14373 reserved_handler
14374 .handle_control_frame(
14375 &reserved_ctx,
14376 route_open_frame_with_admission_facts(
14377 3,
14378 "target",
14379 unique_project_root("admission-facts"),
14380 Some(ConsumerIdentity {
14381 module_id: "fed".to_string(),
14382 launch_nonce: "fed-nonce".to_string(),
14383 }),
14384 None,
14385 ),
14386 )
14387 .await
14388 .unwrap()
14389 });
14390 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14391 .await
14392 .expect("no reserved route.bind within 5s")
14393 .expect("target control channel closed before reserved route.bind");
14394 handler
14395 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
14396 .await
14397 .unwrap();
14398 assert!(reserved_open.await.unwrap().is_empty());
14399 let _ = reserved_rx.recv().await.unwrap();
14400
14401 forwarding
14402 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
14403 .unwrap();
14404 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
14405 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
14406 module_id: Some("target".to_string()),
14407 })
14408 .unwrap();
14409 let census_frame =
14410 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
14411 let response = handler
14412 .handle_control_frame(&census_ctx, census_frame)
14413 .await
14414 .unwrap()
14415 .pop()
14416 .unwrap();
14417 let actual: Value = serde_json::from_slice(&response.body).unwrap();
14418 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
14419 assert!(matches!(
14420 decoded,
14421 ClientControlResponse::SupervisorRoutes { .. }
14422 ));
14423 let routes = actual["modules"][0]["routes"].as_array().unwrap();
14424 assert_eq!(routes.len(), 2);
14425 assert!(routes.iter().all(|route| route["draining"] == true));
14426 assert!(
14429 routes.iter().all(|route| route["drain_reason"] == "reload"),
14430 "draining routes must name the drain's reason: {routes:?}"
14431 );
14432 assert!(routes.iter().any(|route| {
14433 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
14434 }));
14435 assert!(routes.iter().any(|route| {
14436 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
14437 }));
14438
14439 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14440 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
14441 if std::env::var_os("UPDATE_GOLDEN").is_some() {
14442 std::fs::write(
14443 &golden_path,
14444 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
14445 )
14446 .unwrap();
14447 }
14448 let expected: Value =
14449 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
14450 assert_eq!(actual, expected);
14451 }
14452
14453 async fn query_live_roots(
14454 handler: &ControlHandler,
14455 module_ctx: &RouteCtx,
14456 ) -> ModuleControlResponseToModule {
14457 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
14458 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
14459 let response = handler
14460 .handle_control_frame(module_ctx, frame)
14461 .await
14462 .unwrap()
14463 .pop()
14464 .unwrap();
14465 serde_json::from_slice(&response.body).unwrap()
14466 }
14467
14468 #[tokio::test(start_paused = true)]
14469 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
14470 let registry = Arc::new(Registry::default());
14471 let forwarding = Arc::new(ForwardingTable::default());
14472 let handler = ControlHandler::with_forwarding(registry, forwarding);
14473 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
14474 hello_via_sink(
14475 &handler,
14476 &target_ctx,
14477 &mut target_rx,
14478 hello_frame("target", PROTOCOL_VERSION, 1),
14479 )
14480 .await;
14481 let root = unique_project_root("live-roots-known");
14482 let path = ProjectRootId::from_path_allowing_missing(root.path())
14483 .unwrap()
14484 .as_path()
14485 .to_path_buf();
14486 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
14487 let open_handler = handler.clone();
14488 let opened = tokio::spawn(async move {
14489 open_handler
14490 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
14491 .await
14492 .unwrap()
14493 });
14494 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14495 .await
14496 .unwrap()
14497 .unwrap();
14498 handler
14499 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
14500 .await
14501 .unwrap();
14502 assert!(opened.await.unwrap().is_empty());
14503 let _ = client_rx.recv().await.unwrap();
14504
14505 let root = unique_project_root("live-roots-pending");
14506 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
14507 .unwrap()
14508 .as_path()
14509 .to_path_buf();
14510 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
14511 let open_handler = handler.clone();
14512 let pending = tokio::spawn(async move {
14513 open_handler
14514 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
14515 .await
14516 .unwrap()
14517 });
14518 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14519 .await
14520 .unwrap()
14521 .unwrap();
14522 let actual = query_live_roots(&handler, &target_ctx).await;
14523 let ModuleControlResponseToModule::LiveRoots {
14524 roots,
14525 unknown_root_bindings,
14526 total_bindings,
14527 } = actual
14528 else {
14529 panic!("expected live roots")
14530 };
14531 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
14532 assert_eq!(unknown_root_bindings, 0);
14533 assert_eq!(
14534 roots.len(),
14535 2,
14536 "root-known arm must retain each canonical root"
14537 );
14538 assert_eq!(
14539 total_bindings,
14540 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14541 );
14542 let counts = roots
14543 .iter()
14544 .map(|root| (root.project_root.clone(), root.bound, root.pending))
14545 .collect::<Vec<_>>();
14546 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
14547 expected.sort_by(|a, b| a.0.cmp(&b.0));
14548 assert_eq!(
14549 counts, expected,
14550 "roots must sort by path and count pending separately"
14551 );
14552 handler
14553 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
14554 .await
14555 .unwrap();
14556 assert!(pending.await.unwrap().is_empty());
14557 }
14558
14559 #[tokio::test(start_paused = true)]
14560 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
14561 let forwarding = Arc::new(ForwardingTable::default());
14562 let handler =
14563 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14564 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
14565 hello_via_sink(
14566 &handler,
14567 &target_ctx,
14568 &mut target_rx,
14569 hello_frame("target", PROTOCOL_VERSION, 1),
14570 )
14571 .await;
14572 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
14573 let pending = forwarding
14574 .begin_route_bind_relay_for_test(
14575 client_ctx.connection_id,
14576 client_ctx.egress.clone(),
14577 2,
14578 "target",
14579 )
14580 .unwrap();
14581 forwarding
14582 .complete_pending_relay(
14583 target_ctx.connection_id,
14584 pending.corr,
14585 RouteBindRelayOutcome::Accepted,
14586 )
14587 .unwrap();
14588 let actual = query_live_roots(&handler, &target_ctx).await;
14589 let ModuleControlResponseToModule::LiveRoots {
14590 roots,
14591 unknown_root_bindings,
14592 total_bindings,
14593 } = actual
14594 else {
14595 panic!("expected live roots")
14596 };
14597 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
14598 assert_eq!(
14599 unknown_root_bindings, 1,
14600 "unknown-root arm must not read as no bindings"
14601 );
14602 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
14603 assert_eq!(
14604 total_bindings,
14605 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14606 );
14607 }
14608
14609 #[tokio::test(start_paused = true)]
14617 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
14618 let forwarding = Arc::new(ForwardingTable::default());
14619 let handler =
14620 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14621 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
14622 let replies = handler
14623 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
14624 .await
14625 .unwrap();
14626 let queued_by_hello = module_rx.len();
14627
14628 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
14629 let open_handler = handler.clone();
14630 let open = tokio::spawn(async move {
14631 open_handler
14632 .handle_control_frame(
14633 &client_ctx,
14634 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
14635 )
14636 .await
14637 .unwrap()
14638 });
14639 let mut spins = 0;
14641 while module_rx.len() == queued_by_hello {
14642 spins += 1;
14643 assert!(spins < 10_000, "route.open never queued a route.bind");
14644 tokio::task::yield_now().await;
14645 }
14646
14647 for reply in replies {
14649 module_ctx.egress.send(reply).await.unwrap();
14650 }
14651
14652 let first = module_rx.recv().await.unwrap().frame;
14653 assert_eq!(
14654 first.header.ty,
14655 FrameType::HelloAck,
14656 "the first frame a registering module reads must be its HELLO_ACK"
14657 );
14658 assert_eq!(first.header.corr, 7);
14659 let second = module_rx.recv().await.unwrap().frame;
14660 assert_eq!(second.header.ty, FrameType::Request);
14661 assert!(
14662 matches!(
14663 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
14664 ModuleControlRequest::RouteBind { .. }
14665 ),
14666 "the route.bind follows the ack"
14667 );
14668 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
14669
14670 handler
14671 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
14672 .await
14673 .unwrap();
14674 assert!(open.await.unwrap().is_empty());
14675 let _ = client_rx.recv().await.unwrap();
14676 }
14677
14678 #[tokio::test(start_paused = true)]
14679 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
14680 let handler = ControlHandler::with_forwarding(
14681 Arc::new(Registry::default()),
14682 Arc::new(ForwardingTable::default()),
14683 );
14684 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
14685 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
14686 hello_via_sink(
14687 &handler,
14688 &first_ctx,
14689 &mut first_rx,
14690 hello_frame("first", PROTOCOL_VERSION, 1),
14691 )
14692 .await;
14693 hello_via_sink(
14694 &handler,
14695 &second_ctx,
14696 &mut second_rx,
14697 hello_frame("second", PROTOCOL_VERSION, 2),
14698 )
14699 .await;
14700 let root = unique_project_root("second-only");
14701 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
14702 let cloned = handler.clone();
14703 let open = tokio::spawn(async move {
14704 cloned
14705 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
14706 .await
14707 .unwrap()
14708 });
14709 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
14710 .await
14711 .unwrap()
14712 .unwrap();
14713 let first = query_live_roots(&handler, &first_ctx).await;
14714 let second = query_live_roots(&handler, &second_ctx).await;
14715 assert!(
14716 matches!(
14717 first,
14718 ModuleControlResponseToModule::LiveRoots {
14719 total_bindings: 0,
14720 ..
14721 }
14722 ),
14723 "cross-module scope must not expose another module's roots"
14724 );
14725 assert!(
14726 matches!(
14727 second,
14728 ModuleControlResponseToModule::LiveRoots {
14729 total_bindings: 1,
14730 ..
14731 }
14732 ),
14733 "second module must see its pending route"
14734 );
14735 handler
14736 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
14737 .await
14738 .unwrap();
14739 assert!(open.await.unwrap().is_empty());
14740 }
14741
14742 #[tokio::test(start_paused = true)]
14743 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
14744 let handler = ControlHandler::with_forwarding(
14745 Arc::new(Registry::default()),
14746 Arc::new(ForwardingTable::default()),
14747 );
14748 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
14749 hello_via_sink(
14750 &handler,
14751 &target_ctx,
14752 &mut target_rx,
14753 hello_frame("target", PROTOCOL_VERSION, 1),
14754 )
14755 .await;
14756 let actual = query_live_roots(&handler, &target_ctx).await;
14757 let ModuleControlResponseToModule::LiveRoots {
14758 roots,
14759 unknown_root_bindings,
14760 total_bindings,
14761 } = actual
14762 else {
14763 panic!("expected live roots")
14764 };
14765 assert!(roots.is_empty());
14766 assert_eq!(unknown_root_bindings, 0);
14767 assert_eq!(total_bindings, 0);
14768 assert_eq!(
14769 total_bindings,
14770 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14771 );
14772 }
14773
14774 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
14781 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14782 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
14783 let text = std::fs::read_to_string(&path)
14784 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
14785 let vectors: Vec<(String, Value)> = text
14786 .lines()
14787 .filter(|line| !line.trim().is_empty())
14788 .map(|line| {
14789 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
14790 let id = entry["corpus_id"]
14791 .as_str()
14792 .expect("every vector carries a corpus_id")
14793 .to_string();
14794 (id, entry["package"].clone())
14795 })
14796 .collect();
14797 assert_eq!(
14801 vectors.len(),
14802 3,
14803 "vendored fed corpus changed size; re-sync from subc-federation"
14804 );
14805
14806 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
14821 let richest = vectors
14822 .iter()
14823 .filter_map(|(_, package)| package.as_object())
14824 .map(|object| {
14825 object
14826 .keys()
14827 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
14828 .count()
14829 })
14830 .max()
14831 .unwrap_or(0);
14832 assert!(
14833 richest >= 2,
14834 "vendored corpus no longer carries a package with unmodelled fields, \
14835 so the relay test can no longer distinguish a verbatim relay from a lossy one"
14836 );
14837
14838 vectors
14839 }
14840
14841 #[tokio::test]
14849 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
14850 for (corpus_id, package) in fed_admission_facts_vectors() {
14851 let registry = Arc::new(Registry::default());
14852 let forwarding = Arc::new(ForwardingTable::default());
14853 let supervisor = SupervisorHandle::new();
14854 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14855 let handler =
14856 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14857 .with_supervisor(supervisor)
14858 .with_admission_facts_config(
14859 Some("fed".to_string()),
14860 Some(vec!["target".to_string()]),
14861 );
14862
14863 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14864 hello_via_sink(
14865 &handler,
14866 &target_ctx,
14867 &mut target_rx,
14868 hello_frame("target", PROTOCOL_VERSION, 1),
14869 )
14870 .await;
14871
14872 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
14873 let route_handler = handler.clone();
14874 let expected = package.clone();
14875 let route_task = tokio::spawn(async move {
14876 route_handler
14877 .handle_control_frame(
14878 &client_ctx,
14879 route_open_frame_with_admission_facts(
14880 20,
14881 "target",
14882 unique_project_root("admission-facts"),
14883 Some(subc_control::ConsumerIdentity {
14884 module_id: "fed".to_string(),
14885 launch_nonce: "fed-nonce".to_string(),
14886 }),
14887 Some(package),
14888 ),
14889 )
14890 .await
14891 .unwrap()
14892 });
14893
14894 let bind_frame = target_rx.recv().await.unwrap();
14895 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14896 let ModuleControlRequest::RouteBind {
14897 admission_facts, ..
14898 } = bind
14899 else {
14900 panic!("{corpus_id}: expected route.bind")
14901 };
14902 assert_eq!(
14903 admission_facts,
14904 Some(expected),
14905 "{corpus_id}: relay must not add, drop or reshape any field"
14906 );
14907
14908 handler
14909 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14910 .await
14911 .unwrap();
14912 route_task.await.unwrap();
14913 }
14914 }
14915
14916 #[tokio::test]
14917 async fn route_bind_never_relays_the_openers_build_class() {
14918 let registry = Arc::new(Registry::default());
14919 let forwarding = Arc::new(ForwardingTable::default());
14920 let handler =
14921 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14922 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
14923 hello_via_sink(
14924 &handler,
14925 &target_ctx,
14926 &mut target_rx,
14927 hello_frame("target", PROTOCOL_VERSION, 1),
14928 )
14929 .await;
14930
14931 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
14932 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
14933 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
14934 busy_refusals: None,
14935 target: RouteTarget::ToolProvider {
14936 module_id: "target".to_string(),
14937 },
14938 identity,
14939 consumer_identity: None,
14940 consumer_capabilities: None,
14941 role_versions: None,
14942 admission_facts: None,
14943 scope: None,
14944 })
14945 .unwrap();
14946 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
14947
14948 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
14949 let route_handler = handler.clone();
14950 let route_task = tokio::spawn(async move {
14951 route_handler
14952 .handle_control_frame(&client_ctx, open)
14953 .await
14954 .unwrap()
14955 });
14956 let bind_frame = target_rx.recv().await.unwrap();
14957 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14958 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
14959 panic!("expected route.bind")
14960 };
14961 assert_eq!(
14962 identity.build_class, None,
14963 "route.bind carried the build class"
14964 );
14965 assert!(
14966 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
14967 "the route.bind bytes still name build_class"
14968 );
14969 handler
14970 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14971 .await
14972 .unwrap();
14973 route_task.await.unwrap();
14974 }
14975
14976 #[tokio::test]
14977 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
14978 let registry = Arc::new(Registry::default());
14979 let forwarding = Arc::new(ForwardingTable::default());
14980 let supervisor = SupervisorHandle::new();
14981 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14982 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
14983 let handler =
14984 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14985 .with_supervisor(supervisor)
14986 .with_admission_facts_config(
14987 Some("fed".to_string()),
14988 Some(vec!["target".to_string()]),
14989 );
14990
14991 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
14992 hello_via_sink(
14993 &handler,
14994 &target_ctx,
14995 &mut target_rx,
14996 hello_frame("target", PROTOCOL_VERSION, 1),
14997 )
14998 .await;
14999 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
15000 hello_via_sink(
15001 &handler,
15002 &other_ctx,
15003 &mut other_rx,
15004 hello_frame("other", PROTOCOL_VERSION, 2),
15005 )
15006 .await;
15007
15008 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
15009 let expected_facts = facts.clone();
15010 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
15011 let route_handler = handler.clone();
15012 let route_task = tokio::spawn(async move {
15013 route_handler
15014 .handle_control_frame(
15015 &client_ctx,
15016 route_open_frame_with_admission_facts(
15017 10,
15018 "target",
15019 unique_project_root("admission-facts"),
15020 Some(subc_control::ConsumerIdentity {
15021 module_id: "fed".to_string(),
15022 launch_nonce: "fed-nonce".to_string(),
15023 }),
15024 Some(facts.clone()),
15025 ),
15026 )
15027 .await
15028 .unwrap()
15029 });
15030 let bind_frame = target_rx.recv().await.unwrap();
15031 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15032 let ModuleControlRequest::RouteBind {
15033 admission_facts, ..
15034 } = bind
15035 else {
15036 panic!("expected route.bind")
15037 };
15038 assert_eq!(admission_facts, Some(expected_facts));
15039 handler
15040 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15041 .await
15042 .unwrap();
15043 assert!(route_task.await.unwrap().is_empty());
15044 assert!(matches!(
15045 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
15046 .unwrap(),
15047 ClientControlResponse::RouteOpen { .. }
15048 ));
15049
15050 let direct = handler
15051 .handle_control_frame(
15052 &route_ctx(ConnectionId::new(73)).0,
15053 route_open_frame_with_admission_facts(
15054 11,
15055 "target",
15056 unique_project_root("admission-facts"),
15057 None,
15058 Some(json!({"x": 1})),
15059 ),
15060 )
15061 .await
15062 .unwrap();
15063 assert_eq!(
15064 parse_error(&direct[0])["code"],
15065 "admission_facts_not_permitted"
15066 );
15067
15068 let different_reserved = handler
15069 .handle_control_frame(
15070 &route_ctx(ConnectionId::new(77)).0,
15071 route_open_frame_with_admission_facts(
15072 15,
15073 "target",
15074 unique_project_root("admission-facts"),
15075 Some(subc_control::ConsumerIdentity {
15076 module_id: "other".to_string(),
15077 launch_nonce: "other-nonce".to_string(),
15078 }),
15079 Some(json!({"x": 1})),
15080 ),
15081 )
15082 .await
15083 .unwrap();
15084 assert_eq!(
15085 parse_error(&different_reserved[0])["code"],
15086 "admission_facts_not_permitted"
15087 );
15088
15089 let other_target = handler
15090 .handle_control_frame(
15091 &route_ctx(ConnectionId::new(74)).0,
15092 route_open_frame_with_admission_facts(
15093 12,
15094 "other",
15095 unique_project_root("admission-facts"),
15096 Some(subc_control::ConsumerIdentity {
15097 module_id: "fed".to_string(),
15098 launch_nonce: "fed-nonce".to_string(),
15099 }),
15100 Some(json!({"x": 1})),
15101 ),
15102 )
15103 .await
15104 .unwrap();
15105 assert_eq!(
15106 parse_error(&other_target[0])["code"],
15107 "admission_facts_target_not_allowed"
15108 );
15109
15110 let nonexistent = handler
15111 .handle_control_frame(
15112 &route_ctx(ConnectionId::new(75)).0,
15113 route_open_frame_with_admission_facts(
15114 13,
15115 "missing",
15116 unique_project_root("admission-facts"),
15117 None,
15118 Some(json!({"x": 1})),
15119 ),
15120 )
15121 .await
15122 .unwrap();
15123 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
15124
15125 let described = handler
15126 .handle_control_frame(
15127 &route_ctx(ConnectionId::new(76)).0,
15128 Frame::build(
15129 FrameType::Request,
15130 control_flags(),
15131 0,
15132 0,
15133 14,
15134 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15135 )
15136 .unwrap(),
15137 )
15138 .await
15139 .unwrap();
15140 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15141 serde_json::from_slice(&described[0].body).unwrap()
15142 else {
15143 panic!("expected server.describe response")
15144 };
15145 assert!(capabilities
15146 .iter()
15147 .any(|cap| cap == "admission_facts_relay_v1"));
15148 }
15149
15150 #[tokio::test]
15151 async fn admission_facts_without_configured_carrier_are_rejected() {
15152 let registry = Arc::new(Registry::default());
15153 let forwarding = Arc::new(ForwardingTable::default());
15154 let handler = ControlHandler::with_forwarding(registry, forwarding);
15155 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
15156 hello_via_sink(
15157 &handler,
15158 &target_ctx,
15159 &mut target_rx,
15160 hello_frame("target", PROTOCOL_VERSION, 1),
15161 )
15162 .await;
15163
15164 let responses = handler
15165 .handle_control_frame(
15166 &route_ctx(ConnectionId::new(79)).0,
15167 route_open_frame_with_admission_facts(
15168 16,
15169 "target",
15170 unique_project_root("admission-facts"),
15171 None,
15172 Some(json!({"x": 1})),
15173 ),
15174 )
15175 .await
15176 .unwrap();
15177 assert_eq!(
15178 parse_error(&responses[0])["code"],
15179 "admission_facts_not_permitted"
15180 );
15181 }
15182
15183 #[tokio::test]
15184 async fn route_open_relays_consumer_capabilities_verbatim() {
15185 let registry = Arc::new(Registry::default());
15186 let forwarding = Arc::new(ForwardingTable::default());
15187 let handler =
15188 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15189 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
15190 hello_via_sink(
15191 &handler,
15192 &module_ctx,
15193 &mut module_rx,
15194 hello_frame("aft", PROTOCOL_VERSION, 7),
15195 )
15196 .await;
15197
15198 let expected = vec!["elicitation".to_string(), "roots".to_string()];
15199 let expected_for_request = expected.clone();
15200 let project_root = unique_project_root("consumer-capabilities-present");
15201 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
15202 let route_handler = handler.clone();
15203 let route_task = tokio::spawn(async move {
15204 route_handler
15205 .handle_control_frame(
15206 &client_ctx,
15207 route_open_frame_with_consumer_capabilities(
15208 401,
15209 "aft",
15210 project_root,
15211 Some(expected_for_request),
15212 ),
15213 )
15214 .await
15215 .unwrap()
15216 });
15217 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15218 .await
15219 .unwrap()
15220 .unwrap();
15221 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15222 let ModuleControlRequest::RouteBind {
15223 consumer_capabilities,
15224 ..
15225 } = bind
15226 else {
15227 panic!("expected route.bind request, got {bind:?}");
15228 };
15229 assert_eq!(consumer_capabilities, Some(expected.clone()));
15230
15231 handler
15232 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15233 .await
15234 .unwrap();
15235 let route_response = route_task.await.unwrap();
15236 assert!(route_response.is_empty());
15237 let published = client_rx.recv().await.unwrap();
15238 assert!(matches!(
15239 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15240 ClientControlResponse::RouteOpen { .. }
15241 ));
15242 }
15243
15244 #[tokio::test]
15245 async fn route_open_without_consumer_capabilities_relays_none() {
15246 let registry = Arc::new(Registry::default());
15247 let forwarding = Arc::new(ForwardingTable::default());
15248 let handler =
15249 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15250 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
15251 hello_via_sink(
15252 &handler,
15253 &module_ctx,
15254 &mut module_rx,
15255 hello_frame("aft", PROTOCOL_VERSION, 7),
15256 )
15257 .await;
15258
15259 let project_root = unique_project_root("consumer-capabilities-absent");
15260 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
15261 let route_handler = handler.clone();
15262 let route_task = tokio::spawn(async move {
15263 route_handler
15264 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
15265 .await
15266 .unwrap()
15267 });
15268 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15269 .await
15270 .unwrap()
15271 .unwrap();
15272 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15273 let ModuleControlRequest::RouteBind {
15274 consumer_capabilities,
15275 ..
15276 } = bind
15277 else {
15278 panic!("expected route.bind request, got {bind:?}");
15279 };
15280 assert_eq!(consumer_capabilities, None);
15281
15282 handler
15283 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15284 .await
15285 .unwrap();
15286 let route_response = route_task.await.unwrap();
15287 assert!(route_response.is_empty());
15288 let published = client_rx.recv().await.unwrap();
15289 assert!(matches!(
15290 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15291 ClientControlResponse::RouteOpen { .. }
15292 ));
15293 }
15294
15295 async fn bind_role_versions_for(
15299 sent: Option<BTreeMap<String, String>>,
15300 connection: u64,
15301 ) -> Option<BTreeMap<String, String>> {
15302 let registry = Arc::new(Registry::default());
15303 let forwarding = Arc::new(ForwardingTable::default());
15304 let handler =
15305 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15306 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
15307 hello_via_sink(
15308 &handler,
15309 &module_ctx,
15310 &mut module_rx,
15311 hello_frame("aft", PROTOCOL_VERSION, 7),
15312 )
15313 .await;
15314 let project_root = unique_project_root("role-versions");
15315 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
15316 let route_handler = handler.clone();
15317 let route_task = tokio::spawn(async move {
15318 route_handler
15319 .handle_control_frame(
15320 &client_ctx,
15321 route_open_frame_with_role_versions(403, "aft", project_root, sent),
15322 )
15323 .await
15324 .unwrap()
15325 });
15326 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15327 .await
15328 .expect("a well-formed route.open reaches the module as a bind")
15329 .unwrap();
15330 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15331 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
15332 panic!("expected route.bind request, got {bind:?}");
15333 };
15334 handler
15335 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15336 .await
15337 .unwrap();
15338 assert!(route_task.await.unwrap().is_empty());
15339 let published = client_rx.recv().await.unwrap();
15340 assert!(matches!(
15341 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15342 ClientControlResponse::RouteOpen { .. }
15343 ));
15344 role_versions
15345 }
15346
15347 #[tokio::test]
15348 async fn route_open_relays_role_versions_verbatim() {
15349 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
15350 assert_eq!(
15351 bind_role_versions_for(Some(sent.clone()), 141).await,
15352 Some(sent)
15353 );
15354 }
15355
15356 #[tokio::test]
15359 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
15360 assert_eq!(bind_role_versions_for(None, 143).await, None);
15361 assert_eq!(
15362 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
15363 None
15364 );
15365 }
15366
15367 #[tokio::test]
15368 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
15369 let registry = Arc::new(Registry::default());
15370 let forwarding = Arc::new(ForwardingTable::default());
15371 let handler =
15372 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15373 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
15374 hello_via_sink(
15375 &handler,
15376 &module_ctx,
15377 &mut module_rx,
15378 hello_frame("aft", PROTOCOL_VERSION, 7),
15379 )
15380 .await;
15381
15382 let nine: BTreeMap<String, String> = (0..9)
15383 .map(|index| (format!("role-{index}"), "v1".to_string()))
15384 .collect();
15385 for (label, malformed) in [
15386 (
15387 "invalid role name",
15388 role_versions(&[("Tool_Provider", "v1")]),
15389 ),
15390 ("invalid version", role_versions(&[("tool-provider", "v0")])),
15391 ("nine entries", nine),
15392 ] {
15393 let responses = tokio::time::timeout(
15396 Duration::from_secs(1),
15397 handler.handle_control_frame(
15398 &route_ctx(ConnectionId::new(148)).0,
15399 route_open_frame_with_role_versions(
15400 404,
15401 "aft",
15402 unique_project_root("role-versions-malformed"),
15403 Some(malformed),
15404 ),
15405 ),
15406 )
15407 .await
15408 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
15409 .unwrap();
15410 assert_eq!(responses.len(), 1, "{label}");
15411 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
15412 let error = parse_error(&responses[0]);
15413 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
15414 assert_eq!(
15415 error["detail"]["field"], "role_versions",
15416 "{label}: {error}"
15417 );
15418 assert!(
15419 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
15420 "{label}: a malformed declaration is terminal"
15421 );
15422 assert!(
15423 module_rx.try_recv().is_err(),
15424 "{label}: the module must never see a bind"
15425 );
15426 }
15427 }
15428
15429 #[tokio::test]
15433 async fn route_role_versions_capability_is_advertised() {
15434 let handler = ControlHandler::new(Arc::new(Registry::default()));
15435 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
15436 let ack = hello_via_sink(
15437 &handler,
15438 &ctx,
15439 &mut rx,
15440 hello_frame("m", PROTOCOL_VERSION, 1),
15441 )
15442 .await;
15443 let ack = parse_ack(&ack);
15444 assert!(
15445 ack.subc_capabilities
15446 .iter()
15447 .any(|c| c == "route-role-versions/v1"),
15448 "{:?}",
15449 ack.subc_capabilities
15450 );
15451
15452 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15453 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15454 let reply = handler
15455 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
15456 .await
15457 .unwrap()
15458 .pop()
15459 .unwrap();
15460 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15461 serde_json::from_slice(&reply.body).unwrap()
15462 else {
15463 panic!("not a server.describe reply");
15464 };
15465 assert!(
15466 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
15467 "{capabilities:?}"
15468 );
15469 }
15470
15471 #[tokio::test]
15475 async fn catalog_events_capability_is_advertised() {
15476 let handler = ControlHandler::new(Arc::new(Registry::default()));
15477 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15478 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15479 let reply = handler
15480 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
15481 .await
15482 .unwrap()
15483 .pop()
15484 .unwrap();
15485 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15486 serde_json::from_slice(&reply.body).unwrap()
15487 else {
15488 panic!("not a server.describe reply");
15489 };
15490 assert!(
15491 capabilities
15492 .iter()
15493 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
15494 "{capabilities:?}"
15495 );
15496 }
15497
15498 #[tokio::test]
15499 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
15500 let handler = ControlHandler::new(Arc::new(Registry::default()));
15501 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
15502 let ack = hello_via_sink(
15503 &handler,
15504 &module,
15505 &mut module_rx,
15506 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
15507 )
15508 .await;
15509 let ack = parse_ack(&ack);
15510 assert!(
15511 ack.subc_capabilities
15512 .iter()
15513 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15514 "{:?}",
15515 ack.subc_capabilities
15516 );
15517 assert!(ack
15518 .subc_capabilities
15519 .iter()
15520 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15521
15522 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
15523 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15524 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15525 let reply = handler
15526 .handle_control_frame(&client, frame)
15527 .await
15528 .unwrap()
15529 .pop()
15530 .unwrap();
15531 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15532 serde_json::from_slice(&reply.body).unwrap()
15533 else {
15534 panic!("not a server.describe reply");
15535 };
15536 assert!(
15537 capabilities
15538 .iter()
15539 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15540 "{capabilities:?}"
15541 );
15542 assert!(capabilities
15543 .iter()
15544 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15545 }
15546
15547 #[tokio::test]
15548 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
15549 let handler = ControlHandler::new(Arc::new(Registry::default()));
15550 let (module, mut rx) = route_ctx(ConnectionId::new(156));
15551 let ack = hello_via_sink(
15552 &handler,
15553 &module,
15554 &mut rx,
15555 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
15556 )
15557 .await;
15558 assert!(parse_ack(&ack)
15559 .subc_capabilities
15560 .iter()
15561 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15562 assert!(parse_ack(&ack)
15563 .subc_capabilities
15564 .iter()
15565 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15566 let (client, _rx) = route_ctx(ConnectionId::new(157));
15567 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15568 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15569 let reply = handler
15570 .handle_control_frame(&client, frame)
15571 .await
15572 .unwrap()
15573 .pop()
15574 .unwrap();
15575 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15576 serde_json::from_slice(&reply.body).unwrap()
15577 else {
15578 panic!("server.describe reply")
15579 };
15580 assert!(capabilities
15581 .iter()
15582 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15583 assert!(capabilities
15584 .iter()
15585 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15586 }
15587
15588 #[tokio::test]
15589 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
15590 let handler = ControlHandler::new(Arc::new(Registry::default()));
15591 let (module, mut rx) = route_ctx(ConnectionId::new(158));
15592 let ack = hello_via_sink(
15593 &handler,
15594 &module,
15595 &mut rx,
15596 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
15597 )
15598 .await;
15599 assert!(parse_ack(&ack)
15600 .subc_capabilities
15601 .iter()
15602 .any(|cap| cap == "scope-workdir/v1"));
15603 let (client, _rx) = route_ctx(ConnectionId::new(159));
15604 let frame = Frame::build(
15605 FrameType::Request,
15606 control_flags(),
15607 0,
15608 0,
15609 5,
15610 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15611 )
15612 .unwrap();
15613 let reply = handler
15614 .handle_control_frame(&client, frame)
15615 .await
15616 .unwrap()
15617 .remove(0);
15618 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15619 serde_json::from_slice(&reply.body).unwrap()
15620 else {
15621 panic!("server.describe reply")
15622 };
15623 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
15624 }
15625
15626 #[tokio::test]
15627 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
15628 let handler = ControlHandler::new(Arc::new(Registry::default()));
15629 let (module, mut rx) = route_ctx(ConnectionId::new(160));
15630 let ack = hello_via_sink(
15631 &handler,
15632 &module,
15633 &mut rx,
15634 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
15635 )
15636 .await;
15637 assert!(parse_ack(&ack)
15638 .subc_capabilities
15639 .iter()
15640 .any(|cap| cap == "worker-runs/v1"));
15641 let (client, _rx) = route_ctx(ConnectionId::new(161));
15642 let frame = Frame::build(
15643 FrameType::Request,
15644 control_flags(),
15645 0,
15646 0,
15647 5,
15648 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15649 )
15650 .unwrap();
15651 let reply = handler
15652 .handle_control_frame(&client, frame)
15653 .await
15654 .unwrap()
15655 .remove(0);
15656 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15657 serde_json::from_slice(&reply.body).unwrap()
15658 else {
15659 panic!("server.describe reply")
15660 };
15661 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
15662 }
15663
15664 #[tokio::test]
15665 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
15666 let handler = ControlHandler::new(Arc::new(Registry::default()));
15667 let (module, mut rx) = route_ctx(ConnectionId::new(154));
15668 let ack = hello_via_sink(
15669 &handler,
15670 &module,
15671 &mut rx,
15672 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
15673 )
15674 .await;
15675 assert!(parse_ack(&ack)
15676 .subc_capabilities
15677 .iter()
15678 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
15679 let (client, _rx) = route_ctx(ConnectionId::new(155));
15680 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15681 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15682 let reply = handler
15683 .handle_control_frame(&client, frame)
15684 .await
15685 .unwrap()
15686 .pop()
15687 .unwrap();
15688 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15689 serde_json::from_slice(&reply.body).unwrap()
15690 else {
15691 panic!("not a server.describe reply")
15692 };
15693 assert!(capabilities
15694 .iter()
15695 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
15696 }
15697
15698 #[tokio::test]
15699 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
15700 let registry = Arc::new(Registry::default());
15701 let forwarding = Arc::new(ForwardingTable::default());
15702 let handler =
15703 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15704 .with_health_probe_timeout(Duration::from_secs(5));
15705 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
15706 hello_via_sink(
15707 &handler,
15708 &module_ctx,
15709 &mut module_rx,
15710 non_routable_hello_frame_with_control_ops(
15711 "mcp",
15712 300,
15713 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15714 ),
15715 )
15716 .await;
15717 assert!(registry
15718 .get_module("mcp")
15719 .unwrap()
15720 .unwrap()
15721 .manifest
15722 .provides
15723 .is_empty());
15724
15725 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
15726 let route_response = handler
15727 .handle_control_frame(
15728 &route_client_ctx,
15729 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
15730 )
15731 .await
15732 .unwrap();
15733 assert_eq!(route_response[0].header.ty, FrameType::Error);
15734 assert_eq!(
15735 parse_error(&route_response[0])["code"],
15736 "target_unavailable"
15737 );
15738 assert!(parse_error(&route_response[0])["message"]
15739 .as_str()
15740 .unwrap()
15741 .contains("does not provide the requested target"));
15742 assert!(module_rx.try_recv().is_err());
15743
15744 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
15745 let health_handler = handler.clone();
15746 let health_task = tokio::spawn(async move {
15747 health_handler
15748 .handle_control_frame(
15749 &health_client_ctx,
15750 supervisor_health_probe_frame(302, "mcp"),
15751 )
15752 .await
15753 .unwrap()
15754 });
15755 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15756 .await
15757 .unwrap()
15758 .unwrap();
15759 assert_eq!(
15760 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
15761 ModuleControlRequest::HealthCheck {}
15762 );
15763 handler
15764 .handle_control_frame(
15765 &module_ctx,
15766 health_response(health_frame.header.corr, HealthStatus::Ok),
15767 )
15768 .await
15769 .unwrap();
15770 let health_response = health_task.await.unwrap();
15771 assert_eq!(health_response[0].header.ty, FrameType::Response);
15772 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
15773 ClientControlResponse::SupervisorHealthProbe {
15774 module_id, status, ..
15775 } => {
15776 assert_eq!(module_id, "mcp");
15777 assert_eq!(status, HealthStatus::Ok);
15778 }
15779 other => panic!("unexpected health response: {other:?}"),
15780 }
15781
15782 forwarding
15787 .cleanup_connection(module_ctx.connection_id)
15788 .unwrap();
15789 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
15790 let cleanup_response = tokio::time::timeout(
15791 Duration::from_millis(200),
15792 handler.handle_control_frame(
15793 &cleanup_probe_ctx,
15794 supervisor_health_probe_frame(303, "mcp"),
15795 ),
15796 )
15797 .await
15798 .expect("probe should fail immediately when the control lane is gone")
15799 .unwrap();
15800 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
15801 assert_eq!(
15802 parse_error(&cleanup_response[0])["code"],
15803 "target_unavailable"
15804 );
15805 assert!(parse_error(&cleanup_response[0])["message"]
15806 .as_str()
15807 .unwrap()
15808 .contains("no module connection"));
15809
15810 handler
15811 .cleanup_connection(module_ctx.connection_id)
15812 .unwrap();
15813 }
15814
15815 #[tokio::test]
15816 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
15817 let registry = Arc::new(Registry::default());
15818 let supervisor_handle = SupervisorHandle::new();
15819 let supervisor =
15820 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
15821 .with_handle(supervisor_handle.clone())
15822 .with_connection_file_path(
15823 std::env::temp_dir()
15824 .join(format!("subc-route-open-warming-{}", std::process::id())),
15825 );
15826 let module = supervisor
15827 .supervise_configured(
15828 ModuleSpec {
15829 carry_containment: false,
15830 module_id: "warming".to_string(),
15831 program: fake_aft_stub_path(),
15832 args: Vec::new(),
15833 env: Vec::new(),
15834 reserved: false,
15835 reserved_prefixes: Vec::new(),
15836 protocol: ModuleProtocol::Subc,
15837 overlap: Default::default(),
15838 },
15839 true,
15840 )
15841 .unwrap();
15842 assert_eq!(module.state().unwrap(), ModuleState::Running);
15843
15844 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15845 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
15846 let response = handler
15847 .handle_control_frame(
15848 &ctx,
15849 route_open_frame(304, "warming", unique_project_root("warming")),
15850 )
15851 .await
15852 .unwrap();
15853 module.stop().await.unwrap();
15854
15855 assert_eq!(response[0].header.ty, FrameType::Error);
15856 let error = parse_error(&response[0]);
15857 assert_eq!(error["code"], "module_warming");
15858 assert!(error["message"]
15859 .as_str()
15860 .unwrap()
15861 .contains("state=running, enabled=true, live=false"));
15862 }
15863
15864 #[test]
15865 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
15866 let handler = ControlHandler::new(Arc::new(Registry::default()));
15867 let capture = EventCapture::default();
15868 let _subscriber =
15869 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15870 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
15871 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
15872 let pending = (0..limit).collect::<Vec<_>>();
15873 let response = handler
15874 .route_open_capacity_refusal(
15875 &ctx,
15876 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
15877 "busy",
15878 pending.len(),
15879 limit,
15880 )
15881 .unwrap();
15882 assert_eq!(parse_error(&response)["code"], "target_unavailable");
15883 let event = capture
15884 .events()
15885 .into_iter()
15886 .find(|event| {
15887 event.target == "control"
15888 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
15889 })
15890 .expect("connection admission refusal event");
15891 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
15892 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
15893 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
15896 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
15897 }
15898
15899 #[test]
15900 fn route_open_target_cap_logs_admission_reason_and_capacity() {
15901 let handler = ControlHandler::new(Arc::new(Registry::default()));
15902 let capture = EventCapture::default();
15903 let _subscriber =
15904 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15905 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
15906 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
15907 let guards = (0..limit)
15908 .map(|_| {
15909 handler
15910 .route_bind_concurrency
15911 .try_admit("busy", limit)
15912 .unwrap()
15913 })
15914 .collect::<Vec<_>>();
15915 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
15916 Err(in_flight) => in_flight,
15917 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
15918 };
15919 let response = handler
15920 .route_open_target_capacity_refusal(
15921 &ctx,
15922 &RouteOpenCaller::unknown(),
15923 &route_open_frame(397, "busy", unique_project_root("target-cap")),
15924 "busy",
15925 in_flight,
15926 )
15927 .unwrap();
15928 assert_eq!(parse_error(&response)["code"], "target_unavailable");
15929 let event = capture
15930 .events()
15931 .into_iter()
15932 .find(|event| {
15933 event.target == "control"
15934 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
15935 })
15936 .expect("target admission refusal event");
15937 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
15938 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
15939 drop(guards);
15940 }
15941
15942 #[cfg(unix)]
15943 #[tokio::test(start_paused = true)]
15944 async fn supervisor_list_shows_a_failed_modules_next_retry() {
15945 let registry = Arc::new(Registry::default());
15946 let handle = SupervisorHandle::new();
15947 let supervisor =
15948 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
15949 .with_handle(handle.clone());
15950 let module = supervisor
15951 .spawn(crate::ModuleSpec {
15952 carry_containment: false,
15953 module_id: "retry-list".into(),
15954 program: PathBuf::from("/bin/sh"),
15955 args: vec!["-c".into(), "exit 17".into()],
15956 env: vec![],
15957 reserved: false,
15958 reserved_prefixes: vec![],
15959 protocol: ModuleProtocol::None,
15960 overlap: Default::default(),
15961 })
15962 .unwrap();
15963 let deadline = std::time::Instant::now() + Duration::from_secs(10);
15964 while module.state().unwrap() != crate::ModuleState::Failed {
15965 assert!(
15966 std::time::Instant::now() < deadline,
15967 "waiting for budget failure"
15968 );
15969 tokio::task::yield_now().await;
15970 }
15971 let handler = ControlHandler::new(registry).with_supervisor(handle);
15972 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
15973 let replies = handler.handle_supervisor_list(frame).await.unwrap();
15974 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
15975 assert_eq!(body["modules"][0]["state"], "failed");
15976 let retry = body["modules"][0]["next_retry_at_ms"]
15977 .as_u64()
15978 .expect("scheduled retry on the wire");
15979 let now = std::time::SystemTime::now()
15980 .duration_since(std::time::UNIX_EPOCH)
15981 .unwrap()
15982 .as_millis() as u64;
15983 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
15984 module.set_enabled(false).await.unwrap();
15985 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
15986 let replies = handler.handle_supervisor_list(frame).await.unwrap();
15987 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
15988 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
15989 }
15990
15991 #[tokio::test]
15992 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
15993 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
15994 let supervisor =
15995 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
15996 let module = supervisor
15997 .spawn(crate::ModuleSpec {
15998 carry_containment: false,
15999 module_id: "pid-read-budget".into(),
16000 program: std::env::current_exe().unwrap(),
16001 args: vec![
16002 "--exact".into(),
16003 "control::tests::pid_resource_read_fixture".into(),
16004 ],
16005 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16006 .into_iter()
16007 .map(|key| (key.into(), dir.display().to_string()))
16008 .chain(std::iter::once((
16009 "SUBC_SUBTREE_PID_FIXTURE".into(),
16010 "1".into(),
16011 )))
16012 .collect(),
16013 reserved: false,
16014 reserved_prefixes: Vec::new(),
16015 protocol: ModuleProtocol::None,
16016 overlap: Default::default(),
16017 })
16018 .unwrap();
16019 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16020 while !matches!(
16021 module.child_resource_usage(),
16022 subc_control::ChildResourceUsage::Measured(_)
16023 ) {
16024 assert!(
16025 tokio::time::Instant::now() < ready_deadline,
16026 "fixture process was not measurable"
16027 );
16028 tokio::time::sleep(Duration::from_millis(10)).await;
16029 }
16030 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
16031 let reading = {
16032 let _guard = crate::subtree_resources::fault_read_guard().await;
16033 crate::subtree_resources::read_process(module.clone()).await
16034 };
16035 stop_resource_fixture(&module, &dir).await;
16036 assert!(
16037 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
16038 "{reading:?}"
16039 );
16040 }
16041
16042 #[test]
16043 fn pid_resource_read_fixture() {
16044 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
16045 let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
16046 let deadline = std::time::Instant::now() + Duration::from_secs(60);
16047 while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
16048 std::thread::sleep(Duration::from_millis(5));
16049 }
16050 std::fs::write(dir.join("module-stopped"), "").unwrap();
16051 }
16052 }
16053
16054 async fn stop_resource_fixture(
16055 module: &crate::supervise::SupervisedModule,
16056 dir: &std::path::Path,
16057 ) {
16058 let stopping = module.stop();
16059 tokio::pin!(stopping);
16060 tokio::select! {
16061 result = &mut stopping => result.unwrap(),
16062 () = async {
16063 while module.state().unwrap() != crate::supervise::ModuleState::Draining {
16064 tokio::time::sleep(Duration::from_millis(5)).await;
16065 }
16066 std::fs::write(dir.join("stop-module"), "").unwrap();
16067 } => stopping.await.unwrap(),
16068 }
16069 #[cfg(windows)]
16070 assert!(
16071 dir.join("module-stopped").exists(),
16072 "the resource fixture must exit cooperatively, not wait for the drain kill"
16073 );
16074 assert_eq!(
16075 module.state().unwrap(),
16076 crate::supervise::ModuleState::Stopped
16077 );
16078 }
16079
16080 #[tokio::test]
16081 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
16082 let registry = Arc::new(Registry::default());
16083 let handle = SupervisorHandle::new();
16084 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16085 .with_handle(handle.clone());
16086 supervisor
16087 .supervise_configured(
16088 crate::ModuleSpec {
16089 module_id: "disabled-resource-reading".into(),
16090 program: std::env::current_exe().unwrap(),
16091 args: Vec::new(),
16092 env: Vec::new(),
16093 reserved: false,
16094 reserved_prefixes: Vec::new(),
16095 protocol: ModuleProtocol::None,
16096 overlap: Default::default(),
16097 carry_containment: false,
16098 },
16099 false,
16100 )
16101 .unwrap();
16102 let handler = ControlHandler::new(registry).with_supervisor(handle);
16103 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16104 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16105 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16106 assert!(body["modules"][0].get("tree").is_none());
16107 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
16108 }
16109
16110 #[tokio::test]
16111 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
16112 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
16113 let registry = Arc::new(Registry::default());
16114 let handle = SupervisorHandle::new();
16115 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16116 .with_handle(handle.clone());
16117 let module = supervisor
16118 .spawn(crate::ModuleSpec {
16119 carry_containment: false,
16120 module_id: "list-process-priority".into(),
16121 program: std::env::current_exe().unwrap(),
16122 args: vec![
16123 "--exact".into(),
16124 "control::tests::pid_resource_read_fixture".into(),
16125 ],
16126 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16127 .into_iter()
16128 .map(|key| (key.into(), dir.display().to_string()))
16129 .chain(std::iter::once((
16130 "SUBC_SUBTREE_PID_FIXTURE".into(),
16131 "1".into(),
16132 )))
16133 .collect(),
16134 reserved: false,
16135 reserved_prefixes: Vec::new(),
16136 protocol: ModuleProtocol::None,
16137 overlap: Default::default(),
16138 })
16139 .unwrap();
16140 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16141 loop {
16142 let image = module.running_image_agreement().await;
16143 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
16144 break;
16145 }
16146 assert!(
16147 tokio::time::Instant::now() < ready_deadline,
16148 "the fixture process never became observable: {image:?}"
16149 );
16150 tokio::time::sleep(Duration::from_millis(10)).await;
16151 }
16152 let guard = crate::subtree_resources::fault_read_guard().await;
16153 let (target, release, _) = crate::subtree_resources::held();
16154 module.set_subtree_target_for_test(target);
16155 let handler = ControlHandler::new(registry).with_supervisor(handle);
16156 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16157 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16158 drop(release);
16159 drop(guard);
16160 stop_resource_fixture(&module, &dir).await;
16161 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16162 assert_eq!(
16163 body["modules"][0]["resources"]["status"], "measured",
16164 "the optional tree timeout must not discard the process reading: {body}"
16165 );
16166 assert!(body["modules"][0].get("tree").is_none());
16167 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
16168 assert_eq!(
16169 body["modules"][0]["pending_reload"]["image"]["status"], "match",
16170 "the tree timeout must not hide a completed image observation"
16171 );
16172 }
16173
16174 #[cfg(any(target_os = "linux", target_os = "macos"))]
16175 #[tokio::test]
16176 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
16177 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
16178 .with_daemon_provenance(
16179 std::process::id(),
16180 0,
16181 Some(std::env::current_exe().unwrap()),
16182 None,
16183 None,
16184 );
16185 let (probe, release, began) = ExecutableIdentityProbe::held();
16186 handler.daemon_provenance.probe = probe;
16187 let observe = |handler: ControlHandler, corr| {
16188 tokio::spawn(async move {
16189 let (ctx, _) = route_ctx(ConnectionId::new(corr));
16190 let frame =
16191 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
16192 .unwrap();
16193 handler
16194 .handle_supervisor_provenance(&ctx, frame, None)
16195 .await
16196 .unwrap()
16197 })
16198 };
16199 let explicit = observe(handler.clone(), 114);
16200 began.await.unwrap();
16201 let concurrent = observe(handler.clone(), 115);
16202 tokio::time::sleep(Duration::from_millis(300)).await;
16205 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16206 let list = tokio::time::timeout(
16207 Duration::from_secs(1),
16208 handler.handle_supervisor_list(frame),
16209 )
16210 .await;
16211 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
16212 drop(release);
16213 let replies = explicit.await.unwrap();
16214 let concurrent_replies = concurrent.await.unwrap();
16215 assert!(
16216 explicit_was_pending,
16217 "explicit provenance must await the full observation"
16218 );
16219 let list = list
16220 .expect("a concurrent explicit probe must not stall list")
16221 .unwrap();
16222 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
16223 assert_eq!(list_body["op"], "supervisor.list");
16224 for replies in [replies, concurrent_replies] {
16225 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16226 assert_eq!(
16227 body["daemon"]["daemon_observed"]["running_image"]["status"],
16228 "match"
16229 );
16230 }
16231 }
16232
16233 #[tokio::test]
16234 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
16235 let registry = Arc::new(Registry::default());
16236 let handle = SupervisorHandle::new();
16237 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16238 .with_handle(handle.clone());
16239 let module = supervisor
16240 .supervise_configured(
16241 crate::ModuleSpec {
16242 carry_containment: false,
16243 module_id: "memory-read".into(),
16244 program: PathBuf::from("disabled-module"),
16245 args: Vec::new(),
16246 env: Vec::new(),
16247 reserved: false,
16248 reserved_prefixes: Vec::new(),
16249 protocol: ModuleProtocol::Subc,
16250 overlap: Default::default(),
16251 },
16252 true,
16253 )
16254 .unwrap();
16255 let _guard = crate::subtree_resources::fault_read_guard().await;
16256 let (target, release, began) = crate::subtree_resources::held();
16257 let stalled_target = target.clone();
16260 let stalled = tokio::task::spawn_blocking(move || {
16261 stalled_target.tree(&subc_os::ProcessTable::default())
16262 });
16263 began.await.unwrap();
16264 module.set_subtree_target_for_test(target);
16265 let handler = ControlHandler::new(registry).with_supervisor(handle);
16266 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
16267 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16268 let replies = handler
16269 .handle_client_control_request(
16270 &ctx,
16271 frame,
16272 ClientControlRequest::SupervisorList {},
16273 &mut None,
16274 )
16275 .await
16276 .unwrap();
16277 assert!(
16278 replies.is_empty(),
16279 "list reply must be detached from the connection reader"
16280 );
16281 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16282 let replies = handler
16283 .handle_client_control_request(
16284 &ctx,
16285 frame,
16286 ClientControlRequest::SupervisorSpawnSnapshot {},
16287 &mut None,
16288 )
16289 .await
16290 .unwrap();
16291 assert_eq!(
16292 replies.len(),
16293 1,
16294 "another request must complete while the subtree read stalls"
16295 );
16296 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16297 .await
16298 .unwrap()
16299 .unwrap();
16300 release.send(()).unwrap();
16301 drop(release);
16302 stalled.await.unwrap();
16303 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
16304 assert!(
16305 body["modules"][0].get("tree").is_none(),
16306 "an unknown tree reading must be omitted after the bounded read expires"
16307 );
16308 handler
16312 .resource_tracker
16313 .read(handler.supervisor.list(), false, "released_tree_fixture")
16314 .await;
16315 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16316 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16317 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16318 assert_eq!(
16319 body["modules"][0]["tree"]["source"], "process_tree",
16320 "the next request must export the completed refresh"
16321 );
16322 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
16323 }
16324
16325 #[tokio::test]
16326 async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
16327 use subc_control::{
16328 AgentCommandResources, ChildMemoryKind, ChildResourceReading,
16329 ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
16330 TreeReading, TreeSource,
16331 };
16332 let registry = Arc::new(Registry::default());
16333 let handle = SupervisorHandle::new();
16334 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16335 .with_handle(handle.clone());
16336 supervisor
16337 .supervise_configured(
16338 crate::ModuleSpec {
16339 carry_containment: false,
16340 module_id: "budget-worker".into(),
16341 program: PathBuf::from("disabled-module"),
16342 args: Vec::new(),
16343 env: Vec::new(),
16344 reserved: false,
16345 reserved_prefixes: Vec::new(),
16346 protocol: ModuleProtocol::Subc,
16347 overlap: Default::default(),
16348 },
16349 true,
16350 )
16351 .unwrap();
16352 let handler = ControlHandler::new(registry).with_supervisor(handle);
16353 let now = u64::try_from(
16354 std::time::SystemTime::now()
16355 .duration_since(std::time::UNIX_EPOCH)
16356 .unwrap()
16357 .as_millis(),
16358 )
16359 .unwrap();
16360 let measured = now - 1_000;
16361 let hour = CpuLastHour::new(100, 60_000);
16362 let daemon = DaemonResources::new(7, 1, "test")
16363 .with_measured_at_ms(Some(measured))
16364 .with_resources(Some(ChildResourceUsage::Measured(
16365 ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
16366 )))
16367 .with_cpu_last_hour(Some(hour.clone()));
16368 let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
16369 .with_measured_at_ms(Some(measured))
16370 .with_cpu_last_hour(Some(hour.clone()));
16371 let tree = TreeReading::new(
16372 150,
16373 50,
16374 ChildMemoryKind::ResidentSet,
16375 1,
16376 TreeSource::ProcessTree,
16377 )
16378 .with_measured_at_ms(Some(measured));
16379 handler.resource_tracker.seed_committed_for_test(
16380 &handler.supervisor.list(),
16381 crate::resource_totals::Readings {
16382 modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
16383 daemon: Some(daemon),
16384 agent_commands: Some(vec![agents]),
16385 ..Default::default()
16386 },
16387 now,
16388 );
16389 list_diagnostics::scope(async {
16390 tokio::time::sleep(Duration::from_millis(110)).await;
16391 assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
16392 let request =
16393 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16394 let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
16395 let ClientControlResponse::SupervisorList {
16396 modules,
16397 daemon,
16398 agent_commands,
16399 ..
16400 } = serde_json::from_slice(&replies[0].body).unwrap()
16401 else {
16402 panic!("expected a list response")
16403 };
16404 assert!(matches!(
16405 modules[0].resources,
16406 Some(ChildResourceUsage::Unavailable {
16407 reason: ChildResourceUnavailableReason::Unreadable
16408 })
16409 ));
16410 assert_eq!(
16411 (
16412 modules[0].tree.as_ref().unwrap().cpu_ms,
16413 modules[0].tree.as_ref().unwrap().measured_at_ms
16414 ),
16415 (150, Some(measured))
16416 );
16417 let daemon = daemon.unwrap();
16418 let agents = agent_commands.unwrap();
16419 assert_eq!(daemon.measured_at_ms, Some(measured));
16420 assert_eq!(
16421 (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
16422 (1, 50, Some(measured))
16423 );
16424 for hour in [
16425 modules[0].cpu_last_hour.as_ref(),
16426 daemon.cpu_last_hour.as_ref(),
16427 agents[0].cpu_last_hour.as_ref(),
16428 ] {
16429 let hour = hour.unwrap();
16430 assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
16431 }
16432 })
16433 .await;
16434 }
16435
16436 #[tokio::test]
16437 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
16438 let registry = Arc::new(Registry::default());
16439 let handle = SupervisorHandle::new();
16440 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16441 .with_handle(handle.clone());
16442 supervisor
16443 .supervise_configured(
16444 crate::ModuleSpec {
16445 carry_containment: false,
16446 module_id: "drift".into(),
16447 program: PathBuf::from("disabled-module"),
16448 args: Vec::new(),
16449 env: Vec::new(),
16450 reserved: false,
16451 reserved_prefixes: Vec::new(),
16452 protocol: ModuleProtocol::Subc,
16453 overlap: Default::default(),
16454 },
16455 false,
16456 )
16457 .unwrap();
16458 let handler = ControlHandler::new(registry).with_supervisor(handle);
16459 for configured in [None, Some(false), Some(true)] {
16460 handler.capability_evaluator.configure(
16461 configured.map(|enabled| ("drift".to_string(), enabled)),
16462 BTreeMap::new(),
16463 );
16464 let frame =
16465 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16466 let replies = handler
16467 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
16468 .await
16469 .unwrap();
16470 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16471 assert!(body.get("configured_enabled").is_none());
16472 assert_eq!(body["op"], "supervisor.ack");
16473 let frame =
16474 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16475 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16476 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16477 assert_eq!(
16478 body["modules"][0].get("configured_enabled"),
16479 configured.map(Value::Bool).as_ref()
16480 );
16481 assert!(body["modules"][0].get("tree").is_none());
16482 assert!(body["modules"][0].get("subtree").is_none());
16483 }
16484 }
16485
16486 #[tokio::test(flavor = "current_thread")]
16487 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
16488 let handler = ControlHandler::new(Arc::new(Registry::default()));
16489 let capture = EventCapture::default();
16490 let _subscriber =
16491 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16492 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
16493 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16494
16495 let _ = handler
16496 .handle_client_control_request(
16497 &ctx,
16498 frame,
16499 ClientControlRequest::SupervisorSetEnabled {
16500 module_id: "broca".to_string(),
16501 enabled: false,
16502 },
16503 &mut None,
16504 )
16505 .await
16506 .unwrap();
16507
16508 let events = capture
16509 .events()
16510 .into_iter()
16511 .filter(|event| {
16512 event.target == "control"
16513 && event.fields.get("message").map(String::as_str)
16514 == Some("supervisor request received")
16515 })
16516 .collect::<Vec<_>>();
16517 assert_eq!(events.len(), 1, "one supervisor request log line");
16518 let event = &events[0];
16519 assert_eq!(event.level, tracing::Level::INFO);
16520 assert_eq!(
16521 event.fields.get("op"),
16522 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
16523 );
16524 assert_eq!(
16525 event.fields.get("module_id"),
16526 Some(&"\"broca\"".to_string())
16527 );
16528 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
16529 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
16530 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
16531 }
16532
16533 #[tokio::test(flavor = "current_thread")]
16534 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
16535 let registry = Arc::new(Registry::default());
16536 let module_connection = ConnectionId::new(303);
16537 registry
16538 .register_with_control_ops(
16539 manifest("aft", PROTOCOL_VERSION),
16540 PROTOCOL_VERSION,
16541 module_connection,
16542 module_baseline_control_ops(),
16543 )
16544 .unwrap();
16545 let handler = ControlHandler::new(registry);
16546 let capture = EventCapture::default();
16547 let _subscriber =
16548 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16549 let (ctx, _rx) = route_ctx(module_connection);
16550 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16551
16552 let _ = handler
16553 .handle_client_control_request(
16554 &ctx,
16555 frame,
16556 ClientControlRequest::SupervisorSetEnabled {
16557 module_id: "broca".to_string(),
16558 enabled: false,
16559 },
16560 &mut None,
16561 )
16562 .await
16563 .unwrap();
16564
16565 let callers = capture
16566 .events()
16567 .into_iter()
16568 .filter(|event| {
16569 event.fields.get("message").map(String::as_str)
16570 == Some("supervisor request received")
16571 })
16572 .map(|event| event.fields.get("caller").cloned())
16573 .collect::<Vec<_>>();
16574 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
16575 }
16576
16577 #[tokio::test(flavor = "current_thread")]
16578 async fn supervisor_list_does_not_log_request_received() {
16579 let handler = ControlHandler::new(Arc::new(Registry::default()));
16580 let capture = EventCapture::default();
16581 let _subscriber =
16582 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16583 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
16584 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16585
16586 handler
16587 .handle_client_control_request(
16588 &ctx,
16589 frame,
16590 ClientControlRequest::SupervisorList {},
16591 &mut None,
16592 )
16593 .await
16594 .unwrap();
16595
16596 assert!(capture.events().into_iter().all(|event| {
16597 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
16598 }));
16599 }
16600
16601 #[tokio::test(flavor = "current_thread")]
16602 async fn supervisor_rescan_preview_does_not_log_request_received() {
16603 let handler = ControlHandler::new(Arc::new(Registry::default()));
16604 let capture = EventCapture::default();
16605 let _subscriber =
16606 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16607 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
16608 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16609
16610 handler
16611 .handle_client_control_request(
16612 &ctx,
16613 frame,
16614 ClientControlRequest::SupervisorRescan { preview: true },
16615 &mut None,
16616 )
16617 .await
16618 .unwrap();
16619
16620 assert!(capture.events().into_iter().all(|event| {
16621 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
16622 }));
16623 }
16624
16625 struct ApproveOperator;
16626 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
16627 fn prompt(
16628 &self,
16629 _: &str,
16630 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
16631 ) -> crate::operator_confirm::ProviderResult {
16632 crate::operator_confirm::ProviderResult::Approved
16633 }
16634 }
16635
16636 #[tokio::test]
16637 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
16638 assert_eq!(
16639 ControlHandler::default().privileged_operator_prompt,
16640 PrivilegedOperatorPrompt::Log
16641 );
16642 for (initial, changed, running) in [
16643 ("log", "enforce", PrivilegedOperatorPrompt::Log),
16644 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
16645 ] {
16646 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
16647 let path = dir.join("subc.jsonc");
16648 std::fs::write(
16649 &path,
16650 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
16651 )
16652 .unwrap();
16653 let config = crate::daemon_config::load(&path).unwrap().unwrap();
16654 let registry = Arc::new(Registry::default());
16655 let handle = SupervisorHandle::new();
16656 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
16657 .with_handle(handle.clone());
16658 let handler = ControlHandler::new(registry)
16659 .with_supervisor(handle)
16660 .with_privileged_operator_prompt(config.privileged_operator_prompt)
16661 .with_supervisor_rescan(supervisor, &path, config.port);
16662 handler.forwarding.operator_confirms().configure(
16663 Arc::new(ApproveOperator),
16664 crate::operator_confirm::OperatorLimits::default(),
16665 );
16666 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
16667
16668 for (disk_mode, needs_restart) in [
16669 (Some(initial), false),
16670 (Some(changed), true),
16671 (Some(changed), true),
16672 (None, running == PrivilegedOperatorPrompt::Enforce),
16673 (Some(initial), false),
16674 ] {
16675 let mut doc = json!({"version":1});
16676 if let Some(mode) = disk_mode {
16677 doc["privileged_operator_prompt"] = json!(mode);
16678 }
16679 std::fs::write(&path, doc.to_string()).unwrap();
16680 for preview in [true, false] {
16681 let request = ClientControlRequest::SupervisorRescan { preview };
16682 let frame = Frame::build(
16683 FrameType::Request,
16684 control_flags(),
16685 0,
16686 0,
16687 1,
16688 serde_json::to_vec(&request).unwrap(),
16689 )
16690 .unwrap();
16691 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
16692 assert_eq!(frames.len(), 1);
16693 let ClientControlResponse::SupervisorRescan { result } =
16694 serde_json::from_slice(&frames[0].body).unwrap()
16695 else {
16696 panic!("expected a rescan response");
16697 };
16698 let expected = if needs_restart {
16699 vec!["privileged_operator_prompt"]
16700 } else {
16701 Vec::new()
16702 };
16703 assert_eq!(result.restart_required, expected);
16704 assert_eq!(handler.privileged_operator_prompt, running);
16705 assert_eq!(
16706 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
16707 running
16708 );
16709 }
16710 }
16711 }
16712 }
16713
16714 #[tokio::test]
16715 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
16716 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
16717 let path = dir.join("subc.jsonc");
16718 let registry = Arc::new(Registry::default());
16719 let handle = SupervisorHandle::new();
16720 let supervisor =
16721 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
16722 let handler = ControlHandler::new(registry)
16723 .with_supervisor(handle)
16724 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
16725 .with_supervisor_rescan(supervisor, &path, None);
16726 handler.forwarding.operator_confirms().configure(
16727 Arc::new(ApproveOperator),
16728 crate::operator_confirm::OperatorLimits::default(),
16729 );
16730 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
16731 for preview in [true, false] {
16732 std::fs::write(
16733 &path,
16734 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
16735 )
16736 .unwrap();
16737 let request = ClientControlRequest::SupervisorRescan { preview };
16738 let frame = Frame::build(
16739 FrameType::Request,
16740 control_flags(),
16741 0,
16742 0,
16743 1,
16744 serde_json::to_vec(&request).unwrap(),
16745 )
16746 .unwrap();
16747 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
16748 assert_eq!(frames.len(), 1);
16749 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
16750 assert_eq!(
16751 handler.privileged_operator_prompt,
16752 PrivilegedOperatorPrompt::Enforce
16753 );
16754 }
16755 }
16756
16757 #[tokio::test]
16762 async fn route_open_refusal_names_the_check_that_refused() {
16763 let handler = ControlHandler::new(Arc::new(Registry::default()));
16764 let capture = EventCapture::default();
16765 let _subscriber =
16766 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16767 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
16768 let response = handler
16769 .handle_control_frame(
16770 &ctx,
16771 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
16772 )
16773 .await
16774 .unwrap();
16775
16776 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
16777 let event = capture
16778 .events()
16779 .into_iter()
16780 .find(|event| {
16781 event.target == "control"
16782 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
16783 })
16784 .expect("route.open refusal event");
16785 assert_eq!(
16786 event.fields.get("reason"),
16787 Some(&"\"not_registered\"".to_string())
16788 );
16789 }
16790
16791 #[tokio::test]
16792 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
16793 let registry = Arc::new(Registry::default());
16794 let supervisor_handle = SupervisorHandle::new();
16795 let supervisor =
16796 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
16797 .with_handle(supervisor_handle.clone())
16798 .with_connection_file_path(std::env::temp_dir().join(format!(
16799 "subc-route-open-refusal-info-{}",
16800 std::process::id()
16801 )));
16802 let module = supervisor
16803 .supervise_configured(
16804 ModuleSpec {
16805 carry_containment: false,
16806 module_id: "warming".to_string(),
16807 program: fake_aft_stub_path(),
16808 args: Vec::new(),
16809 env: Vec::new(),
16810 reserved: false,
16811 reserved_prefixes: Vec::new(),
16812 protocol: ModuleProtocol::Subc,
16813 overlap: Default::default(),
16814 },
16815 true,
16816 )
16817 .unwrap();
16818 assert_eq!(module.state().unwrap(), ModuleState::Running);
16819
16820 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16821 assert!(handler
16822 .counters()
16823 .snapshot()
16824 .get("route_open_refused_by_code")
16825 .is_none());
16826 let capture = EventCapture::default();
16827 let _subscriber =
16828 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16829 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
16830 let response = handler
16831 .handle_control_frame(
16832 &ctx,
16833 route_open_frame(394, "warming", unique_project_root("refusal-info")),
16834 )
16835 .await
16836 .unwrap();
16837 module.stop().await.unwrap();
16838
16839 assert_eq!(parse_error(&response[0])["code"], "module_warming");
16840 let event = capture
16841 .events()
16842 .into_iter()
16843 .find(|event| {
16844 event.target == "control"
16845 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
16846 })
16847 .expect("route.open refusal event");
16848 assert_eq!(
16849 event.fields.get("module_id"),
16850 Some(&"\"warming\"".to_string())
16851 );
16852 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
16853 assert_eq!(
16854 event.fields.get("reason"),
16855 Some(&"\"supervised_not_registered\"".to_string())
16856 );
16857 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
16858 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
16859 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
16860 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
16861 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
16862 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
16863 assert_eq!(
16864 handler.counters().snapshot()["route_open_refused_by_code"],
16865 json!({ "module_warming": 1 })
16866 );
16867 }
16868
16869 const OUTAGE_START: &str = "route.open refusing module: not serving";
16870 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
16871
16872 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
16873 capture
16874 .events()
16875 .into_iter()
16876 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
16877 .collect()
16878 }
16879
16880 fn supervise_stub(
16881 registry: &Arc<Registry>,
16882 module_id: &str,
16883 enabled: bool,
16884 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
16885 let supervisor_handle = SupervisorHandle::new();
16886 let supervisor =
16887 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
16888 .with_handle(supervisor_handle.clone())
16889 .with_connection_file_path(std::env::temp_dir().join(format!(
16890 "subc-route-outage-{module_id}-{}",
16891 std::process::id()
16892 )));
16893 let module = supervisor
16894 .supervise_configured(
16895 ModuleSpec {
16896 carry_containment: false,
16897 module_id: module_id.to_string(),
16898 program: fake_aft_stub_path(),
16899 args: Vec::new(),
16900 env: Vec::new(),
16901 reserved: false,
16902 reserved_prefixes: Vec::new(),
16903 protocol: ModuleProtocol::Subc,
16904 overlap: Default::default(),
16905 },
16906 enabled,
16907 )
16908 .unwrap();
16909 (supervisor_handle, module)
16910 }
16911
16912 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
16913 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
16914 module_id: module_id.to_string(),
16915 drain_timeout_ms: Some(50),
16916 })
16917 .unwrap();
16918 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
16919 }
16920
16921 #[test]
16925 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
16926 let registry = Arc::new(Registry::default());
16927 let forwarding = Arc::new(ForwardingTable::default());
16928 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16929 let second = ControlHandler::with_forwarding(registry, forwarding);
16930 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
16931 }
16932
16933 #[tokio::test(flavor = "current_thread")]
16937 async fn route_open_unknown_module_refusals_add_no_outage_state() {
16938 let handler = ControlHandler::new(Arc::new(Registry::default()));
16939 let capture = EventCapture::default();
16940 let _subscriber =
16941 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16942 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
16943 for corr in 0..8 {
16944 let response = handler
16945 .handle_control_frame(
16946 &ctx,
16947 route_open_frame(
16948 380 + corr,
16949 &format!("nobody-{corr}"),
16950 unique_project_root("outage-unknown"),
16951 ),
16952 )
16953 .await
16954 .unwrap();
16955 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
16956 }
16957
16958 assert_eq!(handler.route_outages.tracked_module_count(), 0);
16959 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
16960 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
16961 }
16962
16963 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16968 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
16969 let registry = Arc::new(Registry::default());
16970 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
16971 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16972 let capture = EventCapture::default();
16973 let _subscriber =
16974 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16975 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
16976 handler.route_outages.record_accepted("outage-restart");
16979
16980 let response = handler
16981 .handle_control_frame(
16982 &ctx,
16983 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
16984 )
16985 .await
16986 .unwrap();
16987 assert_eq!(response[0].header.ty, FrameType::Error);
16988 let starts = outage_lines(&capture, OUTAGE_START);
16989 assert_eq!(starts.len(), 1, "{starts:?}");
16990 assert_eq!(starts[0].level, tracing::Level::WARN);
16991 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
16992 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
16993 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
16994 handler.route_outages.record_accepted("outage-restart");
16995 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
16996
16997 let restart = handler
16998 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
16999 .await
17000 .unwrap();
17001 assert_eq!(
17002 restart[0].header.ty,
17003 FrameType::Response,
17004 "{:?}",
17005 parse_error(&restart[0])
17006 );
17007 handler
17008 .handle_control_frame(
17009 &ctx,
17010 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
17011 )
17012 .await
17013 .unwrap();
17014 module.stop().await.unwrap();
17015
17016 let starts = outage_lines(&capture, OUTAGE_START);
17017 assert_eq!(starts.len(), 2, "{starts:?}");
17018 assert_eq!(starts[1].level, tracing::Level::INFO);
17019 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
17020 }
17021
17022 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17026 async fn failed_operator_restart_leaves_no_operator_mark() {
17027 let registry = Arc::new(Registry::default());
17028 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
17029 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17030 let capture = EventCapture::default();
17031 let _subscriber =
17032 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17033 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
17034 handler.route_outages.record_accepted("outage-disabled");
17035
17036 let restart = handler
17037 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
17038 .await
17039 .unwrap();
17040 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
17041 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
17042
17043 handler
17044 .handle_control_frame(
17045 &ctx,
17046 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
17047 )
17048 .await
17049 .unwrap();
17050 let starts = outage_lines(&capture, OUTAGE_START);
17051 assert_eq!(starts.len(), 1, "{starts:?}");
17052 assert_eq!(starts[0].level, tracing::Level::WARN);
17053 }
17054
17055 #[tokio::test(flavor = "current_thread")]
17056 async fn route_open_unknown_module_escapes_target_module_id() {
17057 let handler = ControlHandler::new(Arc::new(Registry::default()));
17058 let capture = EventCapture::default();
17059 let _subscriber =
17060 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17061 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
17062 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17063 let response = handler
17064 .handle_control_frame(
17065 &ctx,
17066 route_open_frame(
17067 395,
17068 hostile_module_id,
17069 unique_project_root("hostile-target-module-id"),
17070 ),
17071 )
17072 .await
17073 .unwrap();
17074
17075 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17076 let event = capture
17077 .events()
17078 .into_iter()
17079 .find(|event| {
17080 event.target == "control"
17081 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17082 })
17083 .expect("route.open unknown-module refusal event");
17084 let logged = event.fields.get("module_id").expect("module_id field");
17085 assert!(!logged.bytes().any(|byte| byte < 0x20));
17086 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17087 }
17088
17089 #[tokio::test(flavor = "current_thread")]
17092 async fn route_open_refusal_carries_daemon_origin() {
17093 let handler = ControlHandler::new(Arc::new(Registry::default()));
17094 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17095 let response = handler
17096 .handle_control_frame(
17097 &ctx,
17098 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
17099 )
17100 .await
17101 .unwrap();
17102
17103 assert_eq!(response.len(), 1);
17104 assert_eq!(response[0].header.ty, FrameType::Error);
17105 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17106 assert!(
17107 response[0].header.flags.is_daemon_origin(),
17108 "route.open refusal must carry daemon origin: {:?}",
17109 response[0].header.flags
17110 );
17111 }
17112
17113 #[tokio::test(flavor = "current_thread")]
17117 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
17118 let registry = Arc::new(Registry::default());
17119 let forwarding = Arc::new(ForwardingTable::default());
17120 let handler =
17121 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17122 let module_connection = ConnectionId::new(95);
17123 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17124 hello_via_sink(
17125 &handler,
17126 &module_ctx,
17127 &mut module_rx,
17128 hello_frame("aft", PROTOCOL_VERSION, 395),
17129 )
17130 .await;
17131 forwarding
17132 .begin_module_drain("aft", RouteCloseReason::Restart)
17133 .unwrap()
17134 .expect("the module endpoint drains");
17135
17136 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
17137 let response = handler
17138 .handle_control_frame(
17139 &client_ctx,
17140 route_open_frame(396, "aft", unique_project_root("drain-origin")),
17141 )
17142 .await
17143 .unwrap();
17144
17145 assert_eq!(response.len(), 1);
17146 assert_eq!(response[0].header.ty, FrameType::Error);
17147 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17148 assert!(
17149 response[0].header.flags.is_daemon_origin(),
17150 "drain module_reloading must carry daemon origin: {:?}",
17151 response[0].header.flags
17152 );
17153 }
17154
17155 #[tokio::test(flavor = "current_thread")]
17159 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
17160 let registry = Arc::new(Registry::default());
17161 let forwarding = Arc::new(ForwardingTable::default());
17162 let handler =
17163 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17164 let module_connection = ConnectionId::new(95);
17165 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17166 hello_via_sink(
17167 &handler,
17168 &module_ctx,
17169 &mut module_rx,
17170 hello_frame("aft", PROTOCOL_VERSION, 395),
17171 )
17172 .await;
17173
17174 let client_connection = ConnectionId::new(96);
17175 let (client_ctx, _client_rx) = route_ctx(client_connection);
17176 let (route_task, bind) = relay_route_open(
17177 &handler,
17178 client_connection,
17179 &client_ctx.egress,
17180 &mut module_rx,
17181 396,
17182 "aft",
17183 "spoofed-reloading",
17184 )
17185 .await;
17186 let rejection = Frame::build(
17189 FrameType::Error,
17190 crate::router::daemon_error_flags(),
17191 0,
17192 0,
17193 bind.header.corr,
17194 serde_json::to_vec(&ErrorBody::new(
17195 "module_reloading",
17196 "module refused route.bind",
17197 ))
17198 .unwrap(),
17199 )
17200 .unwrap();
17201 handler
17202 .handle_control_frame(&module_ctx, rejection)
17203 .await
17204 .unwrap();
17205
17206 let response = route_task.await.unwrap();
17207 assert_eq!(response.len(), 1);
17208 assert_eq!(response[0].header.ty, FrameType::Error);
17209 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17210 assert!(
17211 !response[0].header.flags.is_daemon_origin(),
17212 "a module's own rejection must not carry daemon origin: {:?}",
17213 response[0].header.flags
17214 );
17215 }
17216
17217 #[tokio::test(flavor = "current_thread")]
17218 async fn route_open_module_rejection_uses_daemon_counter_key() {
17219 let registry = Arc::new(Registry::default());
17220 let forwarding = Arc::new(ForwardingTable::default());
17221 let handler =
17222 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17223 let module_connection = ConnectionId::new(95);
17224 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17225 hello_via_sink(
17226 &handler,
17227 &module_ctx,
17228 &mut module_rx,
17229 hello_frame("aft", PROTOCOL_VERSION, 395),
17230 )
17231 .await;
17232
17233 let client_connection = ConnectionId::new(96);
17234 let (client_ctx, _client_rx) = route_ctx(client_connection);
17235 let capture = EventCapture::default();
17236 static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
17243 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
17244 let _subscriber =
17245 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17246 let (route_task, bind) = relay_route_open(
17247 &handler,
17248 client_connection,
17249 &client_ctx.egress,
17250 &mut module_rx,
17251 396,
17252 "aft",
17253 "hostile-module-code",
17254 )
17255 .await;
17256 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
17257 let rejection = Frame::build(
17258 FrameType::Error,
17259 control_flags(),
17260 0,
17261 0,
17262 bind.header.corr,
17263 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
17264 )
17265 .unwrap();
17266 handler
17267 .handle_control_frame(&module_ctx, rejection)
17268 .await
17269 .unwrap();
17270
17271 let response = route_task.await.unwrap();
17272 assert_eq!(parse_error(&response[0])["code"], hostile_code);
17273 let counters = handler.counters().snapshot();
17274 assert_eq!(
17275 counters["route_open_refused_by_code"],
17276 json!({ "module_rejected": 1 })
17277 );
17278 assert!(counters["route_open_refused_by_code"]
17279 .get(hostile_code)
17280 .is_none());
17281
17282 let event = capture
17283 .events()
17284 .into_iter()
17285 .find(|event| {
17286 event.target == "control"
17287 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
17288 })
17289 .expect("route.open module-rejection refusal event");
17290 let logged = event.fields.get("module_code").expect("module_code field");
17291 assert!(!logged.bytes().any(|byte| byte < 0x20));
17292 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17293 }
17294
17295 #[tokio::test]
17296 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
17297 let registry = Arc::new(Registry::default());
17298 let supervisor_handle = SupervisorHandle::new();
17299 let missing_program = std::env::temp_dir().join(format!(
17300 "subc-route-open-missing-program-{}",
17301 std::process::id()
17302 ));
17303 let supervisor =
17304 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17305 .with_handle(supervisor_handle.clone());
17306 let module = supervisor
17307 .supervise_configured(
17308 ModuleSpec {
17309 carry_containment: false,
17310 module_id: "failed".to_string(),
17311 program: missing_program,
17312 args: Vec::new(),
17313 env: Vec::new(),
17314 reserved: false,
17315 reserved_prefixes: Vec::new(),
17316 protocol: ModuleProtocol::Subc,
17317 overlap: Default::default(),
17318 },
17319 true,
17320 )
17321 .unwrap();
17322 assert_eq!(module.state().unwrap(), ModuleState::Failed);
17323
17324 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17325 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
17326 let response = handler
17327 .handle_control_frame(
17328 &ctx,
17329 route_open_frame(305, "failed", unique_project_root("failed")),
17330 )
17331 .await
17332 .unwrap();
17333
17334 assert_eq!(response[0].header.ty, FrameType::Error);
17335 let error = parse_error(&response[0]);
17336 assert_eq!(error["code"], "target_unavailable");
17337 assert!(error["message"]
17338 .as_str()
17339 .unwrap()
17340 .contains("state=failed, enabled=true, live=false"));
17341 }
17342
17343 #[tokio::test]
17344 async fn route_open_role_mismatch_remains_target_unavailable() {
17345 let registry = Arc::new(Registry::default());
17346 let handler = ControlHandler::new(Arc::clone(®istry));
17347 handler
17348 .handle_control(
17349 ConnectionId::new(41),
17350 non_routable_hello_frame_with_control_ops("health-only", 306, None),
17351 )
17352 .unwrap();
17353
17354 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
17355 let response = handler
17356 .handle_control_frame(
17357 &ctx,
17358 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
17359 )
17360 .await
17361 .unwrap();
17362
17363 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17364 assert!(parse_error(&response[0])["message"]
17365 .as_str()
17366 .unwrap()
17367 .contains("does not provide the requested target"));
17368 }
17369
17370 #[tokio::test]
17371 async fn route_open_inactive_registration_remains_target_unavailable() {
17372 let registry = Arc::new(Registry::default());
17373 let handler = ControlHandler::new(Arc::clone(®istry));
17374 handler
17375 .handle_control(
17376 ConnectionId::new(43),
17377 hello_frame("inactive", PROTOCOL_VERSION, 308),
17378 )
17379 .unwrap();
17380 assert!(registry
17381 .set_module_state_for_test("inactive", ChannelState::Closed)
17382 .unwrap());
17383
17384 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
17385 let response = handler
17386 .handle_control_frame(
17387 &ctx,
17388 route_open_frame(309, "inactive", unique_project_root("inactive")),
17389 )
17390 .await
17391 .unwrap();
17392
17393 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17394 assert!(parse_error(&response[0])["message"]
17395 .as_str()
17396 .unwrap()
17397 .contains("is not active"));
17398 }
17399
17400 #[tokio::test]
17401 async fn late_health_reply_is_recorded_through_the_module_response_path() {
17402 let registry = Arc::new(Registry::default());
17403 let forwarding = Arc::new(ForwardingTable::default());
17404 let supervisor_handle = SupervisorHandle::new();
17405 let supervisor =
17406 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
17407 .with_forwarding(Arc::clone(&forwarding))
17408 .with_handle(supervisor_handle.clone());
17409 let module = supervisor
17410 .supervise_configured(
17411 crate::ModuleSpec {
17412 carry_containment: false,
17413 module_id: "late-health-response".to_string(),
17414 program: PathBuf::from("disabled-module"),
17415 args: Vec::new(),
17416 env: Vec::new(),
17417 reserved: false,
17418 reserved_prefixes: Vec::new(),
17419 protocol: ModuleProtocol::Subc,
17420 overlap: Default::default(),
17421 },
17422 false,
17423 )
17424 .unwrap();
17425 let handler =
17426 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17427 .with_supervisor(supervisor_handle);
17428 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
17429 handler
17430 .handle_control_frame(
17431 &module_ctx,
17432 hello_frame_with_control_ops(
17433 "late-health-response",
17434 PROTOCOL_VERSION,
17435 7,
17436 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17437 ),
17438 )
17439 .await
17440 .unwrap();
17441 let probe_started_at = Instant::now() - Duration::from_millis(80);
17442 let pending = forwarding
17443 .begin_health_probe_rpc_for(
17444 "late-health-response",
17445 MODULE_CONTROL_OP_HEALTH_CHECK,
17446 probe_started_at,
17447 Instant::now() - Duration::from_millis(1),
17448 )
17449 .unwrap();
17450 assert!(forwarding
17451 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
17452 .unwrap());
17453
17454 let responses = handler
17455 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
17456 .await
17457 .unwrap();
17458
17459 assert!(responses.is_empty());
17460 let health = module.status().unwrap().health;
17461 assert_eq!(health.late_answer_count, 1);
17462 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
17463 }
17464
17465 #[tokio::test]
17466 async fn health_probe_timeout_and_module_death_are_typed() {
17467 let registry = Arc::new(Registry::default());
17468 let forwarding = Arc::new(ForwardingTable::default());
17469 let handler =
17470 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17471 .with_health_probe_timeout(Duration::from_millis(50));
17472 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
17473 hello_via_sink(
17474 &handler,
17475 &module_ctx,
17476 &mut module_rx,
17477 hello_frame_with_control_ops(
17478 "aft",
17479 PROTOCOL_VERSION,
17480 7,
17481 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17482 ),
17483 )
17484 .await;
17485
17486 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
17487 let responses = handler
17488 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
17489 .await
17490 .unwrap();
17491 assert_eq!(responses[0].header.ty, FrameType::Error);
17492 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
17493 let _ = module_rx.try_recv();
17494
17495 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
17496 let health_handler = handler.clone();
17497 let death_task = tokio::spawn(async move {
17498 health_handler
17499 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
17500 .await
17501 .unwrap()
17502 });
17503 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
17504 .await
17505 .unwrap()
17506 .unwrap();
17507 handler
17508 .cleanup_connection(module_ctx.connection_id)
17509 .unwrap();
17510 let responses = death_task.await.unwrap();
17511 assert_eq!(responses[0].header.ty, FrameType::Error);
17512 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
17513 }
17514
17515 #[test]
17516 fn hello_requires_exact_protocol_version() {
17517 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
17518 let registry = Arc::new(Registry::default());
17519 let handler = ControlHandler::new(Arc::clone(®istry));
17520 let responses = handler
17521 .handle_control(
17522 ConnectionId::new(connection),
17523 hello_frame("aft", offered, 9),
17524 )
17525 .unwrap();
17526
17527 assert_eq!(responses.len(), 1);
17528 assert_eq!(responses[0].header.ty, FrameType::Error);
17529 let error = parse_error(&responses[0]);
17530 assert_eq!(error["code"], "version_unsupported");
17531 assert!(registry.get_module("aft").unwrap().is_none());
17532 assert_eq!(registry.active_registration_count().unwrap(), 0);
17533 }
17534 }
17535
17536 #[test]
17537 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
17538 let registry = Arc::new(Registry::default());
17539 let forwarding = Arc::new(ForwardingTable::default());
17540 let handler =
17541 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17542 let module_connection = ConnectionId::new(301);
17543 let registration = registry
17544 .register_with_control_ops(
17545 manifest("aft-push", PROTOCOL_VERSION),
17546 PROTOCOL_VERSION,
17547 module_connection,
17548 module_baseline_control_ops(),
17549 )
17550 .unwrap();
17551 let (module_tx, _module_rx) = mpsc::channel(8);
17552 let endpoint = forwarding
17553 .register_module_connection(
17554 module_connection,
17555 "aft-push".to_string(),
17556 PROTOCOL_VERSION,
17557 manifest_concurrency(®istration.manifest),
17558 FrameSink::new(module_tx),
17559 )
17560 .unwrap();
17561
17562 let unknown = Frame::build(
17564 FrameType::Push,
17565 control_flags(),
17566 0,
17567 0,
17568 5,
17569 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
17570 )
17571 .unwrap();
17572 let out = handler.handle_status_update(endpoint, unknown).unwrap();
17573 assert!(
17574 out.is_empty(),
17575 "unknown push op must be ignored, got {out:?}"
17576 );
17577
17578 let malformed = Frame::build(
17580 FrameType::Push,
17581 control_flags(),
17582 0,
17583 0,
17584 6,
17585 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
17586 )
17587 .unwrap();
17588 let out = handler.handle_status_update(endpoint, malformed).unwrap();
17589 assert_eq!(out.len(), 1);
17590 assert_eq!(out[0].header.ty, FrameType::Error);
17591 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
17592 }
17593
17594 #[test]
17595 fn hello_rejected_when_connection_already_owns_client_routes() {
17596 let registry = Arc::new(Registry::default());
17597 let forwarding = Arc::new(ForwardingTable::default());
17598 let handler =
17599 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17600 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
17602 let client_connection = ConnectionId::new(202);
17603
17604 let responses = handler
17607 .handle_control(
17608 client_connection,
17609 hello_frame("aft-second", PROTOCOL_VERSION, 9),
17610 )
17611 .unwrap();
17612 assert_eq!(responses[0].header.ty, FrameType::Error);
17613 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
17614 assert!(registry.get_module("aft-second").unwrap().is_none());
17615 }
17616
17617 #[tokio::test]
17618 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
17619 let registry = Arc::new(Registry::default());
17620 let forwarding = Arc::new(ForwardingTable::default());
17621 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
17622 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
17623 hello_via_sink(
17624 &handler,
17625 &module_ctx,
17626 &mut module_rx,
17627 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
17628 )
17629 .await;
17630 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
17631 let pending = forwarding
17632 .begin_route_bind_relay_for_test(
17633 client_ctx.connection_id,
17634 client_ctx.egress.clone(),
17635 2,
17636 "alpha",
17637 )
17638 .unwrap();
17639 forwarding
17640 .complete_pending_relay(
17641 module_ctx.connection_id,
17642 pending.corr,
17643 RouteBindRelayOutcome::Accepted,
17644 )
17645 .unwrap();
17646 client_rx.try_recv().unwrap();
17647 for module_id in ["beta", "alpha"] {
17648 let replies = handler
17649 .handle_control_frame(
17650 &module_ctx,
17651 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
17652 )
17653 .await
17654 .unwrap();
17655 assert_eq!(replies.len(), 1, "second HELLO must be refused");
17656 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
17657 }
17658 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
17659 assert!(registry.get_module("beta").unwrap().is_none());
17660 assert!(matches!(
17661 forwarding
17662 .lookup_data_route(
17663 client_ctx.connection_id,
17664 pending.client_channel,
17665 pending.client_epoch,
17666 )
17667 .unwrap(),
17668 DataRoute::Client(DataRouteState::Bound(_))
17669 ));
17670 assert!(handler
17671 .hello_launch_nonces
17672 .lock()
17673 .unwrap()
17674 .presented(module_ctx.connection_id, Some("alpha-nonce")));
17675 assert!(module_rx.try_recv().is_err());
17676 }
17677
17678 #[test]
17679 fn reserved_module_hello_requires_matching_launch_nonce() {
17680 let registry = Arc::new(Registry::default());
17681 let supervisor = SupervisorHandle::new();
17682 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
17685 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
17686
17687 let no_nonce = handler
17689 .handle_control(
17690 ConnectionId::new(1),
17691 hello_frame("vault", PROTOCOL_VERSION, 1),
17692 )
17693 .unwrap();
17694 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
17695 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
17696 assert!(registry.get_module("vault").unwrap().is_none());
17697
17698 let wrong = handler
17700 .handle_control(
17701 ConnectionId::new(2),
17702 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
17703 )
17704 .unwrap();
17705 assert_eq!(wrong[0].header.ty, FrameType::Error);
17706 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
17707 assert!(registry.get_module("vault").unwrap().is_none());
17708
17709 let ok = handler
17711 .handle_control(
17712 ConnectionId::new(3),
17713 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
17714 )
17715 .unwrap();
17716 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
17717 assert!(registry.get_module("vault").unwrap().is_some());
17718 }
17719
17720 #[test]
17721 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
17722 let registry = Arc::new(Registry::default());
17723 let supervisor = SupervisorHandle::new();
17724 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
17725 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
17726 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
17727
17728 let squat = handler
17729 .handle_control(
17730 ConnectionId::new(1),
17731 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
17732 )
17733 .unwrap();
17734 assert_eq!(squat[0].header.ty, FrameType::Error);
17735 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
17736 assert!(parse_error(&squat[0])["message"]
17737 .as_str()
17738 .unwrap()
17739 .contains("fed:"));
17740
17741 let accepted_peer = handler
17742 .handle_control(
17743 ConnectionId::new(2),
17744 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
17745 )
17746 .unwrap();
17747 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
17748
17749 let accepted_short = handler
17750 .handle_control(
17751 ConnectionId::new(3),
17752 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
17753 )
17754 .unwrap();
17755 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
17756
17757 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
17758 let response = handler
17759 .handle_control(
17760 ConnectionId::new(conn),
17761 hello_frame(module_id, PROTOCOL_VERSION, conn),
17762 )
17763 .unwrap();
17764 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
17765 }
17766 }
17767
17768 #[test]
17769 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
17770 let registry = Arc::new(Registry::default());
17771 let supervisor = SupervisorHandle::new();
17772 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
17773 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
17774 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
17775 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
17776
17777 let owner_nonce = handler
17778 .handle_control(
17779 ConnectionId::new(1),
17780 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
17781 )
17782 .unwrap();
17783 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
17784 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
17785 assert!(registry.get_module("fed:special").unwrap().is_none());
17786
17787 let exact_nonce = handler
17788 .handle_control(
17789 ConnectionId::new(2),
17790 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
17791 )
17792 .unwrap();
17793 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
17794 assert!(registry.get_module("fed:special").unwrap().is_some());
17795 }
17796
17797 #[test]
17798 fn non_reserved_module_ignores_launch_nonce() {
17799 let registry = Arc::new(Registry::default());
17800 let handler = ControlHandler::new(Arc::clone(®istry));
17803 let no_nonce = handler
17804 .handle_control(
17805 ConnectionId::new(1),
17806 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
17807 )
17808 .unwrap();
17809 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
17810 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
17811
17812 let echoed_nonce = handler
17813 .handle_control(
17814 ConnectionId::new(2),
17815 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
17816 )
17817 .unwrap();
17818 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
17819 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
17820 }
17821
17822 #[test]
17823 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
17824 let handler = ControlHandler::default();
17825 let conn = ConnectionId::new(1);
17826 let malformed = Frame::build(
17827 FrameType::Hello,
17828 control_flags(),
17829 0,
17830 0,
17831 3,
17832 b"{not json".to_vec(),
17833 )
17834 .unwrap();
17835
17836 let error = handler.handle_control(conn, malformed).unwrap();
17837 assert_eq!(error[0].header.ty, FrameType::Error);
17838 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
17839
17840 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
17841 let pong = handler.handle_control(conn, ping).unwrap();
17842 assert_eq!(pong[0].header.ty, FrameType::Pong);
17843 assert_eq!(pong[0].header.corr, 4);
17844 }
17845
17846 #[test]
17847 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
17848 let registry = Arc::new(Registry::default());
17849 let handler = ControlHandler::new(Arc::clone(®istry));
17850
17851 handler
17852 .handle_control(
17853 ConnectionId::new(1),
17854 hello_frame("aft", PROTOCOL_VERSION, 1),
17855 )
17856 .unwrap();
17857 let duplicate = handler
17858 .handle_control(
17859 ConnectionId::new(2),
17860 hello_frame("aft", PROTOCOL_VERSION, 2),
17861 )
17862 .unwrap();
17863
17864 assert_eq!(duplicate[0].header.ty, FrameType::Error);
17865 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17866 let registration = registry.get_module("aft").unwrap().unwrap();
17867 assert_eq!(registration.connection_id, ConnectionId::new(1));
17868 }
17869
17870 #[test]
17871 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
17872 let registry = Arc::new(Registry::default());
17873 let forwarding = Arc::new(ForwardingTable::default());
17874 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
17875 let handler =
17876 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17877 .with_process_liveness(process_liveness);
17878 let (ctx, route_channel, route_epoch) =
17879 bind_liveness_route(®istry, &forwarding, "aft-dead");
17880 let responses = handler
17881 .handle_route_poll(
17882 &ctx,
17883 route_poll_frame(41, PollKind::Liveness, route_channel),
17884 route_channel,
17885 route_epoch,
17886 PollKind::Liveness,
17887 )
17888 .unwrap();
17889
17890 assert_eq!(responses.len(), 1);
17891 assert_eq!(responses[0].header.ty, FrameType::Response);
17892 assert_route_poll_liveness(&responses[0], false);
17893 }
17894
17895 #[test]
17896 fn liveness_poll_without_process_source_uses_bound_route() {
17897 let registry = Arc::new(Registry::default());
17898 let forwarding = Arc::new(ForwardingTable::default());
17899 let handler =
17900 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17901 let (ctx, route_channel, route_epoch) =
17902 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
17903 let responses = handler
17904 .handle_route_poll(
17905 &ctx,
17906 route_poll_frame(42, PollKind::Liveness, route_channel),
17907 route_channel,
17908 route_epoch,
17909 PollKind::Liveness,
17910 )
17911 .unwrap();
17912
17913 assert_route_poll_liveness(&responses[0], true);
17914 }
17915
17916 #[test]
17917 fn liveness_poll_untracked_process_source_uses_bound_route() {
17918 let registry = Arc::new(Registry::default());
17919 let forwarding = Arc::new(ForwardingTable::default());
17920 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
17921 let handler =
17922 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17923 .with_process_liveness(process_liveness);
17924 let (ctx, route_channel, route_epoch) =
17925 bind_liveness_route(®istry, &forwarding, "aft-untracked");
17926 let responses = handler
17927 .handle_route_poll(
17928 &ctx,
17929 route_poll_frame(43, PollKind::Liveness, route_channel),
17930 route_channel,
17931 route_epoch,
17932 PollKind::Liveness,
17933 )
17934 .unwrap();
17935
17936 assert_route_poll_liveness(&responses[0], true);
17937 }
17938
17939 #[tokio::test]
17940 async fn unknown_op_returns_unknown_control_op() {
17941 let handler = ControlHandler::default();
17942 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
17943 let request = Frame::build(
17944 FrameType::Request,
17945 control_flags(),
17946 0,
17947 0,
17948 55,
17949 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
17950 )
17951 .unwrap();
17952
17953 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
17954
17955 assert_eq!(response.len(), 1);
17956 assert_eq!(response[0].header.ty, FrameType::Error);
17957 assert_eq!(response[0].header.corr, 55);
17958 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
17959 }
17960
17961 #[tokio::test]
17962 async fn supervisor_provenance_rejects_unknown_exact_module() {
17963 let handler = ControlHandler::default();
17964 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
17965 let request = Frame::build(
17966 FrameType::Request,
17967 control_flags(),
17968 0,
17969 0,
17970 57,
17971 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
17972 )
17973 .unwrap();
17974
17975 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
17976
17977 assert_eq!(response.len(), 1);
17978 assert_eq!(response[0].header.ty, FrameType::Error);
17979 assert_eq!(response[0].header.corr, 57);
17980 let error = parse_error(&response[0]);
17981 assert_eq!(error["code"], "unknown_module");
17982 assert_eq!(error["message"], "module_id 'missing' is not supervised");
17983 }
17984
17985 #[test]
17986 fn provenance_probe_override_keeps_handler_tests_deterministic() {
17987 let expected = subc_control::RunningImageAgreement::Unavailable {
17988 reason: subc_control::RunningImageUnavailableReason::HashFailed,
17989 };
17990 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
17991 assert_eq!(handler.provenance_probe_override, Some(expected));
17992 }
17993
17994 #[test]
17995 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
17996 let verdict = reload_verdict(
17997 std::path::Path::new("/bin/new"),
17998 Some(std::path::Path::new("/bin/old")),
17999 subc_control::RunningImageAgreement::Unavailable {
18000 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18001 },
18002 );
18003 assert!(matches!(
18004 verdict.path,
18005 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
18006 if configured == std::path::Path::new("/bin/new")
18007 && spawned_from == std::path::Path::new("/bin/old")
18008 ));
18009 }
18010
18011 #[test]
18012 fn launch_spec_without_running_digest_is_unavailable_not_match() {
18013 use subc_control::{
18014 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
18015 };
18016 let configured = crate::supervise::ModuleSpec {
18017 carry_containment: false,
18018 module_id: "legacy".into(),
18019 program: "/module".into(),
18020 args: Vec::new(),
18021 env: Vec::new(),
18022 reserved: false,
18023 reserved_prefixes: Vec::new(),
18024 protocol: subc_control::ModuleProtocol::Subc,
18025 overlap: Default::default(),
18026 };
18027 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
18028 assert_eq!(
18029 launch_spec_agreement(&configured, false, Some(&digest)),
18030 Agreement::Unavailable {
18031 reason: Reason::NotRunning
18032 }
18033 );
18034 assert_eq!(
18035 launch_spec_agreement(&configured, true, None),
18036 Agreement::Unavailable {
18037 reason: Reason::SpawnedSpecUnavailable
18038 }
18039 );
18040 assert_eq!(
18041 launch_spec_agreement(&configured, true, Some(&digest)),
18042 Agreement::Match
18043 );
18044 }
18045
18046 #[cfg(windows)]
18047 #[test]
18048 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
18049 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
18050 let configured = dir.join("module.exe");
18051 std::fs::write(&configured, b"executable path fixture").unwrap();
18052 let spawned = std::fs::canonicalize(&configured).unwrap();
18053 assert_ne!(
18054 configured, spawned,
18055 "fixture must exercise different path spellings"
18056 );
18057 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
18058 let lower_case = configured.to_string_lossy().to_lowercase();
18059 for spelling in [
18060 &configured,
18061 Path::new(&forward_slashes),
18062 Path::new(&lower_case),
18063 ] {
18064 let verdict = reload_verdict(
18065 spelling,
18066 Some(&spawned),
18067 subc_control::RunningImageAgreement::Unavailable {
18068 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18069 },
18070 );
18071 assert_eq!(
18072 verdict.path,
18073 ReloadPathAgreement::Match,
18074 "configured {spelling:?}, spawned {spawned:?}"
18075 );
18076 }
18077 }
18078
18079 #[cfg(windows)]
18080 #[test]
18081 fn windows_reload_path_keeps_different_existing_program_pending() {
18082 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
18083 let old = dir.join("old.exe");
18084 let new = dir.join("new.exe");
18085 std::fs::write(&old, b"same executable content").unwrap();
18086 std::fs::copy(&old, &new).unwrap();
18087 let spawned = std::fs::canonicalize(&old).unwrap();
18088 let image = subc_control::RunningImageAgreement::Unavailable {
18089 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18090 };
18091 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
18092 assert!(
18093 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
18094 if configured == new && spawned_from == spawned)
18095 );
18096 let missing = dir.join("missing.exe");
18097 assert!(matches!(
18098 reload_verdict(&missing, Some(&spawned), image).path,
18099 ReloadPathAgreement::Mismatch { .. }
18100 ));
18101 }
18102
18103 #[test]
18104 fn reload_verdict_detects_replaced_image_at_same_path() {
18105 let image = subc_control::RunningImageAgreement::Mismatch {
18106 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
18107 digest: "old".into(),
18108 },
18109 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
18110 digest: "new".into(),
18111 },
18112 };
18113 let verdict = reload_verdict(
18114 std::path::Path::new("/bin/same"),
18115 Some(std::path::Path::new("/bin/same")),
18116 image.clone(),
18117 );
18118 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18119 assert_eq!(verdict.image, image);
18120 }
18121
18122 #[test]
18123 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
18124 let image = subc_control::RunningImageAgreement::Unavailable {
18125 reason: subc_control::RunningImageUnavailableReason::NotRunning,
18126 };
18127 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
18128 assert_eq!(
18129 verdict.path,
18130 subc_control::ReloadPathAgreement::Unavailable {
18131 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
18132 }
18133 );
18134 assert_eq!(verdict.image, image);
18135
18136 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
18137 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18138 };
18139 let verdict = reload_verdict(
18140 std::path::Path::new("/bin/same"),
18141 Some(std::path::Path::new("/bin/same")),
18142 unconfirmed.clone(),
18143 );
18144 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18145 assert_eq!(verdict.image, unconfirmed);
18146 }
18147
18148 #[test]
18149 fn reload_verdict_preserves_each_image_unavailability_reason() {
18150 use subc_control::RunningImageUnavailableReason as Reason;
18151
18152 for reason in [
18153 Reason::NotRunning,
18154 Reason::UnsupportedPlatform,
18155 Reason::RunningExecutableUnreadable,
18156 Reason::SpawnedPathUnreadable,
18157 Reason::HashFailed,
18158 Reason::ProcessIdentityUnconfirmed,
18159 Reason::Unknown("future_probe_reason".to_string()),
18160 ] {
18161 let image = subc_control::RunningImageAgreement::Unavailable {
18162 reason: reason.clone(),
18163 };
18164 let verdict = reload_verdict(
18165 std::path::Path::new("/bin/same"),
18166 Some(std::path::Path::new("/bin/same")),
18167 image.clone(),
18168 );
18169 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18170 assert_eq!(verdict.image, image, "{reason:?}");
18171 }
18172 }
18173
18174 #[tokio::test]
18175 async fn malformed_control_bodies_return_invalid_control_body() {
18176 let handler = ControlHandler::default();
18177 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
18178
18179 for (corr, body) in [
18180 (56, br#"{"route_channel":1}"#.as_slice()),
18181 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
18182 (
18183 58,
18184 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
18185 ),
18186 ] {
18187 let request = Frame::build(
18188 FrameType::Request,
18189 control_flags(),
18190 0,
18191 0,
18192 corr,
18193 body.to_vec(),
18194 )
18195 .unwrap();
18196 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18197
18198 assert_eq!(response.len(), 1);
18199 assert_eq!(response[0].header.ty, FrameType::Error);
18200 assert_eq!(response[0].header.corr, corr);
18201 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
18202 }
18203 }
18204
18205 #[tokio::test]
18206 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
18207 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18208 let registry = Arc::new(Registry::default());
18209 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18210 let router = Router::with_control_handler(Arc::clone(&control));
18211 let connection = router.begin_connection();
18212 let (ctx, mut rx) = route_ctx(connection.id());
18213
18214 router
18215 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
18216 .await
18217 .unwrap();
18218 let response = rx.recv().await.unwrap();
18219 let ack = parse_ack(&response);
18220 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18221 let channel = 1;
18222
18223 let goodbye =
18224 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
18225 router.route_for_connection(&ctx, goodbye).await.unwrap();
18226 assert!(rx.try_recv().is_err());
18227 assert!(registry.get_module("aft").unwrap().is_none());
18228
18229 router
18230 .route_for_connection(&ctx, channel_request(channel, 13))
18231 .await
18232 .unwrap();
18233 let error_frame = rx.recv().await.unwrap();
18234 assert_eq!(error_frame.header.ty, FrameType::Error);
18235 assert_eq!(error_frame.header.channel, channel);
18236 let captured = crate::router::test_log::captured_logs(&logs);
18237 assert_eq!(
18238 captured
18239 .lines()
18240 .filter(|line| {
18241 line.contains("module_id=aft")
18242 && line.contains("reason=explicit_goodbye")
18243 && line.contains("module registration ended")
18244 })
18245 .count(),
18246 1,
18247 "unexpected GOODBYE registry log: {captured}"
18248 );
18249 }
18250
18251 #[tokio::test]
18252 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
18253 let registry = Arc::new(Registry::default());
18254 let handler = ControlHandler::new(registry).with_capability_config(
18255 [("prov".to_string(), true), ("cons".to_string(), true)],
18256 BTreeMap::new(),
18257 );
18258 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
18259 register_capability_manifest(
18260 &handler,
18261 &provider_ctx,
18262 &mut provider_rx,
18263 capability_manifest("prov", &["thing/v1"], &[]),
18264 1,
18265 )
18266 .await;
18267 let mut consumer = capability_manifest("cons", &[], &[]);
18268 consumer.capabilities.as_mut().unwrap().requires.push(
18269 subc_protocol::manifest::CapabilityRequirement {
18270 capability: "thing/v1".to_string(),
18271 need: subc_protocol::manifest::CapabilityNeed::Required,
18272 },
18273 );
18274 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
18275 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
18276 assert_eq!(
18277 handler.capability_evaluator.verdict("cons", "thing/v1"),
18278 Some(CapabilityVerdict::Provided)
18279 );
18280 let (mut client_rx, _) = open_route_for_capability_test(
18281 &handler,
18282 &provider_ctx,
18283 &mut provider_rx,
18284 703,
18285 3,
18286 "prov",
18287 None,
18288 )
18289 .await;
18290 let goodbye =
18291 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18292 handler
18293 .handle_control_frame(&provider_ctx, goodbye)
18294 .await
18295 .unwrap();
18296 assert_eq!(
18297 handler.capability_evaluator.verdict("cons", "thing/v1"),
18298 Some(CapabilityVerdict::NeverProvided)
18299 );
18300 let closed = client_rx
18301 .try_recv()
18302 .expect("GOODBYE pushes route.closed before route GOODBYE");
18303 assert!(
18304 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
18305 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
18306 );
18307 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
18308 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
18309 }
18310
18311 struct RouteClosedFixture {
18312 handler: ControlHandler,
18313 #[cfg(unix)]
18314 supervisor: Supervisor,
18315 module_ctx: RouteCtx,
18316 #[cfg(unix)]
18317 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18318 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18319 route: Arc<crate::forwarding::RouteBinding>,
18320 }
18321
18322 async fn route_closed_fixture() -> RouteClosedFixture {
18323 let registry = Arc::new(Registry::default());
18324 let handler = ControlHandler::new(Arc::clone(®istry));
18325 #[cfg(unix)]
18326 let supervisor = Supervisor::new(registry, RestartPolicy::default())
18327 .with_forwarding(Arc::clone(&handler.forwarding));
18328 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
18329 register_capability_manifest(
18330 &handler,
18331 &module_ctx,
18332 &mut module_rx,
18333 capability_manifest("prov", &[], &[]),
18334 1,
18335 )
18336 .await;
18337 let (client_rx, bind) = open_route_for_capability_test(
18338 &handler,
18339 &module_ctx,
18340 &mut module_rx,
18341 702,
18342 2,
18343 "prov",
18344 None,
18345 )
18346 .await;
18347 let ModuleControlRequest::RouteBind {
18348 route_channel: channel,
18349 epoch: route_epoch,
18350 ..
18351 } = bind
18352 else {
18353 panic!("expected route.bind");
18354 };
18355 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
18356 handler
18357 .forwarding
18358 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
18359 .unwrap()
18360 else {
18361 panic!("the acknowledged route must be bound");
18362 };
18363 RouteClosedFixture {
18364 handler,
18365 #[cfg(unix)]
18366 supervisor,
18367 module_ctx,
18368 #[cfg(unix)]
18369 module_rx,
18370 client_rx,
18371 route,
18372 }
18373 }
18374
18375 impl RouteClosedFixture {
18376 #[cfg(unix)]
18377 async fn cut(&mut self) {
18378 let notices = async {
18379 let mut draining = self.module_rx.recv().await.unwrap();
18380 assert!(matches!(
18381 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
18382 &draining.body
18383 )
18384 .unwrap(),
18385 subc_protocol::session::ModuleControlCommand::Draining {
18386 reason: RouteCloseReason::Disable,
18387 ..
18388 }
18389 ));
18390 draining.flushed.take().unwrap().send(()).unwrap();
18391 let mut closing = self.client_rx.recv().await.unwrap();
18392 assert!(matches!(
18393 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
18394 ClientControlPush::RouteClosing {
18395 reason: RouteCloseReason::Restart,
18396 ..
18397 }
18398 ));
18399 closing.flushed.take().unwrap().send(()).unwrap();
18400 };
18401 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
18402 result.unwrap();
18403 }
18404
18405 fn disconnect(&mut self) -> ClientControlPush {
18406 self.handler
18407 .cleanup_connection(self.module_ctx.connection_id)
18408 .unwrap();
18409 let closed = self.client_rx.try_recv().unwrap();
18410 assert_eq!(closed.header.ty, FrameType::Push);
18411 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
18412 assert!(matches!(
18413 &push,
18414 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
18415 if channels == &[self.route.client_channel]
18416 ));
18417 assert_eq!(
18418 self.client_rx.try_recv().unwrap().header.ty,
18419 FrameType::Goodbye
18420 );
18421 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
18422 push
18423 }
18424 }
18425
18426 #[cfg(unix)]
18427 #[tokio::test(start_paused = true)]
18428 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
18429 let mut fixture = route_closed_fixture().await;
18430 fixture.cut().await;
18431 assert!(matches!(
18432 fixture.disconnect(),
18433 ClientControlPush::RouteClosed {
18434 reason: RouteCloseReason::Restart,
18435 drained: true,
18436 ..
18437 }
18438 ));
18439 }
18440
18441 #[cfg(unix)]
18442 #[tokio::test(start_paused = true)]
18443 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
18444 let mut fixture = route_closed_fixture().await;
18445 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
18446 let started = tokio::time::Instant::now();
18447 fixture.cut().await;
18448 assert!(started.elapsed() >= Duration::from_secs(2));
18449 assert_eq!(fixture.route.flow.in_flight(), 1);
18450 assert!(matches!(
18451 fixture.disconnect(),
18452 ClientControlPush::RouteClosed {
18453 reason: RouteCloseReason::Restart,
18454 drained: false,
18455 ..
18456 }
18457 ));
18458 }
18459
18460 #[tokio::test]
18461 async fn module_connection_crash_reports_crash_and_not_drained() {
18462 let mut fixture = route_closed_fixture().await;
18463 assert!(matches!(
18467 fixture.disconnect(),
18468 ClientControlPush::RouteClosed {
18469 reason: RouteCloseReason::Crash,
18470 drained: false,
18471 ..
18472 }
18473 ));
18474 }
18475
18476 #[tokio::test]
18477 async fn dropping_router_connection_releases_registration() {
18478 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18479 let registry = Arc::new(Registry::default());
18480 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18481 let router = Router::with_control_handler(Arc::clone(&control));
18482 let connection = router.begin_connection();
18483 let connection_id = connection.id();
18484 let (ctx, mut rx) = route_ctx(connection_id);
18485
18486 router
18487 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
18488 .await
18489 .unwrap();
18490 let response = rx.recv().await.unwrap();
18491 let ack = parse_ack(&response);
18492 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18493 assert!(registry.get_module("aft").unwrap().is_some());
18494
18495 drop(connection);
18496
18497 assert!(registry.get_module("aft").unwrap().is_none());
18498 assert_eq!(registry.active_registration_count().unwrap(), 0);
18499
18500 control
18501 .cleanup_connection(ConnectionId::new(u64::MAX))
18502 .unwrap();
18503 let captured = crate::router::test_log::captured_logs(&logs);
18504 println!("captured registry lifecycle logs:\n{captured}");
18505 let events: Vec<_> = captured
18506 .lines()
18507 .filter(|line| {
18508 line.contains("module registered module_id=aft ")
18509 || line.contains("module registration ended")
18510 })
18511 .collect();
18512 assert_eq!(
18513 events.len(),
18514 2,
18515 "unexpected registry lifecycle logs: {captured}"
18516 );
18517 assert!(events[0].contains("module registered module_id=aft "));
18518 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
18519 assert!(events[1].contains(&format!(
18520 "module_id=aft connection_id={}",
18521 connection_id.get()
18522 )));
18523 assert!(events[1].contains("reason=connection_closed"));
18524 assert!(events[1].contains("module registration ended"));
18525 assert_eq!(
18526 captured
18527 .lines()
18528 .filter(|line| line.contains("module registered module_id=aft "))
18529 .count(),
18530 1,
18531 "legacy registration admission line must appear once: {captured}"
18532 );
18533 }
18534
18535 #[tokio::test]
18536 async fn hello_registration_keeps_legacy_module_registered_line_once() {
18537 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18538 let registry = Arc::new(Registry::default());
18539 let control = ControlHandler::new(Arc::clone(®istry));
18540 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
18541 hello_via_sink(
18542 &control,
18543 &ctx,
18544 &mut rx,
18545 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
18546 )
18547 .await;
18548
18549 let captured = crate::router::test_log::captured_logs(&logs);
18550 let admissions: Vec<_> = captured
18551 .lines()
18552 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
18553 .collect();
18554 assert_eq!(
18555 admissions.len(),
18556 1,
18557 "legacy admission line must remain exactly once: {captured}"
18558 );
18559 assert!(admissions[0].contains("routable_provider=true"));
18560 assert!(admissions[0].contains("connection_id=777"));
18561 }
18562
18563 fn capability_manifest(
18564 module_id: &str,
18565 provides: &[&str],
18566 must_never_reach: &[&str],
18567 ) -> ModuleManifest {
18568 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
18569 manifest.capabilities = Some(CapabilityDeclarations {
18570 provides: provides
18571 .iter()
18572 .map(|capability| (*capability).to_string())
18573 .collect(),
18574 requires: Vec::new(),
18575 must_never_reach: must_never_reach
18576 .iter()
18577 .map(|capability| (*capability).to_string())
18578 .collect(),
18579 });
18580 manifest
18581 }
18582
18583 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
18584 Frame::build(
18585 FrameType::Hello,
18586 control_flags(),
18587 0,
18588 0,
18589 corr,
18590 serde_json::to_vec(&ModuleHelloBody {
18591 protocol_ver: manifest.protocol_ver,
18592 manifest,
18593 control_ops: None,
18594 launch_nonce: None,
18595 })
18596 .expect("capability test HELLO serializes"),
18597 )
18598 .expect("capability test HELLO frame builds")
18599 }
18600
18601 fn catalog_update_with_capabilities_frame(
18602 corr: u64,
18603 capabilities: CapabilityDeclarations,
18604 ) -> Frame {
18605 Frame::build(
18606 FrameType::Request,
18607 control_flags(),
18608 0,
18609 0,
18610 corr,
18611 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
18612 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
18613 capabilities: Some(capabilities),
18614 ready: None,
18615 })
18616 .expect("capability catalog.update serializes"),
18617 )
18618 .expect("capability catalog.update frame builds")
18619 }
18620
18621 async fn register_capability_manifest(
18622 handler: &ControlHandler,
18623 ctx: &RouteCtx,
18624 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
18625 manifest: ModuleManifest,
18626 corr: u64,
18627 ) {
18628 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
18629 }
18630
18631 async fn open_route_for_capability_test(
18632 handler: &ControlHandler,
18633 target_ctx: &RouteCtx,
18634 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
18635 client_connection_id: u64,
18636 corr: u64,
18637 target_module_id: &str,
18638 consumer_identity: Option<ConsumerIdentity>,
18639 ) -> (
18640 mpsc::Receiver<crate::router::OutboundFrame>,
18641 ModuleControlRequest,
18642 ) {
18643 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
18644 let route_handler = handler.clone();
18645 let target_module_id = target_module_id.to_string();
18646 let route_task = tokio::spawn(async move {
18647 route_handler
18648 .handle_control_frame(
18649 &client_ctx,
18650 route_open_frame_with_admission_facts(
18651 corr,
18652 &target_module_id,
18653 unique_project_root("admission-facts"),
18654 consumer_identity,
18655 None,
18656 ),
18657 )
18658 .await
18659 .expect("capability test route.open succeeds")
18660 });
18661 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
18662 .await
18663 .expect("capability test route.open must reach route.bind")
18664 .expect("target control receiver stays open");
18665 let bind_request: ModuleControlRequest =
18666 serde_json::from_slice(&bind.body).expect("route.bind decodes");
18667 handler
18668 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
18669 .await
18670 .expect("capability test route.bind ACK succeeds");
18671 assert!(route_task.await.expect("route.open task joins").is_empty());
18672 let opened = client_rx
18673 .recv()
18674 .await
18675 .expect("successful route.open publishes a response");
18676 assert!(matches!(
18677 serde_json::from_slice::<ClientControlResponse>(&opened.body),
18678 Ok(ClientControlResponse::RouteOpen { .. })
18679 ));
18680 (client_rx, bind_request)
18681 }
18682
18683 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
18684 assert_eq!(frame.header.ty, FrameType::Push);
18685 assert_eq!(frame.header.channel, 0);
18686 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
18687 .expect("route.closed control push decodes");
18688 let ClientControlPush::RouteClosed { channels, .. } = &push else {
18689 panic!("expected route.closed");
18690 };
18691 assert_eq!(channels.len(), 1, "exactly one violating route closed");
18692 let closed_channel = channels[0];
18693 let channels = channels.clone();
18694 assert_eq!(
18695 push,
18696 ClientControlPush::RouteClosed {
18697 module_id: target_module_id.to_string(),
18698 channels,
18699 reason: RouteCloseReason::CapabilityDenied,
18700 drained: false,
18701 abandoned: 0,
18702 excluded_subscriptions: 0,
18703 terminal: Some(false),
18704 }
18705 );
18706 closed_channel
18707 }
18708
18709 #[tokio::test]
18710 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
18711 let registry = Arc::new(Registry::default());
18712 let forwarding = Arc::new(ForwardingTable::default());
18713 let supervisor = SupervisorHandle::new();
18714 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
18715 let handler =
18716 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18717 .with_supervisor(supervisor);
18718 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
18719 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
18720 register_capability_manifest(
18721 &handler,
18722 &target_ctx,
18723 &mut target_rx,
18724 capability_manifest("target", &["credentials-provider/v1"], &[]),
18725 1,
18726 )
18727 .await;
18728 register_capability_manifest(
18729 &handler,
18730 &opener_ctx,
18731 &mut opener_rx,
18732 capability_manifest("opener", &[], &["credentials-provider/v1"]),
18733 2,
18734 )
18735 .await;
18736
18737 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
18738 let replies = handler
18739 .handle_control_frame(
18740 &client_ctx,
18741 route_open_frame_with_admission_facts(
18742 3,
18743 "target",
18744 unique_project_root("admission-facts"),
18745 Some(ConsumerIdentity {
18746 module_id: "opener".to_string(),
18747 launch_nonce: "opener-nonce".to_string(),
18748 }),
18749 None,
18750 ),
18751 )
18752 .await
18753 .expect("denied route.open returns a typed frame");
18754 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
18755 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
18756 assert!(
18757 target_rx.try_recv().is_err(),
18758 "forbidden route.open must not relay route.bind"
18759 );
18760 }
18761
18762 #[tokio::test]
18763 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
18764 let registry = Arc::new(Registry::default());
18765 let forwarding = Arc::new(ForwardingTable::default());
18766 let supervisor = SupervisorHandle::new();
18767 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
18768 let handler =
18769 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18770 .with_supervisor(supervisor);
18771 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
18772 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
18773 register_capability_manifest(
18774 &handler,
18775 &target_ctx,
18776 &mut target_rx,
18777 capability_manifest("target", &["credentials-provider/v1"], &[]),
18778 1,
18779 )
18780 .await;
18781 register_capability_manifest(
18782 &handler,
18783 &old_opener_ctx,
18784 &mut old_opener_rx,
18785 capability_manifest("opener", &[], &[]),
18786 2,
18787 )
18788 .await;
18789 let (mut client_rx, _) = open_route_for_capability_test(
18790 &handler,
18791 &target_ctx,
18792 &mut target_rx,
18793 712,
18794 3,
18795 "target",
18796 Some(ConsumerIdentity {
18797 module_id: "opener".to_string(),
18798 launch_nonce: "opener-nonce".to_string(),
18799 }),
18800 )
18801 .await;
18802 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
18803
18804 handler
18805 .cleanup_connection(old_opener_ctx.connection_id)
18806 .expect("old opener registration cleans up");
18807 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
18808 register_capability_manifest(
18809 &handler,
18810 &new_opener_ctx,
18811 &mut new_opener_rx,
18812 capability_manifest("opener", &[], &["credentials-provider/v1"]),
18813 4,
18814 )
18815 .await;
18816
18817 let closed_channel = assert_capability_denied_push(
18818 client_rx
18819 .try_recv()
18820 .expect("HELLO deny addition must emit route.closed")
18821 .frame,
18822 "target",
18823 );
18824 let goodbye = client_rx
18825 .try_recv()
18826 .expect("HELLO deny addition must send the client a route GOODBYE")
18827 .frame;
18828 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18829 assert_eq!(goodbye.header.channel, closed_channel);
18830 assert_ne!(goodbye.header.channel, 0);
18831 assert_ne!(goodbye.header.epoch, 0);
18832 assert!(client_rx.try_recv().is_err());
18833 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
18834 assert!(matches!(
18835 target_rx.try_recv(),
18836 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
18837 ));
18838 }
18839
18840 #[tokio::test]
18841 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
18842 let registry = Arc::new(Registry::default());
18843 let forwarding = Arc::new(ForwardingTable::default());
18844 let supervisor = SupervisorHandle::new();
18845 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
18846 let handler =
18847 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18848 .with_supervisor(supervisor);
18849 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
18850 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
18851 register_capability_manifest(
18852 &handler,
18853 &target_ctx,
18854 &mut target_rx,
18855 capability_manifest("target", &[], &[]),
18856 1,
18857 )
18858 .await;
18859 register_capability_manifest(
18860 &handler,
18861 &opener_ctx,
18862 &mut opener_rx,
18863 capability_manifest("opener", &[], &["credentials-provider/v1"]),
18864 2,
18865 )
18866 .await;
18867 let (mut client_rx, _) = open_route_for_capability_test(
18868 &handler,
18869 &target_ctx,
18870 &mut target_rx,
18871 722,
18872 3,
18873 "target",
18874 Some(ConsumerIdentity {
18875 module_id: "opener".to_string(),
18876 launch_nonce: "opener-nonce".to_string(),
18877 }),
18878 )
18879 .await;
18880 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
18881
18882 let replies = handler
18883 .handle_control_frame(
18884 &target_ctx,
18885 catalog_update_with_capabilities_frame(
18886 4,
18887 CapabilityDeclarations {
18888 provides: vec!["credentials-provider/v1".to_string()],
18889 requires: Vec::new(),
18890 must_never_reach: Vec::new(),
18891 },
18892 ),
18893 )
18894 .await
18895 .expect("claim catalog.update succeeds");
18896 assert!(matches!(
18897 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
18898 Ok(ModuleControlResponseToModule::CatalogUpdate {})
18899 ));
18900 let closed_channel = assert_capability_denied_push(
18901 client_rx
18902 .try_recv()
18903 .expect("claim addition must emit route.closed")
18904 .frame,
18905 "target",
18906 );
18907 let goodbye = client_rx
18908 .try_recv()
18909 .expect("claim addition must send the client a route GOODBYE")
18910 .frame;
18911 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18912 assert_eq!(goodbye.header.channel, closed_channel);
18913 assert_ne!(goodbye.header.channel, 0);
18914 assert_ne!(goodbye.header.epoch, 0);
18915 assert!(client_rx.try_recv().is_err());
18916 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
18917 assert!(matches!(
18918 target_rx.try_recv(),
18919 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
18920 ));
18921 }
18922
18923 #[tokio::test]
18924 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
18925 let registry = Arc::new(Registry::default());
18926 let forwarding = Arc::new(ForwardingTable::default());
18927 let supervisor = SupervisorHandle::new();
18928 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
18929 let handler =
18930 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18931 .with_supervisor(supervisor);
18932 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
18933 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
18934 register_capability_manifest(
18935 &handler,
18936 &target_ctx,
18937 &mut target_rx,
18938 capability_manifest("target", &["credentials-provider/v1"], &[]),
18939 1,
18940 )
18941 .await;
18942 register_capability_manifest(
18943 &handler,
18944 &opener_ctx,
18945 &mut opener_rx,
18946 capability_manifest("opener", &[], &[]),
18947 2,
18948 )
18949 .await;
18950 let (mut client_rx, _) = open_route_for_capability_test(
18951 &handler,
18952 &target_ctx,
18953 &mut target_rx,
18954 732,
18955 3,
18956 "target",
18957 Some(ConsumerIdentity {
18958 module_id: "opener".to_string(),
18959 launch_nonce: "opener-nonce".to_string(),
18960 }),
18961 )
18962 .await;
18963 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
18964
18965 handler
18966 .handle_control_frame(
18967 &target_ctx,
18968 catalog_update_with_capabilities_frame(
18969 4,
18970 CapabilityDeclarations {
18971 provides: Vec::new(),
18972 requires: Vec::new(),
18973 must_never_reach: Vec::new(),
18974 },
18975 ),
18976 )
18977 .await
18978 .expect("claim removal catalog.update succeeds");
18979 assert_eq!(
18980 forwarding.active_binding_count().unwrap(),
18981 1,
18982 "removing an attested target claim must leave the route census unchanged"
18983 );
18984 assert!(
18985 client_rx.try_recv().is_err(),
18986 "claim removal must not emit route.closed or a client route GOODBYE"
18987 );
18988 assert!(
18989 target_rx.try_recv().is_err(),
18990 "claim removal must not send the target a route GOODBYE"
18991 );
18992 }
18993
18994 #[tokio::test]
18997 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
18998 let registry = Arc::new(Registry::default());
18999 let forwarding = Arc::new(ForwardingTable::default());
19000 let supervisor = SupervisorHandle::new();
19001 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19002 let handler =
19003 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19004 .with_supervisor(supervisor);
19005 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
19006 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
19007 register_capability_manifest(
19008 &handler,
19009 &target_ctx,
19010 &mut target_rx,
19011 capability_manifest("target", &["credentials-provider/v1"], &[]),
19012 1,
19013 )
19014 .await;
19015 register_capability_manifest(
19016 &handler,
19017 &opener_ctx,
19018 &mut opener_rx,
19019 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19020 2,
19021 )
19022 .await;
19023
19024 let (_client_rx, bind) = open_route_for_capability_test(
19025 &handler,
19026 &target_ctx,
19027 &mut target_rx,
19028 742,
19029 3,
19030 "target",
19031 None,
19032 )
19033 .await;
19034 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
19035 panic!("direct scope-honesty route must bind");
19036 };
19037 assert_eq!(principal, Some(Principal::Direct));
19038 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19039 }
19040
19041 #[tokio::test]
19044 async fn must_never_reach_self_route_is_capability_forbidden() {
19045 let registry = Arc::new(Registry::default());
19046 let forwarding = Arc::new(ForwardingTable::default());
19047 let supervisor = SupervisorHandle::new();
19048 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
19049 let handler =
19050 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19051 .with_supervisor(supervisor);
19052 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
19053 register_capability_manifest(
19054 &handler,
19055 &self_ctx,
19056 &mut self_rx,
19057 capability_manifest(
19058 "self-provider",
19059 &["credentials-provider/v1"],
19060 &["credentials-provider/v1"],
19061 ),
19062 1,
19063 )
19064 .await;
19065
19066 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
19067 let replies = handler
19068 .handle_control_frame(
19069 &client_ctx,
19070 route_open_frame_with_admission_facts(
19071 2,
19072 "self-provider",
19073 unique_project_root("admission-facts"),
19074 Some(ConsumerIdentity {
19075 module_id: "self-provider".to_string(),
19076 launch_nonce: "self-nonce".to_string(),
19077 }),
19078 None,
19079 ),
19080 )
19081 .await
19082 .expect("self-route refusal returns a typed frame");
19083 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19084 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19085 assert!(
19086 self_rx.try_recv().is_err(),
19087 "self denial must not relay route.bind"
19088 );
19089 }
19090
19091 #[test]
19092 fn unsupported_channel_zero_frame_returns_error() {
19093 let handler = ControlHandler::default();
19094 let request = Frame::build(
19095 FrameType::Request,
19096 control_flags(),
19097 0,
19098 0,
19099 21,
19100 b"opaque".to_vec(),
19101 )
19102 .unwrap();
19103
19104 let response = handler
19105 .handle_control(ConnectionId::new(1), request)
19106 .unwrap();
19107
19108 assert_eq!(response[0].header.ty, FrameType::Error);
19109 assert_eq!(
19110 parse_error(&response[0])["code"],
19111 "unsupported_control_frame"
19112 );
19113 }
19114
19115 mod swap {
19120 use super::*;
19121
19122 const INCUMBENT: ConnectionId = ConnectionId::new(30);
19123 const CANDIDATE: ConnectionId = ConnectionId::new(40);
19124
19125 struct Swap {
19126 registry: Arc<Registry>,
19127 forwarding: Arc<ForwardingTable>,
19128 handler: ControlHandler,
19129 incumbent_ctx: RouteCtx,
19130 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19131 candidate_ctx: RouteCtx,
19132 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19133 }
19134
19135 async fn swap_with_incumbent() -> Swap {
19136 let registry = Arc::new(Registry::default());
19137 let forwarding = Arc::new(ForwardingTable::default());
19138 let handler =
19139 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
19140 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
19141 hello_via_sink(
19142 &handler,
19143 &incumbent_ctx,
19144 &mut incumbent_rx,
19145 hello_frame("aft", PROTOCOL_VERSION, 7),
19146 )
19147 .await;
19148 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
19149 Swap {
19150 registry,
19151 forwarding,
19152 handler,
19153 incumbent_ctx,
19154 incumbent_rx,
19155 candidate_ctx,
19156 candidate_rx,
19157 }
19158 }
19159
19160 fn register_candidate(swap: &Swap, ready: Option<bool>) {
19161 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
19162 candidate_manifest.ready = ready;
19163 let registration = swap
19164 .registry
19165 .register_candidate_with_control_ops(
19166 candidate_manifest,
19167 PROTOCOL_VERSION,
19168 CANDIDATE,
19169 module_baseline_control_ops(),
19170 )
19171 .unwrap();
19172 swap.forwarding
19173 .register_candidate_module_connection(
19174 CANDIDATE,
19175 "aft".to_string(),
19176 PROTOCOL_VERSION,
19177 manifest_concurrency(®istration.manifest),
19178 swap.candidate_ctx.egress.clone(),
19179 )
19180 .unwrap();
19181 }
19182
19183 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
19184 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
19185 swap.registry.promote_candidate("aft").unwrap().unwrap();
19186 cutover.incumbent.unwrap()
19187 }
19188
19189 #[tokio::test]
19198 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
19199 let mut swap = swap_with_incumbent().await;
19200 let hold = Arc::new(std::sync::Barrier::new(2));
19201 let handler = swap
19202 .handler
19203 .clone()
19204 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
19205 handler
19206 .supervisor
19207 .open_swap("aft", "candidate-nonce".to_string());
19208 let hello = {
19209 let handler = handler.clone();
19210 let candidate_ctx = swap.candidate_ctx.clone();
19211 std::thread::spawn(move || {
19212 handler
19213 .handle_hello(
19214 candidate_ctx.connection_id,
19215 Some(candidate_ctx.egress.clone()),
19216 hello_frame_with_nonce(
19217 "aft",
19218 PROTOCOL_VERSION,
19219 8,
19220 Some("candidate-nonce"),
19221 ),
19222 )
19223 .unwrap()
19224 })
19225 };
19226
19227 hold.wait();
19228 assert!(swap
19231 .registry
19232 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
19233 .unwrap()
19234 .is_some());
19235 assert!(swap
19236 .registry
19237 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19238 .unwrap()
19239 .is_none());
19240 assert!(swap
19241 .forwarding
19242 .module_endpoint_for_connection(CANDIDATE)
19243 .unwrap()
19244 .is_none());
19245
19246 hold.wait();
19247 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
19248 assert_eq!(
19249 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19250 FrameType::HelloAck
19251 );
19252 let published = swap
19253 .registry
19254 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19255 .unwrap()
19256 .expect("candidate is published once its HELLO completes");
19257 assert_eq!(published.connection_id, CANDIDATE);
19258 assert!(swap
19259 .forwarding
19260 .module_endpoint_for_connection(CANDIDATE)
19261 .unwrap()
19262 .is_some());
19263 }
19264
19265 #[tokio::test]
19266 async fn candidate_lookup_after_hello_ack_waits_for_publication() {
19267 let mut swap = swap_with_incumbent().await;
19268 let hold = Arc::new(std::sync::Barrier::new(2));
19269 swap.handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
19270 swap.handler
19271 .supervisor
19272 .open_swap("aft", "candidate-nonce".to_string());
19273 let hello = {
19274 let handler = swap.handler.clone();
19275 let candidate_ctx = swap.candidate_ctx.clone();
19276 std::thread::spawn(move || {
19277 handler
19278 .handle_hello(
19279 CANDIDATE,
19280 Some(candidate_ctx.egress),
19281 hello_frame_with_nonce(
19282 "aft",
19283 PROTOCOL_VERSION,
19284 8,
19285 Some("candidate-nonce"),
19286 ),
19287 )
19288 .unwrap()
19289 })
19290 };
19291 hold.wait();
19292 assert_eq!(
19293 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19294 FrameType::HelloAck
19295 );
19296 let reader_registry = Arc::clone(&swap.registry);
19297 let published = lookup_during_hello_publication(&swap.registry, &hold, move || {
19298 reader_registry
19299 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19300 .unwrap()
19301 });
19302 assert!(hello.join().unwrap().is_empty());
19303 assert_eq!(
19304 published
19305 .expect("ACK makes candidate publication visible")
19306 .connection_id,
19307 CANDIDATE
19308 );
19309 assert_eq!(
19310 swap.registry.generation().unwrap(),
19311 1,
19312 "candidate is not a second catalog entry"
19313 );
19314 }
19315
19316 #[tokio::test]
19317 async fn audit4_ungated_removed_candidate_publication_rolls_back_admission() {
19318 let mut swap = swap_with_incumbent().await;
19319 let hold = Arc::new(std::sync::Barrier::new(2));
19320 let handler = swap
19321 .handler
19322 .clone()
19323 .with_hello_registry_to_forwarding_hold(hold.clone());
19324 handler
19325 .supervisor
19326 .open_swap("aft", "candidate-nonce".into());
19327 let hello = {
19328 let handler = handler.clone();
19329 let ctx = swap.candidate_ctx.clone();
19330 std::thread::spawn(move || {
19331 handler
19332 .handle_hello(
19333 CANDIDATE,
19334 Some(ctx.egress),
19335 hello_frame_with_nonce(
19336 "aft",
19337 PROTOCOL_VERSION,
19338 8,
19339 Some("candidate-nonce"),
19340 ),
19341 )
19342 .unwrap()
19343 })
19344 };
19345 hold.wait();
19346 assert_eq!(
19347 swap.registry
19348 .deregister_connection(CANDIDATE)
19349 .unwrap()
19350 .len(),
19351 1
19352 );
19353 hold.wait();
19354 assert!(hello.join().unwrap().is_empty());
19355 assert_eq!(
19356 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19357 FrameType::HelloAck
19358 );
19359 assert!(swap.registry.get_candidate("aft").unwrap().is_none());
19360 assert_eq!(
19361 handler
19362 .supervisor
19363 .swap_hello_admission("aft", Some("candidate-nonce")),
19364 SwapHelloAdmission::Candidate,
19365 "failed publication must roll back candidate_admitted for an ungated nonce swap"
19366 );
19367 handler.supervisor.close_swap("aft");
19368 }
19369
19370 fn keyed_total(counters: &Value, key: &str) -> u64 {
19371 counters[key]
19372 .as_object()
19373 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
19374 .unwrap_or(0)
19375 }
19376
19377 #[tokio::test]
19378 async fn replacement_logs_old_end_and_new_admission_once() {
19379 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19380 let mut swap = swap_with_incumbent().await;
19381 swap.handler
19382 .supervisor
19383 .open_swap("aft", "candidate-nonce".to_string());
19384 hello_via_sink(
19385 &swap.handler,
19386 &swap.candidate_ctx,
19387 &mut swap.candidate_rx,
19388 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
19389 )
19390 .await;
19391 cutover(&swap);
19392 swap.handler.cleanup_connection(INCUMBENT).unwrap();
19393
19394 let captured = crate::router::test_log::captured_logs(&logs);
19395 println!("captured replacement registry logs:\n{captured}");
19396 let new_admissions: Vec<_> = captured
19397 .lines()
19398 .filter(|line| {
19399 line.contains("connection_id=40")
19400 && line.contains("swap candidate registered; not routable until cutover")
19401 })
19402 .collect();
19403 assert_eq!(
19404 new_admissions.len(),
19405 1,
19406 "unexpected admission logs: {captured}"
19407 );
19408 assert!(new_admissions[0].contains("module_id=aft"));
19409 assert!(new_admissions[0].contains("connection_id=40"));
19410 assert!(
19411 new_admissions[0].contains("swap candidate registered; not routable until cutover")
19412 );
19413 assert!(new_admissions[0].contains("ready=true"));
19414
19415 let old_admissions: Vec<_> = captured
19416 .lines()
19417 .filter(|line| {
19418 line.contains("module registered module_id=aft ")
19419 && line.contains("connection_id=30")
19420 })
19421 .collect();
19422 assert_eq!(
19423 old_admissions.len(),
19424 1,
19425 "unexpected incumbent admission: {captured}"
19426 );
19427
19428 let old_ends: Vec<_> = captured
19429 .lines()
19430 .filter(|line| {
19431 line.contains("connection_id=30") && line.contains("module registration ended")
19432 })
19433 .collect();
19434 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
19435 assert!(old_ends[0].contains("module_id=aft"));
19436 assert!(old_ends[0].contains("connection_id=30"));
19437 assert!(old_ends[0].contains("reason=replaced"));
19438 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
19439 assert_eq!(
19440 captured
19441 .lines()
19442 .filter(|line| line.contains("module registration promoted"))
19443 .count(),
19444 1,
19445 "unexpected promotion logs: {captured}"
19446 );
19447 }
19448
19449 #[tokio::test]
19455 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
19456 let mut swap = swap_with_incumbent().await;
19457 let handler = swap.handler.clone();
19458
19459 let cotenant = ConnectionId::new(31);
19461 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
19462 let (cotenant_task, cotenant_bind) = relay_route_open(
19463 &handler,
19464 cotenant,
19465 &cotenant_ctx.egress,
19466 &mut swap.incumbent_rx,
19467 100,
19468 "aft",
19469 "swap-cotenant",
19470 )
19471 .await;
19472 handler
19473 .handle_control_frame(
19474 &swap.incumbent_ctx,
19475 route_bind_ack(cotenant_bind.header.corr),
19476 )
19477 .await
19478 .unwrap();
19479 assert!(cotenant_task.await.unwrap().is_empty());
19480 let (cotenant_channel, cotenant_epoch) =
19481 published_route(&cotenant_rx.recv().await.unwrap());
19482
19483 let caller = ConnectionId::new(32);
19485 let (caller_ctx, mut caller_rx) = route_ctx(caller);
19486 let (caller_task, caller_bind) = relay_route_open(
19487 &handler,
19488 caller,
19489 &caller_ctx.egress,
19490 &mut swap.incumbent_rx,
19491 101,
19492 "aft",
19493 "swap-caller",
19494 )
19495 .await;
19496 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
19497
19498 register_candidate(&swap, None);
19499 cutover(&swap);
19500
19501 let ack = handler
19503 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
19504 .await;
19505 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
19506 if module_loop_error.is_some() {
19507 handler.cleanup_connection(INCUMBENT).unwrap();
19510 }
19511
19512 assert!(
19514 cotenant_rx.try_recv().is_err(),
19515 "the co-tenant route on the incumbent was torn down by one late ack: \
19516 {module_loop_error:?}"
19517 );
19518 assert!(matches!(
19519 swap.forwarding
19520 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
19521 .unwrap(),
19522 DataRoute::Client(DataRouteState::Bound(_))
19523 ));
19524 assert_eq!(module_loop_error, None);
19525 assert!(swap
19526 .registry
19527 .get_module_by_connection(INCUMBENT)
19528 .unwrap()
19529 .is_some());
19530
19531 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
19533 .await
19534 .expect("the incumbent is told to drop the abandoned binding")
19535 .unwrap()
19536 .frame;
19537 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19538 assert_eq!(goodbye.header.channel, abandoned_channel);
19539 assert_eq!(goodbye.header.epoch, abandoned_epoch);
19540 assert!(swap.incumbent_rx.try_recv().is_err());
19541
19542 let response = caller_task.await.unwrap();
19544 assert_eq!(response.len(), 1);
19545 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
19546 assert!(caller_rx.try_recv().is_err());
19547
19548 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
19552 let counters = handler.counters().snapshot();
19553 assert_eq!(
19554 keyed_total(&counters, "route_open_accepted_by_principal"),
19555 1
19556 );
19557 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
19558 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
19559 }
19560
19561 #[tokio::test]
19565 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
19566 let mut swap = swap_with_incumbent().await;
19567 register_candidate(&swap, None);
19568 let incumbent = cutover(&swap);
19569 swap.forwarding
19570 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
19571 .unwrap()
19572 .expect("the incumbent is still registered");
19573
19574 let client = ConnectionId::new(33);
19575 let (client_ctx, mut client_rx) = route_ctx(client);
19576 let route_handler = swap.handler.clone();
19577 let open_ctx = RouteCtx {
19578 connection_id: client,
19579 egress: client_ctx.egress.clone(),
19580 };
19581 let mut route_task = tokio::spawn(async move {
19582 route_handler
19583 .handle_control_frame(
19584 &open_ctx,
19585 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
19586 )
19587 .await
19588 .unwrap()
19589 });
19590 let bind = tokio::select! {
19591 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
19592 response = &mut route_task => {
19593 let response = response.unwrap();
19594 panic!(
19595 "post-cutover route.open was refused instead of relayed to the candidate: {}",
19596 parse_error(&response[0])["code"]
19597 );
19598 }
19599 };
19600 swap.handler
19601 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
19602 .await
19603 .unwrap();
19604 assert!(route_task.await.unwrap().is_empty());
19605 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
19606 match swap
19607 .forwarding
19608 .lookup_data_route(client, channel, epoch)
19609 .unwrap()
19610 {
19611 DataRoute::Client(DataRouteState::Bound(route)) => {
19612 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
19613 }
19614 other => panic!("expected a bound route on the candidate, got {other:?}"),
19615 }
19616 assert!(swap.incumbent_rx.try_recv().is_err());
19617 }
19618
19619 #[tokio::test]
19624 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
19625 let swap = swap_with_incumbent().await;
19626 register_candidate(&swap, Some(false));
19627 let update = Frame::build(
19628 FrameType::Request,
19629 control_flags(),
19630 0,
19631 0,
19632 55,
19633 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
19634 provides: manifest("aft", PROTOCOL_VERSION).provides,
19635 capabilities: None,
19636 ready: Some(true),
19637 })
19638 .unwrap(),
19639 )
19640 .unwrap();
19641
19642 let replies = swap
19643 .handler
19644 .handle_control_frame(&swap.candidate_ctx, update)
19645 .await
19646 .unwrap();
19647
19648 assert_eq!(replies.len(), 1);
19649 assert_eq!(
19650 replies[0].header.ty,
19651 FrameType::Response,
19652 "candidate catalog.update was refused: {:?}",
19653 serde_json::from_slice::<Value>(&replies[0].body).ok()
19654 );
19655 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
19656 assert_eq!(
19657 swap.registry
19658 .get_module("aft")
19659 .unwrap()
19660 .unwrap()
19661 .connection_id,
19662 INCUMBENT
19663 );
19664 }
19665 }
19666
19667 mod swap_admission {
19671 use super::*;
19672
19673 const INCUMBENT_NONCE: &str = "incumbent-nonce";
19674 const CANDIDATE_NONCE: &str = "candidate-nonce";
19675
19676 fn handler_with_incumbent(
19677 module_id: &str,
19678 reserved: bool,
19679 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
19680 let registry = Arc::new(Registry::default());
19681 let supervisor = SupervisorHandle::new();
19682 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
19683 if reserved {
19684 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
19685 }
19686 let handler =
19687 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
19688 let incumbent = handler
19689 .handle_control(
19690 ConnectionId::new(1),
19691 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
19692 )
19693 .unwrap();
19694 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
19695 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
19696 (registry, supervisor, handler)
19697 }
19698
19699 #[test]
19705 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
19706 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
19707
19708 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
19709 let replies = handler
19710 .handle_control(
19711 ConnectionId::new(connection),
19712 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
19713 )
19714 .unwrap();
19715 assert_eq!(replies[0].header.ty, FrameType::Error);
19716 assert_eq!(
19717 parse_error(&replies[0])["code"],
19718 "swap_token_invalid",
19719 "nonce {nonce:?}"
19720 );
19721 }
19722 assert!(registry.get_candidate("aft").unwrap().is_none());
19723 assert_eq!(
19724 registry.get_module("aft").unwrap().unwrap().connection_id,
19725 ConnectionId::new(1)
19726 );
19727
19728 let admitted = handler
19731 .handle_control(
19732 ConnectionId::new(4),
19733 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
19734 )
19735 .unwrap();
19736 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
19737 assert_eq!(
19738 registry
19739 .get_candidate("aft")
19740 .unwrap()
19741 .unwrap()
19742 .connection_id,
19743 ConnectionId::new(4)
19744 );
19745 assert_eq!(
19746 registry.get_module("aft").unwrap().unwrap().connection_id,
19747 ConnectionId::new(1),
19748 "the candidate must not take the active slot"
19749 );
19750 let replayed = handler
19751 .handle_control(
19752 ConnectionId::new(5),
19753 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
19754 )
19755 .unwrap();
19756 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
19757
19758 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
19762 let squatter = handler
19763 .handle_control(
19764 ConnectionId::new(6),
19765 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
19766 )
19767 .unwrap();
19768 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
19769 assert!(
19770 registry.get_module("aft").unwrap().is_none(),
19771 "a squatter took the active slot of an id being swapped"
19772 );
19773 }
19774
19775 #[test]
19780 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
19781 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
19782
19783 let replies = handler
19784 .handle_control(
19785 ConnectionId::new(2),
19786 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
19787 )
19788 .unwrap();
19789
19790 assert_eq!(
19791 replies[0].header.ty,
19792 FrameType::HelloAck,
19793 "reserved candidate refused: {:?}",
19794 serde_json::from_slice::<Value>(&replies[0].body).ok()
19795 );
19796 assert_eq!(
19797 registry
19798 .get_candidate("vault")
19799 .unwrap()
19800 .unwrap()
19801 .connection_id,
19802 ConnectionId::new(2)
19803 );
19804 }
19805
19806 #[test]
19809 fn without_an_open_swap_the_ordinary_gates_decide() {
19810 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
19811 supervisor.close_swap("vault");
19812
19813 let candidate = handler
19814 .handle_control(
19815 ConnectionId::new(2),
19816 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
19817 )
19818 .unwrap();
19819 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
19820 let duplicate = handler
19821 .handle_control(
19822 ConnectionId::new(3),
19823 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
19824 )
19825 .unwrap();
19826 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
19827 assert!(registry.get_candidate("vault").unwrap().is_none());
19828 }
19829 }
19830
19831 mod scopes {
19835 mod idle_tests {
19836 use super::*;
19837 include!("scope_idle_tests.rs");
19838 }
19839 mod route_open_timings {
19840 use super::*;
19841 include!("route_open_timing_tests.rs");
19842 }
19843 mod route_open_refusal_log {
19844 use super::*;
19845 include!("route_open_refusal_log_tests.rs");
19846 }
19847 mod activity {
19848 include!("activity_scope_tests.rs");
19849 }
19850 mod delta {
19851 include!("scope_handler_delta_tests.rs");
19852 }
19853 mod call_modes {
19854 include!("scope_call_mode_tests.rs");
19855 }
19856 mod federated {
19857 include!("scope_federated_tests.rs");
19858 }
19859 mod authorize {
19860 include!("route_authorize_tests.rs");
19861 }
19862 #[tokio::test]
19863 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
19864 let rig = rig().await;
19865 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
19866 "s".into(),
19867 1,
19868 "tool".into(),
19869 ))
19870 .unwrap();
19871 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
19872 let (direct, _rx) = wide_ctx(999);
19873 for (ctx, expected) in [
19874 (&rig.owner, FrameType::Response),
19875 (&direct, FrameType::Error),
19876 ] {
19877 let frame =
19878 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
19879 .unwrap();
19880 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
19881 assert_eq!(replies[0].header.ty, expected);
19882 if expected == FrameType::Response {
19883 let reply: subc_control::ScopeServedByReply =
19884 serde_json::from_slice(&replies[0].body).unwrap();
19885 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
19886 } else {
19887 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
19888 }
19889 }
19890 let reply = call(
19891 &rig.handler,
19892 &rig.owner,
19893 &ModuleControlRequestFromModule::ScopeDescribe {
19894 owner: Principal::Reserved {
19895 module_id: OWNER.into(),
19896 },
19897 scope_ref: "s".into(),
19898 },
19899 )
19900 .await;
19901 assert!(matches!(
19902 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
19903 ModuleControlResponseToModule::ScopeDescribe { .. }
19904 ));
19905 }
19906
19907 use subc_protocol::scope::{
19908 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
19909 ScopeStatus,
19910 };
19911
19912 use super::*;
19913
19914 const OWNER: &str = "prefrontal-core";
19915
19916 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
19917 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
19918 }
19919
19920 async fn call(
19921 handler: &ControlHandler,
19922 ctx: &RouteCtx,
19923 request: &ModuleControlRequestFromModule,
19924 ) -> Frame {
19925 let body = serde_json::to_vec(request).unwrap();
19926 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
19927 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
19928 assert_eq!(replies.len(), 1, "{replies:?}");
19929 replies.pop().unwrap()
19930 }
19931
19932 async fn sync(
19933 handler: &ControlHandler,
19934 ctx: &RouteCtx,
19935 generation: u64,
19936 scopes: Vec<ScopeRecord>,
19937 ) -> Result<ModuleControlResponseToModule, String> {
19938 let reply = call(
19939 handler,
19940 ctx,
19941 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
19942 )
19943 .await;
19944 match reply.header.ty {
19945 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
19946 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
19947 }
19948 }
19949
19950 async fn describe(
19951 handler: &ControlHandler,
19952 ctx: &RouteCtx,
19953 owner: &str,
19954 scope_ref: &str,
19955 ) -> ModuleControlResponseToModule {
19956 let reply = call(
19957 handler,
19958 ctx,
19959 &ModuleControlRequestFromModule::ScopeDescribe {
19960 owner: Principal::Reserved {
19961 module_id: owner.to_string(),
19962 },
19963 scope_ref: scope_ref.to_string(),
19964 },
19965 )
19966 .await;
19967 assert_eq!(
19968 reply.header.ty,
19969 FrameType::Response,
19970 "{:?}",
19971 parse_error(&reply)
19972 );
19973 serde_json::from_slice(&reply.body).unwrap()
19974 }
19975
19976 #[tokio::test]
19977 async fn scope_describe_never_echoes_bind_proof_or_hash() {
19978 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
19979 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19980 let mut rig = rig().await;
19981 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19982 .await;
19983 let _bound = rig
19984 .bound(
19985 Some(AFT),
19986 PLEXUS,
19987 Some(
19988 rig.selector("s", Some(1))
19989 .with_bind_proof(Some(PROOF.into())),
19990 ),
19991 )
19992 .await;
19993 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
19994 let output = serde_json::to_string(&reply).unwrap();
19995 assert!(output.contains("\"status\":\"live\""));
19996 assert!(!output.contains(PROOF));
19997 assert!(!output.contains(HASH));
19998 }
19999
20000 async fn module(
20002 handler: &ControlHandler,
20003 connection: u64,
20004 module_id: &str,
20005 nonce: Option<&str>,
20006 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20007 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
20008 hello_via_sink(
20009 handler,
20010 &ctx,
20011 &mut rx,
20012 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
20013 )
20014 .await;
20015 (ctx, rx)
20016 }
20017
20018 #[tokio::test]
20021 async fn a_client_connection_cannot_sync_or_describe() {
20022 let handler = ControlHandler::new(Arc::new(Registry::default()));
20023 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
20024 for request in [
20025 ModuleControlRequestFromModule::ScopeSync {
20026 generation: 1,
20027 scopes: vec![head("s", 1)],
20028 },
20029 ModuleControlRequestFromModule::ScopeDescribe {
20030 owner: Principal::Direct,
20031 scope_ref: "s".to_string(),
20032 },
20033 ] {
20034 let reply = call(&handler, &ctx, &request).await;
20035 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
20036 }
20037 assert!(
20038 !handler
20039 .scopes
20040 .read()
20041 .unwrap()
20042 .describe(
20043 &Principal::Reserved {
20044 module_id: OWNER.to_string()
20045 },
20046 "s"
20047 )
20048 .owner_synced
20049 );
20050 }
20051
20052 #[tokio::test]
20055 async fn a_module_without_a_supervised_launch_cannot_sync() {
20056 let handler = ControlHandler::new(Arc::new(Registry::default()));
20057 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
20058 assert_eq!(
20059 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
20060 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20061 );
20062 }
20063
20064 #[tokio::test]
20065 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
20066 let supervisor = SupervisorHandle::new();
20067 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20068 let handler = ControlHandler::new(Arc::new(Registry::default()))
20069 .with_supervisor(supervisor.clone());
20070 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20071 sync(&handler, &incumbent, 1, vec![head("s", 1)])
20072 .await
20073 .expect("the current launch syncs");
20074
20075 supervisor.open_swap(OWNER, "n2".to_string());
20078 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
20079 assert_eq!(
20080 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20081 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20082 );
20083 sync(&handler, &incumbent, 2, vec![head("s", 1)])
20084 .await
20085 .expect("the serving owner syncs during the swap");
20086
20087 supervisor.close_swap(OWNER);
20090 assert_eq!(
20091 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20092 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20093 );
20094 sync(&handler, &incumbent, 3, vec![head("s", 1)])
20095 .await
20096 .expect("the serving owner syncs after the rollback");
20097 handler.cleanup_connection(candidate.connection_id).unwrap();
20098
20099 supervisor.open_swap(OWNER, "n3".to_string());
20104 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
20105 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
20106 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
20107 .await
20108 .expect("the promoted launch takes authority");
20109 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20110 panic!("unexpected reply {reply:?}");
20111 };
20112 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
20113 assert_eq!(
20114 sync(&handler, &incumbent, 4, Vec::new()).await,
20115 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20116 );
20117 }
20118
20119 #[tokio::test]
20122 async fn closing_the_authority_connection_frees_sync_authority() {
20123 let supervisor = SupervisorHandle::new();
20124 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20125 let handler = ControlHandler::new(Arc::new(Registry::default()))
20126 .with_supervisor(supervisor.clone());
20127 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20128 sync(&handler, &first, 10, vec![head("s", 1)])
20129 .await
20130 .unwrap();
20131 handler.cleanup_connection(first.connection_id).unwrap();
20132
20133 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
20134 sync(&handler, &second, 1, vec![head("s", 1)])
20135 .await
20136 .expect("the next connection takes the released authority");
20137 }
20138
20139 #[tokio::test]
20140 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
20141 let supervisor = SupervisorHandle::new();
20142 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20143 let handler =
20144 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
20145 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
20146 sync(&handler, &first, 10, vec![head("s", 1)])
20147 .await
20148 .unwrap();
20149 handler
20150 .handle_control_frame(
20151 &first,
20152 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
20153 )
20154 .await
20155 .unwrap();
20156 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
20157 sync(&handler, &second, 1, vec![head("s", 1)])
20158 .await
20159 .expect("GOODBYE releases authority even if the old socket remains open");
20160 }
20161
20162 #[tokio::test]
20163 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
20164 let registry = Arc::new(Registry::default());
20165 let supervisor_handle = SupervisorHandle::new();
20166 let supervisor =
20167 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20168 .with_handle(supervisor_handle.clone())
20169 .with_daemon_incarnation("incarnation-7".to_string());
20170 supervisor
20173 .supervise_configured(
20174 ModuleSpec {
20175 carry_containment: false,
20176 module_id: OWNER.to_string(),
20177 program: PathBuf::from("/nonexistent/prefrontal-core"),
20178 args: Vec::new(),
20179 env: Vec::new(),
20180 reserved: false,
20181 reserved_prefixes: Vec::new(),
20182 protocol: ModuleProtocol::Subc,
20183 overlap: Default::default(),
20184 },
20185 false,
20186 )
20187 .unwrap();
20188 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
20189 let handler =
20190 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
20191 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
20192
20193 let ModuleControlResponseToModule::ScopeDescribe {
20195 status,
20196 daemon_incarnation,
20197 owner_synced,
20198 owner_configured,
20199 scope,
20200 ..
20201 } = describe(&handler, &reader, OWNER, "s").await
20202 else {
20203 panic!("not a describe reply");
20204 };
20205 assert_eq!(status, ScopeStatus::NotLive);
20206 assert_eq!(daemon_incarnation, "incarnation-7");
20207 assert!(!owner_synced);
20208 assert!(owner_configured);
20209 assert!(scope.is_none());
20210
20211 let ModuleControlResponseToModule::ScopeDescribe {
20214 status,
20215 owner_configured,
20216 ..
20217 } = describe(&handler, &reader, "ghost", "s").await
20218 else {
20219 panic!("not a describe reply");
20220 };
20221 assert_eq!(status, ScopeStatus::NotLive);
20222 assert!(!owner_configured);
20223
20224 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
20226 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
20227 let ModuleControlResponseToModule::ScopeDescribe {
20228 status,
20229 scope_epoch,
20230 owner_synced,
20231 scope,
20232 ..
20233 } = describe(&handler, &reader, OWNER, "s").await
20234 else {
20235 panic!("not a describe reply");
20236 };
20237 assert_eq!(status, ScopeStatus::Live);
20238 assert_eq!(scope_epoch, Some(4));
20239 assert!(owner_synced);
20240 let stamp = scope.expect("a live scope carries its stamp");
20241 assert!(
20242 stamp.owner_authorized,
20243 "prefrontal-core is the default authority"
20244 );
20245 assert_eq!(stamp.kind, ScopeKind::Head);
20246 }
20247
20248 #[tokio::test]
20249 async fn scope_authority_owners_decides_owner_authorized() {
20250 let supervisor = SupervisorHandle::new();
20251 supervisor.set_spawn_nonce("broca", "b1".to_string());
20252 let handler = ControlHandler::new(Arc::new(Registry::default()))
20253 .with_supervisor(supervisor)
20254 .with_scope_authority_owners(vec!["broca".to_string()]);
20255 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
20256 let mut gated = head("s", 1);
20257 gated.attributes.agent_id = Some("agent".to_string());
20258 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
20259 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
20260 describe(&handler, &broca, "broca", "s").await
20261 else {
20262 panic!("not a describe reply");
20263 };
20264 assert!(scope.unwrap().owner_authorized);
20265 }
20266
20267 #[tokio::test]
20271 async fn scope_ops_and_the_scopes_capability_are_advertised() {
20272 let handler = ControlHandler::new(Arc::new(Registry::default()));
20273 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
20274 let ack = hello_via_sink(
20275 &handler,
20276 &ctx,
20277 &mut rx,
20278 hello_frame("m", PROTOCOL_VERSION, 1),
20279 )
20280 .await;
20281 let ack = parse_ack(&ack);
20282 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
20283 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
20284 }
20285 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
20286
20287 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
20288 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
20289 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
20290 let reply = handler
20291 .handle_control_frame(&client, frame)
20292 .await
20293 .unwrap()
20294 .pop()
20295 .unwrap();
20296 let ClientControlResponse::ServerDescribe { capabilities, .. } =
20297 serde_json::from_slice(&reply.body).unwrap()
20298 else {
20299 panic!("not a server.describe reply");
20300 };
20301 assert!(
20302 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
20303 "{capabilities:?}"
20304 );
20305 }
20306
20307 const PLEXUS: &str = "plexus";
20310 const OTHER: &str = "other";
20311 const AFT: &str = "aft";
20312 const BROCA: &str = "broca";
20313 const MAGIC: &str = "magic-context";
20314
20315 fn nonce(module_id: &str) -> String {
20316 format!("nonce-{module_id}")
20317 }
20318
20319 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20320 let (tx, rx) = mpsc::channel(64);
20321 (
20322 RouteCtx {
20323 connection_id: ConnectionId::new(connection),
20324 egress: FrameSink::new(tx),
20325 },
20326 rx,
20327 )
20328 }
20329
20330 struct Rig {
20334 handler: ControlHandler,
20335 forwarding: Arc<ForwardingTable>,
20336 owner: RouteCtx,
20337 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20338 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
20339 generation: u64,
20340 next_connection: u64,
20341 _supervisor: Supervisor,
20342 }
20343
20344 async fn rig() -> Rig {
20345 rig_with_flow_support(true).await
20346 }
20347
20348 async fn rig_with_flow_support(flow_support: bool) -> Rig {
20349 rig_with_scope_capabilities(flow_support, false).await
20350 }
20351
20352 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
20353 let mut provides = Vec::new();
20354 if flow_support {
20355 provides.push("flow-scopes/v1");
20356 }
20357 if run_support {
20358 provides.push("agent-run-scopes/v1");
20359 }
20360 rig_with_target_capabilities(&provides).await
20361 }
20362
20363 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
20364 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
20365 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20366 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20369 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
20370 Frame::build(
20371 FrameType::Hello,
20372 control_flags(),
20373 0,
20374 0,
20375 connection,
20376 serde_json::to_vec(&body).unwrap(),
20377 )
20378 .unwrap()
20379 }
20380
20381 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
20382 let registry = Arc::new(Registry::default());
20383 let forwarding = Arc::new(ForwardingTable::default());
20384 let supervisor_handle = SupervisorHandle::new();
20385 let supervisor =
20386 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20387 .with_handle(supervisor_handle.clone());
20388 supervisor
20389 .supervise_configured(
20390 ModuleSpec {
20391 carry_containment: false,
20392 module_id: OWNER.to_string(),
20393 program: PathBuf::from("/nonexistent/prefrontal-core"),
20394 args: Vec::new(),
20395 env: Vec::new(),
20396 reserved: false,
20397 reserved_prefixes: Vec::new(),
20398 protocol: ModuleProtocol::Subc,
20399 overlap: Default::default(),
20400 },
20401 false,
20402 )
20403 .unwrap();
20404 for module_id in [OWNER, AFT, BROCA, MAGIC] {
20405 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
20406 }
20407 let handler =
20408 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
20409 .with_supervisor(supervisor_handle);
20410 let (owner, mut owner_rx) = wide_ctx(1);
20411 hello_via_sink(
20412 &handler,
20413 &owner,
20414 &mut owner_rx,
20415 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
20416 )
20417 .await;
20418 let mut modules = BTreeMap::new();
20419 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
20420 let (ctx, mut rx) = wide_ctx(connection);
20421 let hello = hello_with_capabilities(module_id, connection, provides);
20422 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
20423 modules.insert(module_id.to_string(), (ctx, rx));
20424 }
20425 Rig {
20426 handler,
20427 forwarding,
20428 owner,
20429 _owner_rx: owner_rx,
20430 modules,
20431 generation: 0,
20432 next_connection: 100,
20433 _supervisor: supervisor,
20434 }
20435 }
20436
20437 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
20438 ScopeCarrier::new(Principal::Reserved {
20439 module_id: module_id.to_string(),
20440 })
20441 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
20442 }
20443
20444 fn session(scope_epoch: u64) -> ScopeRecord {
20447 let mut record = head("s", scope_epoch);
20448 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
20449 record.attributes.agent_id = Some("agent-1".to_string());
20450 record.attributes.delegates = true;
20451 record
20452 }
20453
20454 impl Rig {
20455 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
20456 self.generation += 1;
20457 sync(&self.handler, &self.owner, self.generation, scopes)
20458 .await
20459 .expect("the owner's sync is accepted");
20460 }
20461
20462 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
20463 ScopeSelector::new(
20464 Principal::Reserved {
20465 module_id: OWNER.to_string(),
20466 },
20467 scope_ref.to_string(),
20468 scope_epoch,
20469 )
20470 }
20471
20472 fn open_frame(
20473 &mut self,
20474 opener: Option<&str>,
20475 target: &str,
20476 scope: Option<ScopeSelector>,
20477 ) -> (
20478 RouteCtx,
20479 mpsc::Receiver<crate::router::OutboundFrame>,
20480 Frame,
20481 ) {
20482 self.next_connection += 1;
20483 let (ctx, rx) = wide_ctx(self.next_connection);
20484 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
20485 busy_refusals: None,
20486 target: RouteTarget::ToolProvider {
20487 module_id: target.to_string(),
20488 },
20489 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
20490 consumer_identity: opener.map(|module_id| ConsumerIdentity {
20491 module_id: module_id.to_string(),
20492 launch_nonce: nonce(module_id),
20493 }),
20494 consumer_capabilities: None,
20495 role_versions: None,
20496 admission_facts: None,
20497 scope,
20498 })
20499 .unwrap();
20500 let frame = Frame::build(
20501 FrameType::Request,
20502 control_flags(),
20503 0,
20504 0,
20505 self.next_connection,
20506 body,
20507 )
20508 .unwrap();
20509 (ctx, rx, frame)
20510 }
20511
20512 async fn refused(
20514 &mut self,
20515 opener: Option<&str>,
20516 target: &str,
20517 scope: Option<ScopeSelector>,
20518 ) -> String {
20519 self.refusal_body(opener, target, scope).await["code"]
20520 .as_str()
20521 .unwrap()
20522 .to_string()
20523 }
20524
20525 async fn refusal_body(
20526 &mut self,
20527 opener: Option<&str>,
20528 target: &str,
20529 scope: Option<ScopeSelector>,
20530 ) -> Value {
20531 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
20532 let replies = tokio::time::timeout(
20533 Duration::from_secs(2),
20534 self.handler.handle_control_frame(&ctx, frame),
20535 )
20536 .await
20537 .expect("the open must be refused before waiting for a bind ack")
20538 .unwrap();
20539 assert_eq!(replies.len(), 1, "{replies:?}");
20540 assert_eq!(replies[0].header.ty, FrameType::Error);
20541 let (_, module_rx) = self.modules.get_mut(target).unwrap();
20542 assert!(
20543 module_rx.try_recv().is_err(),
20544 "a refused open relays nothing"
20545 );
20546 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
20547 parse_error(&replies[0])
20548 }
20549
20550 async fn relayed(
20552 &mut self,
20553 opener: Option<&str>,
20554 target: &str,
20555 scope: Option<ScopeSelector>,
20556 ) -> Relayed {
20557 self.relayed_with_legacy_root(opener, target, scope, None)
20558 .await
20559 }
20560
20561 async fn relayed_with_legacy_root(
20562 &mut self,
20563 opener: Option<&str>,
20564 target: &str,
20565 scope: Option<ScopeSelector>,
20566 legacy_root: Option<&std::path::Path>,
20567 ) -> Relayed {
20568 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
20569 if let Some(root) = legacy_root {
20570 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
20571 body["identity"]["project_root"] = serde_json::json!(root);
20572 frame.body = serde_json::to_vec(&body).unwrap();
20573 }
20574 let handler = self.handler.clone();
20575 let task_ctx = ctx.clone();
20576 let task = tokio::spawn(async move {
20577 handler
20578 .handle_control_frame(&task_ctx, frame)
20579 .await
20580 .unwrap()
20581 });
20582 let (_, module_rx) = self.modules.get_mut(target).unwrap();
20583 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
20584 .await
20585 .expect("the target receives the relayed route.bind")
20586 .unwrap()
20587 .frame;
20588 Relayed {
20589 target: target.to_string(),
20590 client: ctx,
20591 client_rx: rx,
20592 task,
20593 bind,
20594 }
20595 }
20596
20597 async fn ack(&self, relayed: &Relayed) {
20598 let (module, _) = &self.modules[&relayed.target];
20599 self.handler
20600 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
20601 .await
20602 .unwrap();
20603 }
20604
20605 async fn bound(
20607 &mut self,
20608 opener: Option<&str>,
20609 target: &str,
20610 scope: Option<ScopeSelector>,
20611 ) -> Bound {
20612 let relayed = self.relayed(opener, target, scope).await;
20613 self.commit_relayed(relayed).await
20614 }
20615
20616 async fn bound_with_legacy_root(
20617 &mut self,
20618 opener: Option<&str>,
20619 target: &str,
20620 scope: Option<ScopeSelector>,
20621 root: &std::path::Path,
20622 ) -> Bound {
20623 let relayed = self
20624 .relayed_with_legacy_root(opener, target, scope, Some(root))
20625 .await;
20626 self.commit_relayed(relayed).await
20627 }
20628
20629 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
20630 self.ack(&relayed).await;
20631 let Relayed {
20632 target,
20633 client,
20634 mut client_rx,
20635 task,
20636 bind,
20637 } = relayed;
20638 assert!(
20639 task.await.unwrap().is_empty(),
20640 "the open is answered by commit"
20641 );
20642 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
20643 Bound {
20644 target,
20645 client,
20646 client_rx,
20647 channel,
20648 epoch,
20649 bind,
20650 }
20651 }
20652
20653 fn live(&self, route: &Bound) -> bool {
20654 matches!(
20655 self.forwarding
20656 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
20657 .unwrap(),
20658 DataRoute::Client(DataRouteState::Bound(_))
20659 )
20660 }
20661 }
20662
20663 struct Relayed {
20664 target: String,
20665 client: RouteCtx,
20666 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20667 task: tokio::task::JoinHandle<Vec<Frame>>,
20668 bind: Frame,
20669 }
20670
20671 struct Bound {
20672 target: String,
20673 client: RouteCtx,
20674 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20675 channel: u16,
20676 epoch: u32,
20677 bind: Frame,
20678 }
20679
20680 impl Bound {
20681 fn closed_reason(&mut self) -> RouteCloseReason {
20684 let mut reason = None;
20685 let mut goodbye = false;
20686 while let Ok(outbound) = self.client_rx.try_recv() {
20687 let frame = outbound.frame;
20688 match frame.header.ty {
20689 FrameType::Goodbye => {
20690 assert_eq!(
20691 (frame.header.channel, frame.header.epoch),
20692 (self.channel, self.epoch)
20693 );
20694 goodbye = true;
20695 }
20696 FrameType::Push => {
20697 let ClientControlPush::RouteClosed {
20698 reason: r,
20699 module_id,
20700 ..
20701 } = serde_json::from_slice(&frame.body).unwrap()
20702 else {
20703 panic!("unexpected push");
20704 };
20705 assert_eq!(module_id, self.target);
20706 reason = Some(r);
20707 }
20708 other => panic!("unexpected frame {other:?}"),
20709 }
20710 }
20711 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
20712 reason.expect("the client is told why the route closed")
20713 }
20714
20715 fn untouched(&mut self) -> bool {
20716 self.client_rx.try_recv().is_err()
20717 }
20718
20719 fn stamp(&self) -> Option<ScopeStamp> {
20720 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
20721 ModuleControlRequest::RouteBind { scope, .. } => scope,
20722 other => panic!("expected a route.bind, got {other:?}"),
20723 }
20724 }
20725 }
20726
20727 #[tokio::test]
20728 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
20729 ) {
20730 let mut rig = rig().await;
20731 let mut record = session(1);
20732 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
20733 record.child_owners = vec![Principal::Reserved {
20734 module_id: MAGIC.to_string(),
20735 }];
20736 rig.sync(vec![record]).await;
20737 let scope = || Some(rig_selector("s", Some(1)));
20738
20739 rig.bound(Some(OWNER), PLEXUS, scope()).await;
20742 rig.bound(Some(AFT), OTHER, scope()).await;
20743 rig.bound(Some(BROCA), PLEXUS, scope()).await;
20744
20745 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
20749 assert_eq!(
20750 rig.refused(opener, target, scope()).await,
20751 error_codes::SCOPE_NOT_CARRIER,
20752 "{opener:?} -> {target}"
20753 );
20754 }
20755 }
20756
20757 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
20758 ScopeSelector::new(
20759 Principal::Reserved {
20760 module_id: OWNER.to_string(),
20761 },
20762 scope_ref.to_string(),
20763 scope_epoch,
20764 )
20765 }
20766
20767 #[tokio::test]
20768 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
20769 let mut rig = rig().await;
20770 rig.sync(vec![session(1)]).await;
20771 for opener in [OWNER, AFT] {
20772 assert_eq!(
20773 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
20774 .await,
20775 error_codes::SCOPE_EPOCH_REQUIRED,
20776 "{opener}"
20777 );
20778 }
20779 }
20780
20781 #[tokio::test]
20782 async fn admission_separates_not_synced_not_live_and_ended() {
20783 let mut rig = rig().await;
20784 let code = rig
20786 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
20787 .await;
20788 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
20789 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
20790
20791 let ghost = ScopeSelector::new(
20793 Principal::Reserved {
20794 module_id: "ghost".to_string(),
20795 },
20796 "s".to_string(),
20797 Some(1),
20798 );
20799 assert_eq!(
20800 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
20801 error_codes::SCOPE_NOT_LIVE
20802 );
20803
20804 rig.sync(vec![session(2)]).await;
20805 assert_eq!(
20806 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
20807 .await,
20808 error_codes::SCOPE_NOT_LIVE
20809 );
20810 for epoch in [1, 3] {
20811 assert_eq!(
20812 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
20813 .await,
20814 error_codes::SCOPE_ENDED,
20815 "epoch {epoch}"
20816 );
20817 }
20818 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
20820 .await;
20821 }
20822
20823 #[tokio::test]
20824 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
20825 let mut rig = rig().await;
20826 rig.sync(vec![session(1)]).await;
20827 let route = rig
20828 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
20829 .await;
20830 let stamp = route.stamp().expect("a scoped bind carries the stamp");
20831 assert_eq!(stamp.scope_ref, "s");
20832 assert_eq!(stamp.scope_epoch, 1);
20833 assert_eq!(stamp.kind, ScopeKind::Head);
20834 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
20835 assert!(stamp.attributes.delegates);
20836 assert!(stamp.owner_authorized);
20837 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
20838 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
20839
20840 let (broca, mut broca_rx) = wide_ctx(50);
20843 hello_via_sink(
20844 &rig.handler,
20845 &broca,
20846 &mut broca_rx,
20847 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
20848 )
20849 .await;
20850 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
20851 .await
20852 .unwrap();
20853 let own = ScopeSelector::new(
20854 Principal::Reserved {
20855 module_id: BROCA.to_string(),
20856 },
20857 "b".to_string(),
20858 Some(1),
20859 );
20860 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
20861 assert!(!route.stamp().unwrap().owner_authorized);
20862 }
20863
20864 #[tokio::test]
20865 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
20866 let mut rig = rig().await;
20867 let mut record = head("s", 1);
20868 record.carriers = vec![carrier(AFT, None)];
20869 let flow_id = "Flow:run-7/step_2!~";
20870 record.attributes.flow_id = Some(flow_id.to_string());
20871 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
20872 .await
20873 .unwrap();
20874 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20875 panic!("not a sync reply");
20876 };
20877 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
20878 let route = rig
20879 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
20880 .await;
20881 assert!(rig.live(&route), "the stamped bind committed");
20882 let stamp = route.stamp().expect("a flow scope carries a stamp");
20883 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
20884 assert_eq!(stamp.attributes.agent_id, None);
20885 assert!(!stamp.attributes.delegates);
20886 assert!(stamp.owner_authorized);
20887 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
20888 assert_eq!(unscoped.stamp(), None);
20889 }
20890
20891 #[tokio::test]
20892 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
20893 let mut rig = rig_with_flow_support(false).await;
20894 let mut record = session(1);
20895 record.attributes.flow_id = Some("flow:7".to_string());
20896 rig.sync(vec![record]).await;
20897 for opener in [OWNER, AFT] {
20898 let body = rig
20899 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
20900 .await;
20901 assert_eq!(body["code"], "target_flow_unsupported");
20902 let message = body["message"].as_str().unwrap();
20903 for required in [PLEXUS, "flow-scopes/v1"] {
20904 assert!(message.contains(required), "{message}");
20905 }
20906 }
20907 }
20908
20909 #[tokio::test]
20910 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
20911 let mut rig = rig_with_flow_support(true).await;
20912 let mut record = session(1);
20913 record.attributes.flow_id = Some("flow:7".to_string());
20914 rig.sync(vec![record]).await;
20915 for opener in [OWNER, AFT] {
20916 let route = rig
20917 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
20918 .await;
20919 assert!(rig.live(&route));
20920 assert_eq!(
20921 route.stamp().unwrap().attributes.flow_id.as_deref(),
20922 Some("flow:7")
20923 );
20924 }
20925 }
20926
20927 #[tokio::test]
20928 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
20929 use std::future::Future;
20930
20931 let mut rig = rig_with_flow_support(true).await;
20932 let mut record = session(1);
20933 record.attributes.flow_id = Some("flow:7".to_string());
20934 rig.sync(vec![record]).await;
20935 let (client, mut client_rx, frame) =
20936 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
20937 for _ in 0..64 {
20940 client.egress.try_send(route_bind_ack(1)).unwrap();
20941 }
20942 let handler = rig.handler.clone();
20943 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
20944 std::future::poll_fn(|cx| {
20945 assert!(open.as_mut().poll(cx).is_pending());
20946 std::task::Poll::Ready(())
20947 })
20948 .await;
20949 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
20950
20951 let old_connection = rig.modules[PLEXUS].0.connection_id;
20952 rig.handler.cleanup_connection(old_connection).unwrap();
20953 let (replacement, mut replacement_rx) = wide_ctx(200);
20954 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
20955 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20956 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20957 let hello = Frame::build(
20958 FrameType::Hello,
20959 control_flags(),
20960 0,
20961 0,
20962 200,
20963 serde_json::to_vec(&body).unwrap(),
20964 )
20965 .unwrap();
20966 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
20967
20968 client_rx.try_recv().unwrap();
20969 let replies = tokio::time::timeout(Duration::from_secs(2), open)
20970 .await
20971 .expect("the replacement is refused without waiting for a bind ack")
20972 .unwrap();
20973 assert_eq!(replies.len(), 1);
20974 let body = parse_error(&replies[0]);
20975 assert_eq!(body["code"], "target_flow_unsupported");
20976 for required in [PLEXUS, "flow-scopes/v1"] {
20977 assert!(body["message"].as_str().unwrap().contains(required));
20978 }
20979 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
20980 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
20981 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
20982 assert!(rig
20983 .handler
20984 .registry
20985 .get_module_by_connection(replacement.connection_id)
20986 .unwrap()
20987 .is_some());
20988 }
20989
20990 #[tokio::test]
20991 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
20992 use std::future::Future;
20993 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
20994 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
20995 const NEXT_HASH: &str =
20996 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
20997 let mut rig = rig().await;
20998 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
20999 record.carriers.push(ScopeCarrier::new(Principal::Direct));
21000 rig.sync(vec![record.clone()]).await;
21001 let (client, mut rx, frame) = rig.open_frame(
21002 None,
21003 PLEXUS,
21004 Some(
21005 rig.selector("s", Some(1))
21006 .with_bind_proof(Some(PROOF.into())),
21007 ),
21008 );
21009 for _ in 0..64 {
21012 client.egress.try_send(route_bind_ack(1)).unwrap();
21013 }
21014 let handler = rig.handler.clone();
21015 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21016 std::future::poll_fn(|cx| {
21017 assert!(open.as_mut().poll(cx).is_pending());
21018 std::task::Poll::Ready(())
21019 })
21020 .await;
21021 record.bind_proof_sha256 = Some(NEXT_HASH.into());
21022 rig.sync(vec![record]).await;
21023 rx.try_recv().unwrap();
21024 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21025 .await
21026 .unwrap()
21027 .unwrap();
21028 assert_eq!(
21029 parse_error(&replies[0])["code"],
21030 error_codes::SCOPE_BIND_PROOF_MISMATCH
21031 );
21032 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
21033 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21034 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21035 }
21036
21037 #[tokio::test]
21038 async fn scope_bind_proof_reserved_openers_ignore_proof() {
21039 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21040 let mut rig = rig().await;
21041 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21042 .await;
21043 for opener in [OWNER, AFT] {
21044 for proof in [None, Some("malformed+proof=")] {
21045 let route = rig
21046 .bound(
21047 Some(opener),
21048 PLEXUS,
21049 Some(
21050 rig.selector("s", Some(1))
21051 .with_bind_proof(proof.map(str::to_string)),
21052 ),
21053 )
21054 .await;
21055 assert!(rig.live(&route));
21056 }
21057 }
21058 }
21059
21060 #[tokio::test]
21061 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
21062 use std::future::Future;
21063 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21064 const NEXT_HASH: &str =
21065 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21066 let mut rig = rig().await;
21067 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21068 .await;
21069 let (client, mut rx, frame) =
21070 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
21071 for _ in 0..64 {
21072 client.egress.try_send(route_bind_ack(1)).unwrap();
21073 }
21074 let handler = rig.handler.clone();
21075 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21076 std::future::poll_fn(|cx| {
21077 assert!(open.as_mut().poll(cx).is_pending());
21078 std::task::Poll::Ready(())
21079 })
21080 .await;
21081 rig.sync(vec![
21082 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
21083 ])
21084 .await;
21085 rx.try_recv().unwrap();
21086 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21087 let bind = std::future::poll_fn(|cx| {
21088 assert!(open.as_mut().poll(cx).is_pending());
21089 module_rx.poll_recv(cx)
21090 })
21091 .await
21092 .unwrap()
21093 .frame;
21094 handler
21095 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
21096 .await
21097 .unwrap();
21098 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21099 .await
21100 .unwrap()
21101 .unwrap();
21102 assert_eq!(replies.len(), 1);
21103 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
21106 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21107 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
21108 let route = rig
21109 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
21110 .await;
21111 assert!(rig.live(&route));
21112 }
21113
21114 #[tokio::test]
21115 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
21116 {
21117 let mut rig = rig_with_flow_support(false).await;
21118 rig.sync(vec![session(1)]).await;
21119 let route = rig
21120 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21121 .await;
21122 assert!(rig.live(&route));
21123 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
21124 let mut record = session(1);
21125 record.attributes.flow_id = Some("flow:7".to_string());
21126 rig.sync(vec![record]).await;
21127 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
21128 assert!(rig.live(&unscoped));
21129 assert_eq!(unscoped.stamp(), None);
21130 }
21131
21132 #[tokio::test]
21133 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
21134 let mut rig = rig().await;
21135 let mut record = session(1);
21136 record.attributes.flow_id = Some("flow:7".to_string());
21137 rig.sync(vec![record.clone()]).await;
21138 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21139 let mut owner_route = rig
21140 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21141 .await;
21142 let mut carrier_route = rig
21143 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
21144 .await;
21145 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21146 rig.sync(vec![record.clone()]).await;
21147 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21148 assert!(rig.live(&owner_route) && owner_route.untouched());
21149 assert!(rig.live(&carrier_route) && carrier_route.untouched());
21150
21151 record.attributes.flow_id = Some("flow:8".to_string());
21152 rig.sync(vec![record]).await;
21153 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
21154 let before = before.unwrap();
21155 assert_eq!(after.scope_epoch, before.scope_epoch);
21156 assert!(after.version > before.version);
21157 for route in [&mut owner_route, &mut carrier_route] {
21158 assert!(!rig.live(route));
21159 assert_eq!(
21160 route.closed_reason(),
21161 RouteCloseReason::ScopeDelegationChanged
21162 );
21163 }
21164 assert!(rig.live(&unscoped) && unscoped.untouched());
21165 for target in [PLEXUS, OTHER] {
21168 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
21169 let goodbye = module_rx
21170 .try_recv()
21171 .expect("the provider sees the drain")
21172 .frame;
21173 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21174 assert!(module_rx.try_recv().is_err());
21175 }
21176 let rebound = rig
21177 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21178 .await;
21179 assert_eq!(
21180 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
21181 Some("flow:8")
21182 );
21183 }
21184
21185 #[tokio::test]
21190 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
21191 let mut rig = rig().await;
21192 rig.sync(vec![session(1)]).await;
21193 let mut cotenant = rig
21194 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21195 .await;
21196
21197 let mut changed = session(1);
21198 changed.child_owners.push(Principal::Reserved {
21199 module_id: MAGIC.to_string(),
21200 });
21201 let mut ended = None;
21202 for (code, next) in [
21203 (error_codes::SCOPE_CHANGED, vec![changed]),
21204 (error_codes::SCOPE_ENDED, Vec::new()),
21205 ] {
21206 let relayed = rig
21207 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21208 .await;
21209 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
21210 ended = Some(next.is_empty());
21211 rig.sync(next).await;
21212 rig.ack(&relayed).await;
21213 let replies = relayed.task.await.unwrap();
21214 assert_eq!(replies.len(), 1, "{replies:?}");
21215 assert_eq!(parse_error(&replies[0])["code"], code);
21216 assert_eq!(
21217 subc_protocol::error_codes::is_retryable_route_open(code),
21218 code == error_codes::SCOPE_CHANGED
21219 );
21220 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21221 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21223 let mut goodbyes = Vec::new();
21226 while let Ok(outbound) = plexus_rx.try_recv() {
21227 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
21228 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
21229 }
21230 assert!(
21231 goodbyes.contains(&(bind_channel, bind_epoch)),
21232 "{goodbyes:?}"
21233 );
21234 assert!(rig
21235 .handler
21236 .registry
21237 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
21238 .unwrap()
21239 .is_some());
21240 }
21241 assert_eq!(ended, Some(true));
21242 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
21245 }
21246
21247 #[tokio::test]
21250 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
21251 struct Case {
21252 name: &'static str,
21253 change: fn(&mut ScopeRecord),
21254 closed: [Option<RouteCloseReason>; 4],
21257 }
21258 use RouteCloseReason::*;
21259 let cases = [
21260 Case {
21261 name: "a carrier entry removed",
21262 change: |r| {
21263 r.carriers.retain(|c| {
21264 c.principal
21265 != Principal::Reserved {
21266 module_id: AFT.to_string(),
21267 }
21268 })
21269 },
21270 closed: [None, Some(ScopeCarrierRemoved), None, None],
21271 },
21272 Case {
21273 name: "a target removed from a carrier",
21274 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
21275 closed: [None, None, None, Some(ScopeCarrierRemoved)],
21276 },
21277 Case {
21278 name: "a bare carrier narrowed to targets",
21279 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
21280 closed: [None, Some(ScopeCarrierRemoved), None, None],
21281 },
21282 Case {
21283 name: "delegates turned off",
21284 change: |r| r.attributes.delegates = false,
21285 closed: [Some(ScopeDelegationChanged); 4],
21286 },
21287 Case {
21288 name: "agent_id changed",
21289 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
21290 closed: [Some(ScopeDelegationChanged); 4],
21291 },
21292 Case {
21293 name: "a carrier added, child owners changed, the record re-sent",
21294 change: |r| {
21295 r.carriers.push(carrier(MAGIC, None));
21296 r.child_owners.push(Principal::Reserved {
21297 module_id: MAGIC.to_string(),
21298 });
21299 },
21300 closed: [None; 4],
21301 },
21302 Case {
21303 name: "a target added",
21304 change: |r| {
21305 r.carriers[1]
21306 .targets
21307 .as_mut()
21308 .unwrap()
21309 .push("third".to_string())
21310 },
21311 closed: [None; 4],
21312 },
21313 Case {
21314 name: "delegates turned on",
21315 change: |r| r.attributes.delegates = true,
21316 closed: [None; 4],
21317 },
21318 ];
21319 for case in cases {
21320 let mut rig = rig().await;
21321 rig.sync(vec![session(1)]).await;
21322 let scope = || Some(rig_selector("s", Some(1)));
21323 let mut routes = [
21324 rig.bound(Some(OWNER), PLEXUS, scope()).await,
21325 rig.bound(Some(AFT), PLEXUS, scope()).await,
21326 rig.bound(Some(BROCA), PLEXUS, scope()).await,
21327 rig.bound(Some(BROCA), OTHER, scope()).await,
21328 ];
21329 let mut record = session(1);
21330 (case.change)(&mut record);
21331 rig.sync(vec![record]).await;
21332 for (index, expected) in case.closed.iter().enumerate() {
21333 let route = &mut routes[index];
21334 match expected {
21335 Some(reason) => {
21336 assert!(!rig.live(route), "{}: route {index} still live", case.name);
21337 assert_eq!(
21338 route.closed_reason(),
21339 *reason,
21340 "{}: route {index}",
21341 case.name
21342 );
21343 }
21344 None => {
21345 assert!(rig.live(route), "{}: route {index} closed", case.name);
21346 assert!(
21347 route.untouched(),
21348 "{}: route {index} was told something",
21349 case.name
21350 );
21351 }
21352 }
21353 }
21354 }
21355 }
21356
21357 #[tokio::test]
21358 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
21359 for next in [Vec::new(), vec![session(2)]] {
21361 let mut rig = rig().await;
21362 rig.sync(vec![session(1)]).await;
21363 let mut route = rig
21364 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21365 .await;
21366 rig.sync(next).await;
21367 assert!(!rig.live(&route));
21368 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
21369 }
21370
21371 let mut rig = rig().await;
21374 let mut child = session(1);
21375 child.scope_ref = "child".to_string();
21376 child.kind = ScopeKind::Worker;
21377 child.parent = Some(ScopeParent::new(
21378 Principal::Reserved {
21379 module_id: OWNER.to_string(),
21380 },
21381 "s".to_string(),
21382 1,
21383 ));
21384 rig.sync(vec![session(1), child.clone()]).await;
21385 let mut child_route = rig
21386 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
21387 .await;
21388 assert_eq!(
21389 child_route.stamp().unwrap().parent_state,
21390 Some(ParentState::Linked)
21391 );
21392 rig.sync(vec![child]).await;
21393 assert!(!rig.live(&child_route));
21394 assert_eq!(
21395 child_route.closed_reason(),
21396 RouteCloseReason::ScopeParentEnded
21397 );
21398 }
21399
21400 #[tokio::test]
21401 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
21402 ) {
21403 let mut rig = rig().await;
21404 rig.sync(vec![session(1)]).await;
21405 let mut route = rig
21406 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21407 .await;
21408 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21409 rig.sync(vec![session(1)]).await;
21410 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21411 assert!(rig.live(&route) && route.untouched());
21412
21413 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
21417 .forwarding
21418 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21419 .unwrap()
21420 else {
21421 panic!("the route is bound");
21422 };
21423 binding.flow.acquire_tagged(9, false).await.unwrap();
21424 let mut widened = session(1);
21425 widened.carriers.push(carrier(MAGIC, None));
21426 rig.sync(vec![widened]).await;
21427 assert!(rig.live(&route) && route.untouched());
21428 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21429 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
21430 assert_eq!(binding.flow.in_flight(), 1);
21433 binding
21434 .flow
21435 .acquire_tagged(10, false)
21436 .await
21437 .expect("the flow is still open");
21438 }
21439
21440 #[tokio::test]
21443 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
21444 let mut rig = rig().await;
21445 rig.sync(vec![session(1)]).await;
21446 let mut on_incumbent = rig
21447 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21448 .await;
21449
21450 let (candidate, _candidate_rx) = wide_ctx(9);
21453 let registration = rig
21454 .handler
21455 .registry
21456 .register_candidate_with_control_ops(
21457 manifest(PLEXUS, PROTOCOL_VERSION),
21458 PROTOCOL_VERSION,
21459 candidate.connection_id,
21460 module_baseline_control_ops(),
21461 )
21462 .unwrap();
21463 rig.forwarding
21464 .register_candidate_module_connection(
21465 candidate.connection_id,
21466 PLEXUS.to_string(),
21467 PROTOCOL_VERSION,
21468 manifest_concurrency(®istration.manifest),
21469 candidate.egress.clone(),
21470 )
21471 .unwrap();
21472 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
21473 rig.handler
21474 .registry
21475 .promote_candidate(PLEXUS)
21476 .unwrap()
21477 .unwrap();
21478 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
21479
21480 rig.sync(Vec::new()).await;
21481 assert!(!rig.live(&on_incumbent));
21482 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
21483 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21484 let goodbye = incumbent_rx
21485 .try_recv()
21486 .expect("the superseded endpoint is told")
21487 .frame;
21488 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21489 }
21490 }
21491}
21492
21493#[cfg(test)]
21494mod concurrency_default_exposure_tests {
21495 use super::*;
21496
21497 fn hello_body(role_json: &str) -> Vec<u8> {
21498 format!(
21499 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
21500 )
21501 .into_bytes()
21502 }
21503
21504 fn manifest_from(body: &[u8]) -> ModuleManifest {
21505 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
21506 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
21507 .expect("manifest parses")
21508 }
21509
21510 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
21511
21512 #[test]
21513 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
21514 let body = hello_body(&format!(
21515 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
21516 ));
21517 let manifest = manifest_from(&body);
21518 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21521 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
21522 }
21523
21524 #[test]
21525 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
21526 let body = hello_body(&format!(
21527 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
21528 ));
21529 let manifest = manifest_from(&body);
21530 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21531 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21532 }
21533
21534 #[test]
21535 fn non_management_roles_are_never_reported() {
21536 let body = hello_body(
21537 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
21538 );
21539 let manifest = manifest_from(&body);
21540 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21541 }
21542}