1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26 error_codes,
27 manifest::{
28 validate_hello_capability_grammar, validate_hello_confirm_operations,
29 validate_hello_display_name, validate_hello_event_declarations,
30 validate_hello_flow_operations, validate_hello_operation_deferral,
31 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32 validate_hello_self_signal_declarations, validate_role_declarations_value,
33 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34 ProviderRole,
35 },
36 peer::{
37 ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY,
38 PEER_BUILD_CAPABILITY,
39 },
40 provider_declarations::ProviderDeclarations,
41 scope::{
42 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
43 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
44 SCOPE_SYNC_OP,
45 },
46 session::{
47 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
48 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
49 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
50 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
51 },
52 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
53 Priority, RouteTarget, PROTOCOL_VERSION,
54};
55use tokio::time::{timeout_at, Instant};
56use tracing::{debug, info, instrument::WithSubscriber, warn};
57
58use crate::{
59 caller_identity::CallerIdentity,
60 capability_requirements::{
61 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
62 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
63 RequirementStatus, RuntimeModule,
64 },
65 child_roster::DaemonShutdownFlag,
66 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
67 forwarding::{
68 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
69 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
70 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
71 },
72 observability::{
73 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
74 },
75 privileged_prompt::{prompt_text, PrivilegedOperation},
76 provenance::{
77 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
78 },
79 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
80 router::{daemon_error_flags, RouteCtx, RouterError},
81 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
82 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
83 stderr_tail::{CaptureState, TailEntry},
84 supervise::{
85 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
86 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
87 },
88 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
89};
90
91pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
97
98const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
99const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
100const CAP_PING_PONG: &str = "ping_pong_v1";
101const CAP_SESSION_ATTACH: &str = "session_attach_v1";
102const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
103
104const SUBC_CONTROL_OPS: &[&str] = &[
105 "claim.mint",
106 "claim.present",
107 ops::DAEMON_SHUTDOWN_INTENT,
108 ops::ACTIVITY_RECORD,
109 ops::ACTIVITY_QUERY,
110 ops::ACTIVITY_SUBSCRIBE,
111 ops::SERVER_DESCRIBE,
112 ops::CATALOG_LIST,
113 ops::CATALOG_WATCH,
114 ops::ROUTE_OPEN,
115 ops::ROUTE_POLL,
116 ops::ROUTE_CLOSING,
117 ops::ROUTE_CLOSED,
118 ops::SUPERVISOR_LIST,
119 ops::SUPERVISOR_RESTART,
120 ops::SUPERVISOR_SWAP,
121 ops::SUPERVISOR_RELOAD,
122 ops::SUPERVISOR_RESCAN,
123 ops::SUPERVISOR_RELEASE_RESERVED,
124 ops::SUPERVISOR_SET_ENABLED,
125 ops::SUPERVISOR_HEALTH_PROBE,
126 ops::SUPERVISOR_HEALTH,
127 ops::SUPERVISOR_STDERR_TAIL,
128 ops::SUPERVISOR_TERMINALS,
129 ops::SUPERVISOR_ROUTES,
130 ops::SUPERVISOR_PROVENANCE,
131 ops::SUPERVISOR_SPAWN_SNAPSHOT,
132 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
133];
134
135const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[
142 ops::SUPERVISOR_RESCAN_CHECKED,
143 "config.status",
144 "config.plan",
145 "config.apply",
146 "config.stamp",
147 "config.unstamp",
148 "config.history",
149 "config.rollback",
150];
151
152mod caller_attestation;
153mod channel_operations;
154#[cfg(all(test, unix, feature = "test-support"))]
155#[path = "control/channel_operations_tests.rs"]
156mod channel_operations_tests;
157pub(crate) mod claims;
158#[cfg(test)]
159#[path = "control/config_advertisement_tests.rs"]
160mod config_advertisement_tests;
161mod config_gate;
162use caller_attestation::{caller_for_target, scope_acts_for_another};
163#[cfg(all(test, unix, feature = "test-support"))]
164#[path = "control/caller_attestation_tests.rs"]
165mod caller_attestation_tests;
166mod scope_expiry;
167mod scope_watch;
168
169mod agent_commands;
170pub(crate) mod catalog;
171mod rescan_checked;
172use rescan_checked::{RescanMode, RescanPlan};
173
174const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
175 "claim.mint",
176 "claim.present",
177 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
178 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
179 "supervisor.live_roots",
180 "supervisor.agent_commands",
181 SCOPE_SYNC_OP,
182 SCOPE_APPLY_OP,
183 SCOPE_DESCRIBE_OP,
184 ops::SCOPE_WATCH,
185 subc_protocol::scope::ROUTE_AUTHORIZE_OP,
186 "operator.confirm",
187];
188
189const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
193
194const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
195
196pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
204
205pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
220
221pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
239
240const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
241const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
242
243fn reload_verdict(
244 configured: &Path,
245 spawned_from: Option<&Path>,
246 image: subc_control::RunningImageAgreement,
247) -> PendingReloadVerdict {
248 let path = match spawned_from {
249 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
250 ReloadPathAgreement::Match
251 }
252 Some(spawned_from) => ReloadPathAgreement::Mismatch {
253 configured: configured.to_path_buf(),
254 spawned_from: spawned_from.to_path_buf(),
255 },
256 None => ReloadPathAgreement::Unavailable {
257 reason: if matches!(
258 image,
259 subc_control::RunningImageAgreement::Unavailable {
260 reason: subc_control::RunningImageUnavailableReason::NotRunning
261 }
262 ) {
263 ReloadPathUnavailableReason::NotRunning
264 } else {
265 ReloadPathUnavailableReason::SpawnedPathUnavailable
266 },
267 },
268 };
269 PendingReloadVerdict::new(path, image)
270}
271
272fn launch_spec_agreement(
273 configured: &crate::supervise::ModuleSpec,
274 running: bool,
275 spawned: Option<&crate::supervise::LaunchSpecDigest>,
276) -> subc_control::LaunchSpecAgreement {
277 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
278 if !running {
279 return Agreement::Unavailable {
280 reason: Reason::NotRunning,
281 };
282 }
283 match spawned {
284 None => Agreement::Unavailable {
285 reason: Reason::SpawnedSpecUnavailable,
286 },
287 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
288 Agreement::Match
289 }
290 Some(_) => Agreement::Mismatch,
291 }
292}
293
294fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
295 if configured == spawned_from {
296 return true;
297 }
298 #[cfg(windows)]
299 {
300 match (
306 std::fs::canonicalize(configured),
307 std::fs::canonicalize(spawned_from),
308 ) {
309 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
310 _ => false,
311 }
312 }
313 #[cfg(not(windows))]
314 {
315 false
316 }
317}
318
319#[derive(Clone)]
320struct DaemonProvenanceFacts {
321 build: DaemonBuildProvenance,
322 pid: Option<u32>,
323 started_at_ms: Option<u64>,
324 start_clock: Option<crate::clock::StartClock>,
325 executable_path: Option<PathBuf>,
326 executable_identity: Option<SpawnedFileIdentity>,
327 process_start_time: Option<u64>,
328 probe: ExecutableIdentityProbe,
329}
330
331impl Default for DaemonProvenanceFacts {
332 fn default() -> Self {
333 Self {
334 build: DaemonBuildProvenance {
335 build_git_sha: None,
336 build_lock_digest: None,
337 },
338 pid: None,
339 started_at_ms: None,
340 start_clock: None,
341 executable_path: None,
342 executable_identity: None,
343 process_start_time: None,
344 probe: ExecutableIdentityProbe::default(),
345 }
346 }
347}
348
349#[derive(Debug, Clone)]
350struct SupervisorRescanContext {
351 supervisor: Supervisor,
352 config_path: PathBuf,
353 configured_port: Option<u16>,
354 storage_config: Option<crate::daemon_config::StorageConfig>,
355 admission_facts_carrier_module_id: Option<String>,
356 admission_facts_targets: Option<Vec<String>>,
357 scope_authority_owners: Vec<String>,
358 federated_scope_owner: String,
359 privileged_operator_prompt: PrivilegedOperatorPrompt,
360 generation: Arc<std::sync::atomic::AtomicU64>,
366 module_entries: Arc<Mutex<rescan_checked::ModuleEntries>>,
367}
368
369const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
390 "reloading",
391 "supervisor_not_live",
392 "registration_not_active",
393 "no_forwarding_connection",
394 "relay_send_failed",
395];
396
397#[cfg(test)]
399struct ActivityRequestPause {
400 connection: ConnectionId,
401 started: tokio::sync::Notify,
402 cancelled: tokio::sync::Notify,
403 cleanup_started: std::sync::atomic::AtomicBool,
404 cancelled_after_cleanup: std::sync::atomic::AtomicBool,
405}
406
407#[cfg(test)]
408struct ActivityRequestCancellation(Arc<ActivityRequestPause>);
409
410#[cfg(test)]
411impl Drop for ActivityRequestCancellation {
412 fn drop(&mut self) {
413 self.0.cancelled_after_cleanup.store(
414 self.0
415 .cleanup_started
416 .load(std::sync::atomic::Ordering::Acquire),
417 std::sync::atomic::Ordering::Release,
418 );
419 self.0.cancelled.notify_one();
420 }
421}
422
423#[cfg(test)]
425type TestPause = (
426 tokio::sync::oneshot::Sender<()>,
427 std::sync::mpsc::Receiver<()>,
428);
429
430#[cfg(test)]
432type TestHook = Box<dyn FnOnce() + Send>;
433
434#[cfg(test)]
436type AsyncTestPause = (
437 tokio::sync::oneshot::Sender<()>,
438 tokio::sync::oneshot::Receiver<()>,
439);
440
441#[derive(Clone)]
443pub struct ControlHandler {
444 #[cfg(test)]
445 activity_request_pause: Arc<Mutex<Option<Arc<ActivityRequestPause>>>>,
446 #[cfg(test)]
450 cleanup_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
451 #[cfg(test)]
454 after_nonce_attestation: Arc<Mutex<Option<TestHook>>>,
455 #[cfg(test)]
458 hello_ingress_pause: Arc<Mutex<Option<AsyncTestPause>>>,
459 #[cfg(test)]
461 agent_commands_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
462 claims: Arc<claims::Claims>,
463 registry: Arc<Registry>,
464 config_runtime: Option<Arc<crate::config_gate::runtime::RuntimeOwner>>,
465 config_review_admission: Arc<dyn crate::config_gate::ReviewAdmission>,
466 config_review_admission_is_default: bool,
467 activity: Option<crate::activity::ActivityLedger>,
468 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
469 catalog_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
470 scope_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
471 forwarding: Arc<ForwardingTable>,
472 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
473 supervisor: SupervisorHandle,
474 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
475 agent_declarations: agent_commands::Queues,
476 list_admission: Arc<tokio::sync::Semaphore>,
477 subc_capabilities: Arc<[String]>,
478 route_bind_relay_timeout: Duration,
482 route_bind_relay_timeouts: BTreeMap<String, Duration>,
486 route_bind_breakers: RouteBindBreakers,
489 route_bind_concurrency: RouteBindConcurrency,
492 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
497 route_bind_breaker_threshold: u32,
499 route_bind_breaker_cooldown: Duration,
501 health_probe_timeout: Duration,
502 storage_config: Option<crate::daemon_config::StorageConfig>,
505 machine_id: Option<crate::machine_id::MachineId>,
509 admission_facts_carrier_module_id: Option<String>,
510 admission_facts_targets: Option<Vec<String>>,
511 scopes: Arc<RwLock<ScopeTable>>,
515 scope_deadline_added: Arc<tokio::sync::Notify>,
519 scope_authority_owners: Vec<String>,
522 federated_scope_owner: String,
523 privileged_operator_prompt: PrivilegedOperatorPrompt,
525 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
528 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
529 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
530 captured_peers: Arc<Mutex<HashMap<ConnectionId, subc_os::local_peer::CapturedPeer>>>,
531 config_desktop_stamps:
532 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
533 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
534 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
535 pub(crate) operator_apps:
536 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
537 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
542 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
548 #[cfg(test)]
550 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
551 #[cfg(test)]
552 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
553 #[cfg(test)]
554 recovery_exit_test_after_step: Option<fn(usize) -> Result<(), crate::config_gate::GateError>>,
555 #[cfg(test)]
556 recovery_exit_test_completed: Option<fn(u64)>,
557 #[cfg(test)]
558 apply_reconcile_pause: Arc<Mutex<Option<AsyncTestPause>>>,
559 #[cfg(test)]
560 recovery_exit_test_quiet_retry: bool,
561 #[cfg(test)]
562 recovery_exit_test_retry_pause: Option<fn(u64) -> Option<PathBuf>>,
563 #[cfg(test)]
564 config_delivery_test_wakes:
565 Option<tokio::sync::mpsc::UnboundedSender<(&'static str, tokio::time::Instant)>>,
566 #[cfg(all(test, unix))]
567 config_delivery_test_idle:
568 Option<tokio::sync::mpsc::UnboundedSender<tokio::sync::oneshot::Sender<()>>>,
569 #[allow(dead_code)]
573 #[cfg(windows)]
574 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
575 #[allow(dead_code)]
576 #[cfg(windows)]
577 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
578 #[allow(dead_code)]
582 #[cfg(windows)]
583 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
584 #[cfg(all(test, windows))]
587 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
588 #[cfg(test)]
589 caller_lookup: Option<TestCallerLookup>,
590 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
593 rescan: Option<SupervisorRescanContext>,
594 connected_clients: ConnectedClients,
595 counters: DaemonCounters,
596 local_endpoint_unavailable: Option<String>,
599 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
600 daemon_shutdown: DaemonShutdownFlag,
605 daemon_provenance: DaemonProvenanceFacts,
606 #[cfg(test)]
607 control_dispatch_delay: Option<Duration>,
608 #[cfg(test)]
609 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
610 #[cfg(test)]
614 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
615 #[cfg(test)]
621 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
622 #[cfg(test)]
623 hello_ack_to_publication_hold: Option<Arc<std::sync::Barrier>>,
624 #[cfg(test)]
625 hello_before_publication_hold: Option<Arc<std::sync::Barrier>>,
626 #[cfg(test)]
627 hello_after_launch_check_hold: Option<Arc<std::sync::Barrier>>,
628 #[cfg(test)]
629 hello_inside_publication_hold: Option<Arc<std::sync::Barrier>>,
630}
631
632impl fmt::Debug for ControlHandler {
633 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
634 f.debug_struct("ControlHandler")
635 .field("registry", &self.registry)
636 .field("forwarding", &self.forwarding)
637 .field("process_liveness", &self.process_liveness.is_some())
638 .field("supervisor", &self.supervisor)
639 .field("subc_capabilities", &self.subc_capabilities)
640 .finish()
641 }
642}
643
644struct RouteOpenRequest {
645 target: RouteTarget,
646 identity: BindIdentity,
647 consumer_identity: Option<ConsumerIdentity>,
648 consumer_capabilities: Option<Vec<String>>,
649 role_versions: Option<BTreeMap<String, String>>,
650 busy_refusals: Option<bool>,
651 admission_facts: Option<serde_json::Value>,
652 scope: Option<ScopeSelector>,
653 accepts_close_reasons: Option<Vec<String>>,
654}
655
656#[derive(Clone, Copy)]
657enum RouteOpenPhase {
658 Admission,
659 ProjectRoot,
660 EgressReservation,
661 BindSend,
662 BindAckWait,
663}
664
665struct RouteOpenTimings {
669 module_id: String,
670 principal: String,
671 admission: Duration,
672 project_root: Duration,
673 egress_reservation: Duration,
674 bind_send: Duration,
675 bind_ack_wait: Duration,
676 phase: Option<RouteOpenPhase>,
677 phase_started_at: Instant,
678}
679
680impl RouteOpenTimings {
681 fn new() -> Self {
682 Self {
683 module_id: String::new(),
684 principal: "unresolved".to_string(),
686 admission: Duration::ZERO,
687 project_root: Duration::ZERO,
688 egress_reservation: Duration::ZERO,
689 bind_send: Duration::ZERO,
690 bind_ack_wait: Duration::ZERO,
691 phase: Some(RouteOpenPhase::Admission),
692 phase_started_at: Instant::now(),
693 }
694 }
695
696 fn finish(&mut self) {
697 let elapsed = self.phase_started_at.elapsed();
698 match self.phase.take() {
699 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
700 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
701 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
702 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
703 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
704 None => {}
705 }
706 }
707
708 fn enter(&mut self, phase: RouteOpenPhase) {
709 self.finish();
710 self.phase = Some(phase);
711 self.phase_started_at = Instant::now();
712 }
713}
714
715struct RouteBindReservationGuard {
716 forwarding: Arc<ForwardingTable>,
717 endpoint: ModuleEndpointId,
718 relay_corr: u64,
719 armed: bool,
720}
721
722struct ModuleControlRpcGuard {
723 forwarding: Arc<ForwardingTable>,
724 endpoint: ModuleEndpointId,
725 corr: u64,
726 armed: bool,
727}
728
729impl ModuleControlRpcGuard {
730 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
731 Self {
732 forwarding,
733 endpoint,
734 corr,
735 armed: true,
736 }
737 }
738
739 fn disarm(&mut self) {
740 self.armed = false;
741 }
742}
743
744impl Drop for ModuleControlRpcGuard {
745 fn drop(&mut self) {
746 if self.armed {
747 let _ = self
748 .forwarding
749 .cancel_module_control_rpc(self.endpoint, self.corr);
750 }
751 }
752}
753
754impl RouteBindReservationGuard {
755 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
756 Self {
757 forwarding,
758 endpoint,
759 relay_corr,
760 armed: true,
761 }
762 }
763
764 fn release_and_disarm(&mut self) {
765 if !self.armed {
766 return;
767 }
768 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
769 self.endpoint,
770 self.relay_corr,
771 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
772 ) {
773 send_goodbye_target_best_effort(
774 &self.forwarding.counters(),
775 &target,
776 "canceled route.bind",
777 );
778 }
779 self.armed = false;
780 }
781
782 fn disarm(&mut self) {
783 self.armed = false;
784 }
785}
786
787impl Drop for RouteBindReservationGuard {
788 fn drop(&mut self) {
789 self.release_and_disarm();
790 }
791}
792
793#[derive(Debug, Clone, Default)]
823pub(crate) struct RouteBindBreakers {
824 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
825}
826
827#[derive(Debug, Clone, Default)]
828pub(crate) struct RouteBindConcurrency {
829 modules: Arc<Mutex<HashMap<String, usize>>>,
830}
831
832struct RouteBindConcurrencyGuard {
833 concurrency: RouteBindConcurrency,
834 module_id: String,
835}
836
837impl RouteBindConcurrency {
838 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
841 let mut modules = self
842 .modules
843 .lock()
844 .expect("route.bind concurrency mutex poisoned");
845 let in_flight = modules.entry(module_id.to_string()).or_default();
846 if *in_flight >= limit {
847 return Err(*in_flight);
848 }
849 *in_flight += 1;
850 Ok(RouteBindConcurrencyGuard {
851 concurrency: self.clone(),
852 module_id: module_id.to_string(),
853 })
854 }
855}
856
857impl Drop for RouteBindConcurrencyGuard {
858 fn drop(&mut self) {
859 let mut modules = self
860 .concurrency
861 .modules
862 .lock()
863 .expect("route.bind concurrency mutex poisoned");
864 let remove = {
865 let in_flight = modules
866 .get_mut(&self.module_id)
867 .expect("admitted route.bind has a concurrency entry");
868 *in_flight -= 1;
869 *in_flight == 0
870 };
871 if remove {
872 modules.remove(&self.module_id);
873 }
874 }
875}
876
877#[derive(Debug, Default)]
878struct ModuleBreakerState {
879 consecutive_timeouts: u32,
881 cooldown_until: Option<Instant>,
884 probe_in_flight: Option<Arc<()>>,
889}
890
891enum RouteBindAdmission<'a> {
893 Admitted {
894 guard: RouteBindBreakerGuard<'a>,
895 probe: bool,
898 },
899 Refused {
900 consecutive_timeouts: u32,
901 retry_in: Duration,
905 probe_in_flight: bool,
906 },
907}
908
909struct RouteBindBreakerGuard<'a> {
916 breakers: RouteBindBreakers,
917 module_id: &'a str,
918 probe_token: Option<Arc<()>>,
919 settled: bool,
920}
921
922impl RouteBindBreakerGuard<'_> {
923 fn record_accepted(&mut self) -> bool {
927 self.settled = true;
928 self.breakers.record_accepted(self.module_id)
929 }
930
931 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
934 self.settled = true;
935 self.breakers.record_timeout(
936 self.module_id,
937 self.probe_token.as_ref(),
938 threshold,
939 cooldown,
940 )
941 }
942
943 fn record_inconclusive(&mut self) {
952 self.settled = true;
953 self.breakers
954 .record_inconclusive(self.module_id, self.probe_token.as_ref());
955 }
956}
957
958impl Drop for RouteBindBreakerGuard<'_> {
959 fn drop(&mut self) {
960 if !self.settled {
961 self.breakers
962 .record_inconclusive(self.module_id, self.probe_token.as_ref());
963 }
964 }
965}
966
967struct BreakerOpened {
971 consecutive_timeouts: u32,
972 reopened_after_probe: bool,
975}
976
977impl RouteBindBreakers {
978 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
979 self.modules
980 .lock()
981 .expect("route.bind breaker mutex poisoned")
982 }
983
984 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
987 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
988 probe: probe_token.is_some(),
989 guard: RouteBindBreakerGuard {
990 breakers: self.clone(),
991 module_id,
992 probe_token,
993 settled: false,
994 },
995 };
996
997 let mut modules = self.lock();
998 let Some(state) = modules.get_mut(module_id) else {
999 return admitted(None);
1000 };
1001 let Some(cooldown_until) = state.cooldown_until else {
1002 return admitted(None);
1003 };
1004 if state.probe_in_flight.is_some() {
1005 return RouteBindAdmission::Refused {
1006 consecutive_timeouts: state.consecutive_timeouts,
1007 retry_in: Duration::ZERO,
1008 probe_in_flight: true,
1009 };
1010 }
1011 let now = Instant::now();
1012 if now < cooldown_until {
1013 return RouteBindAdmission::Refused {
1014 consecutive_timeouts: state.consecutive_timeouts,
1015 retry_in: cooldown_until - now,
1016 probe_in_flight: false,
1017 };
1018 }
1019 let token = Arc::new(());
1020 state.probe_in_flight = Some(Arc::clone(&token));
1021 admitted(Some(token))
1022 }
1023
1024 fn record_accepted(&self, module_id: &str) -> bool {
1025 self.lock()
1026 .remove(module_id)
1027 .is_some_and(|state| state.cooldown_until.is_some())
1028 }
1029
1030 fn record_timeout(
1031 &self,
1032 module_id: &str,
1033 probe_token: Option<&Arc<()>>,
1034 threshold: u32,
1035 cooldown: Duration,
1036 ) -> Option<BreakerOpened> {
1037 let mut modules = self.lock();
1038 let state = modules.entry(module_id.to_string()).or_default();
1039 let was_open = state.cooldown_until.is_some();
1040 let was_probe = Self::owns_probe(state, probe_token);
1041 if was_probe {
1042 state.probe_in_flight = None;
1043 }
1044 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
1045 if state.consecutive_timeouts < threshold {
1046 return None;
1047 }
1048 state.cooldown_until = Some(Instant::now() + cooldown);
1049 Some(BreakerOpened {
1050 consecutive_timeouts: state.consecutive_timeouts,
1051 reopened_after_probe: was_open && was_probe,
1052 })
1053 }
1054
1055 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
1056 state
1060 .probe_in_flight
1061 .as_ref()
1062 .zip(token)
1063 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
1064 }
1065
1066 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
1067 if let Some(state) = self.lock().get_mut(module_id) {
1068 if Self::owns_probe(state, probe_token) {
1069 state.probe_in_flight = None;
1070 }
1071 }
1072 }
1073
1074 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
1089 self.lock()
1090 .remove(module_id)
1091 .map(|state| state.consecutive_timeouts)
1092 .filter(|discarded| *discarded > 0)
1093 }
1094
1095 fn open_snapshot(&self) -> Option<serde_json::Value> {
1102 let now = Instant::now();
1103 let modules = self.lock();
1104 let open = modules
1105 .iter()
1106 .filter_map(|(module_id, state)| {
1107 let cooldown_until = state.cooldown_until?;
1108 Some((
1109 module_id.clone(),
1110 serde_json::json!({
1111 "consecutive_timeouts": state.consecutive_timeouts,
1112 "cooldown_remaining_ms":
1113 cooldown_until.saturating_duration_since(now).as_millis() as u64,
1114 "probe_in_flight": state.probe_in_flight.is_some(),
1115 }),
1116 ))
1117 })
1118 .collect::<serde_json::Map<String, serde_json::Value>>();
1119 (!open.is_empty()).then_some(serde_json::Value::Object(open))
1120 }
1121}
1122
1123impl ControlHandler {
1124 pub(crate) fn install_registration_service(
1125 &self,
1126 router: std::sync::Weak<crate::Router>,
1127 auth: crate::server::ServerAuth,
1128 ) {
1129 self.supervisor.install_registration_service(router, auth);
1130 }
1131 pub fn new(registry: Arc<Registry>) -> Self {
1132 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
1133 }
1134
1135 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
1136 let counters = forwarding.counters();
1137 let route_bind_breakers = forwarding.route_bind_breakers();
1141 let route_bind_concurrency = forwarding.route_bind_concurrency();
1142 let route_outages = forwarding.route_outages();
1143 Self {
1144 registry,
1145 #[cfg(test)]
1146 activity_request_pause: Arc::new(Mutex::new(None)),
1147 #[cfg(test)]
1148 cleanup_pause: Arc::new(Mutex::new(None)),
1149 #[cfg(test)]
1150 after_nonce_attestation: Arc::new(Mutex::new(None)),
1151 #[cfg(test)]
1152 hello_ingress_pause: Arc::new(Mutex::new(None)),
1153 #[cfg(test)]
1154 agent_commands_pause: Arc::new(Mutex::new(None)),
1155 claims: Arc::new(claims::Claims::default()),
1156 config_runtime: None,
1157 config_review_admission: Arc::new(crate::config_gate::MissingReviewAdmission),
1158 config_review_admission_is_default: true,
1159 activity: None,
1160 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1161 catalog_streams: Arc::new(Mutex::new(HashMap::new())),
1162 scope_streams: Arc::new(Mutex::new(HashMap::new())),
1163 forwarding,
1164 process_liveness: None,
1165 supervisor: SupervisorHandle::new(),
1166 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1167 env!("CARGO_PKG_VERSION").to_owned(),
1168 )),
1169 agent_declarations: agent_commands::Queues::default(),
1170 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1171 subc_capabilities: Arc::from([
1172 CAP_MANIFEST_REGISTRATION.to_string(),
1173 CAP_CHANNEL_LIFECYCLE.to_string(),
1174 CAP_PING_PONG.to_string(),
1175 CAP_SESSION_ATTACH.to_string(),
1176 CAP_ADMISSION_FACTS_RELAY.to_string(),
1177 CAP_SCOPES_V1.to_string(),
1178 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1179 subc_protocol::peer::CALLER_ATTESTATION_CAPABILITY.to_string(),
1180 PEER_BUILD_CAPABILITY.to_string(),
1181 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1182 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1183 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1184 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1185 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1186 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1187 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1188 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1189 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1190 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1191 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1192 subc_protocol::ROUTE_CLOSE_REASONS_CAPABILITY.to_string(),
1193 ]),
1194 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1195 route_bind_relay_timeouts: BTreeMap::new(),
1196 route_bind_breakers,
1197 route_bind_concurrency,
1198 route_outages,
1199 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1200 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1201 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1202 storage_config: None,
1203 machine_id: None,
1204 admission_facts_carrier_module_id: None,
1205 admission_facts_targets: None,
1206 scopes: Arc::new(RwLock::new(ScopeTable::new(
1207 crate::daemon_config::default_scope_authority_owners(),
1208 ))),
1209 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1210 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1211 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1212 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1213 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1214 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1215 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1216 captured_peers: Arc::new(Mutex::new(HashMap::new())),
1217 config_desktop_stamps: Arc::new(Mutex::new(HashMap::new())),
1218 local_connections: Arc::new(Mutex::new(HashSet::new())),
1219 operator_app_policy: Default::default(),
1220 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1221 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1222 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1223 #[cfg(test)]
1224 app_open_test_platform: None,
1225 #[cfg(test)]
1226 app_open_test_launch: None,
1227 #[cfg(test)]
1228 recovery_exit_test_after_step: None,
1229 #[cfg(test)]
1230 recovery_exit_test_completed: None,
1231 #[cfg(test)]
1232 apply_reconcile_pause: Arc::new(Mutex::new(None)),
1233 #[cfg(test)]
1234 recovery_exit_test_quiet_retry: false,
1235 #[cfg(test)]
1236 recovery_exit_test_retry_pause: None,
1237 #[cfg(test)]
1238 config_delivery_test_wakes: None,
1239 #[cfg(all(test, unix))]
1240 config_delivery_test_idle: None,
1241 #[cfg(windows)]
1242 windows_verify: None,
1243 #[cfg(windows)]
1244 windows_flash: None,
1245 #[cfg(windows)]
1246 windows_track: None,
1247 #[cfg(all(test, windows))]
1248 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1249 #[cfg(test)]
1250 caller_lookup: None,
1251 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1252 rescan: None,
1253 connected_clients: ConnectedClients::new(),
1254 local_endpoint_unavailable: None,
1255 counters,
1256 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1257 daemon_shutdown: DaemonShutdownFlag::default(),
1258 daemon_provenance: DaemonProvenanceFacts::default(),
1259 #[cfg(test)]
1260 control_dispatch_delay: None,
1261 #[cfg(test)]
1262 provenance_probe_override: None,
1263 #[cfg(test)]
1264 route_bind_relay_sent_hold: None,
1265 #[cfg(test)]
1266 hello_registry_to_forwarding_hold: None,
1267 #[cfg(test)]
1268 hello_ack_to_publication_hold: None,
1269 #[cfg(test)]
1270 hello_before_publication_hold: None,
1271 #[cfg(test)]
1272 hello_after_launch_check_hold: None,
1273 #[cfg(test)]
1274 hello_inside_publication_hold: None,
1275 }
1276 }
1277
1278 pub fn with_storage_config(
1281 mut self,
1282 storage_config: Option<crate::daemon_config::StorageConfig>,
1283 ) -> Self {
1284 self.storage_config = storage_config;
1285 self
1286 }
1287
1288 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1291 self.activity = Some(crate::activity::ActivityLedger::open(
1292 run_dir.join("activity.jsonl"),
1293 incarnation,
1294 ));
1295 self.supervisor.set_activity(self.activity.clone());
1296 self
1297 }
1298
1299 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1302 self.machine_id = machine_id;
1303 self
1304 }
1305
1306 pub fn with_admission_facts_config(
1310 mut self,
1311 carrier_module_id: Option<String>,
1312 targets: Option<Vec<String>>,
1313 ) -> Self {
1314 self.admission_facts_carrier_module_id = carrier_module_id;
1315 self.admission_facts_targets = targets;
1316 self
1317 }
1318
1319 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1323 self.scopes = Arc::new(RwLock::new(
1324 ScopeTable::new(owners.iter().cloned())
1325 .with_federated_owner(self.federated_scope_owner.clone()),
1326 ));
1327 self.scope_authority_owners = owners;
1328 self
1329 }
1330
1331 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1333 self.scopes = Arc::new(RwLock::new(
1334 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1335 .with_federated_owner(owner.clone()),
1336 ));
1337 self.federated_scope_owner = owner;
1338 self
1339 }
1340
1341 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1343 self.privileged_operator_prompt = mode;
1344 self
1345 }
1346
1347 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1350 self.route_bind_relay_timeout = timeout;
1351 self
1352 }
1353
1354 pub fn with_route_bind_relay_timeouts(
1360 mut self,
1361 timeouts: impl IntoIterator<Item = (String, Duration)>,
1362 ) -> Self {
1363 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1364 self
1365 }
1366
1367 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1373 self.route_bind_relay_timeouts
1374 .get(module_id)
1375 .copied()
1376 .unwrap_or(self.route_bind_relay_timeout)
1377 }
1378
1379 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1387 self.route_bind_breaker_threshold = threshold.max(1);
1388 self.route_bind_breaker_cooldown = cooldown;
1389 self
1390 }
1391
1392 #[cfg(test)]
1393 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1394 self.health_probe_timeout = timeout;
1395 self
1396 }
1397
1398 #[cfg(test)]
1399 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1400 self.control_dispatch_delay = Some(delay);
1401 self
1402 }
1403
1404 #[cfg(test)]
1408 pub(crate) fn with_route_bind_relay_sent_hold(
1409 mut self,
1410 hold: Arc<tokio::sync::Barrier>,
1411 ) -> Self {
1412 self.route_bind_relay_sent_hold = Some(hold);
1413 self
1414 }
1415
1416 #[cfg(test)]
1421 pub(crate) fn with_hello_registry_to_forwarding_hold(
1422 mut self,
1423 hold: Arc<std::sync::Barrier>,
1424 ) -> Self {
1425 self.hello_registry_to_forwarding_hold = Some(hold);
1426 self
1427 }
1428
1429 #[cfg(test)]
1430 fn hold_between_registry_and_forwarding(&self) {
1431 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1432 hold.wait();
1433 hold.wait();
1434 }
1435 }
1436
1437 #[cfg(test)]
1440 fn hold_between_ack_and_publication(&self) {
1441 if let Some(hold) = &self.hello_ack_to_publication_hold {
1442 hold.wait();
1443 hold.wait();
1444 }
1445 }
1446
1447 #[cfg(all(test, unix))]
1448 pub(crate) fn with_hello_before_publication_hold(
1449 mut self,
1450 hold: Arc<std::sync::Barrier>,
1451 ) -> Self {
1452 self.hello_before_publication_hold = Some(hold);
1453 self
1454 }
1455
1456 #[cfg(all(test, unix))]
1457 pub(crate) fn with_hello_after_launch_check_hold(
1458 mut self,
1459 hold: Arc<std::sync::Barrier>,
1460 ) -> Self {
1461 self.hello_after_launch_check_hold = Some(hold);
1462 self
1463 }
1464
1465 #[cfg(all(test, unix))]
1466 pub(crate) fn with_hello_inside_publication_hold(
1467 mut self,
1468 hold: Arc<std::sync::Barrier>,
1469 ) -> Self {
1470 self.hello_inside_publication_hold = Some(hold);
1471 self
1472 }
1473
1474 pub fn with_process_liveness(
1475 mut self,
1476 process_liveness: Arc<dyn ModuleProcessLiveness>,
1477 ) -> Self {
1478 self.process_liveness = Some(process_liveness);
1479 self
1480 }
1481
1482 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1483 self.supervisor = supervisor;
1484 if let Some(owner) = &self.config_runtime {
1485 self.supervisor.install_binding_retirement(owner);
1486 }
1487 self.supervisor.set_activity(self.activity.clone());
1488 self
1489 }
1490
1491 pub fn with_daemon_provenance(
1492 mut self,
1493 pid: u32,
1494 started_at_ms: u64,
1495 executable_path: Option<PathBuf>,
1496 build_git_sha: Option<String>,
1497 build_lock_digest: Option<String>,
1498 ) -> Self {
1499 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1500 let process_start_time = process_start_time(pid);
1501 self.daemon_provenance = DaemonProvenanceFacts {
1502 build: DaemonBuildProvenance {
1503 build_git_sha,
1504 build_lock_digest,
1505 },
1506 pid: Some(pid),
1507 started_at_ms: Some(started_at_ms),
1508 start_clock: None,
1509 executable_path,
1510 executable_identity,
1511 process_start_time,
1512 probe: ExecutableIdentityProbe::default(),
1513 };
1514 self
1515 }
1516
1517 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1518 self.daemon_provenance.start_clock = Some(clock);
1519 self
1520 }
1521
1522 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1524 self.scopes
1525 .write()
1526 .unwrap_or_else(|poisoned| poisoned.into_inner())
1527 .set_wall_clock(Arc::new(clock));
1528 self
1529 }
1530
1531 #[cfg(test)]
1532 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1533 self.provenance_probe_override = Some(result);
1534 self
1535 }
1536
1537 pub fn with_capability_config(
1541 self,
1542 modules: impl IntoIterator<Item = (String, bool)>,
1543 reserved_capabilities: BTreeMap<String, String>,
1544 ) -> Self {
1545 self.capability_evaluator
1546 .configure(modules, reserved_capabilities);
1547 self
1548 }
1549
1550 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1554 self.daemon_shutdown = flag;
1555 self
1556 }
1557
1558 pub fn with_supervisor_rescan(
1559 mut self,
1560 supervisor: Supervisor,
1561 config_path: impl Into<PathBuf>,
1562 configured_port: Option<u16>,
1563 ) -> Self {
1564 self.rescan = Some(SupervisorRescanContext {
1565 supervisor,
1566 config_path: config_path.into(),
1567 configured_port,
1568 storage_config: self.storage_config.clone(),
1569 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1570 admission_facts_targets: self.admission_facts_targets.clone(),
1571 scope_authority_owners: self.scope_authority_owners.clone(),
1572 federated_scope_owner: self.federated_scope_owner.clone(),
1573 privileged_operator_prompt: self.privileged_operator_prompt,
1574 generation: Arc::default(),
1575 module_entries: Arc::default(),
1576 });
1577 self
1578 }
1579
1580 pub(crate) fn with_supervisor_rescan_entries(
1581 self,
1582 modules: &[crate::daemon_config::ConfiguredModule],
1583 ) -> Self {
1584 if let Some(context) = &self.rescan {
1585 let mut entries = context
1586 .module_entries
1587 .lock()
1588 .unwrap_or_else(|e| e.into_inner());
1589 entries.seed(modules);
1590 }
1591 self
1592 }
1593
1594 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1595 self.connected_clients = connected_clients;
1596 self
1597 }
1598
1599 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1602 self.local_endpoint_unavailable = reason;
1603 self
1604 }
1605
1606 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1607 Arc::clone(&self.forwarding)
1608 }
1609
1610 pub(crate) fn counters(&self) -> DaemonCounters {
1611 self.counters.clone()
1612 }
1613
1614 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1617 let handler = Arc::downgrade(&self);
1618 let evaluator = self.capability_evaluator.clone();
1619 tokio::spawn(async move {
1620 loop {
1621 evaluator.wait_for_change_or_deadline().await;
1622 let Some(handler) = handler.upgrade() else {
1623 break;
1624 };
1625 handler.refresh_capability_requirements();
1626 }
1627 });
1628 }
1629
1630 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1636 let context = scope_expiry::ScopeExpiryContext::new(&self);
1637 let deadline_added = self.scope_deadline_added.clone();
1638 tokio::spawn(async move {
1639 loop {
1640 if !context.has_deadlines() {
1641 deadline_added.notified().await;
1642 continue;
1643 }
1644 tokio::time::sleep(Duration::from_secs(1)).await;
1645 if let Err(error) = context.sweep() {
1646 warn!(%error, "scope expiry sweep failed");
1647 }
1648 }
1649 });
1650 }
1651
1652 #[cfg(test)]
1653 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1654 scope_expiry::ScopeExpiryContext::new(self).sweep()
1655 }
1656
1657 fn publish_scope_changes(
1658 &self,
1659 changes: &[crate::scopes::ScopeTagChange],
1660 expired: &[crate::scopes::ScopeExpired],
1661 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1662 scope_expiry::ScopeExpiryContext::new(self).publish_scope_changes(changes, expired)
1663 }
1664
1665 fn runtime_capability_snapshot(
1666 &self,
1667 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1668 let runtime = self
1669 .supervisor
1670 .list()
1671 .into_iter()
1672 .map(|module| {
1673 let status = module.status().map_err(|err| {
1674 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1675 })?;
1676 Ok(RuntimeModule {
1677 module_id: status.module_id,
1678 state: status.state,
1679 enabled: status.enabled,
1680 })
1681 })
1682 .collect::<Result<Vec<_>, RouterError>>()?;
1683 let registrations = self.registry.list_registered_modules().map_err(|err| {
1689 RouterError::backend(
1690 0,
1691 0,
1692 format!("failed to list capability registrations: {err}"),
1693 )
1694 })?;
1695 let registrations = registrations
1696 .into_iter()
1697 .map(|registration| RegisteredModule {
1698 module_id: registration.manifest.module_id,
1699 module_version: registration.manifest.module_version,
1700 capabilities: registration.manifest.capabilities,
1701 })
1702 .collect();
1703 Ok((runtime, registrations))
1704 }
1705
1706 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1713 let cached_registration = RegisteredModule {
1714 module_id: registration.manifest.module_id.clone(),
1715 module_version: registration.manifest.module_version.clone(),
1716 capabilities: registration.manifest.capabilities.clone(),
1717 };
1718 let change = self.capability_evaluator.record_hello(&cached_registration);
1719 log_manifest_change(&cached_registration.module_id, &change);
1720 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1721 self.enforce_capability_denies();
1722 }
1723 self.refresh_capability_requirements();
1724 }
1725
1726 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1730 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1731 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1732 self.supervisor.set_swap_promotion_observer(observer);
1733 }
1734
1735 pub fn refresh_capability_requirements(&self) {
1736 match self.runtime_capability_snapshot() {
1737 Ok((runtime, registrations)) => {
1738 log_requirement_events(
1739 self.capability_evaluator
1740 .evaluate_now(&runtime, ®istrations),
1741 &self.daemon_shutdown,
1742 );
1743 }
1744 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1745 }
1746 }
1747
1748 fn enforce_capability_denies(&self) {
1752 let registrations = match self.registry.list_registered_modules() {
1753 Ok(snapshot) => snapshot,
1754 Err(err) => {
1755 warn!(error = %err, "failed to read registrations for capability deny census");
1756 return;
1757 }
1758 };
1759 let manifests = registrations
1760 .into_iter()
1761 .map(|registration| {
1762 (
1763 registration.manifest.module_id.clone(),
1764 registration.manifest,
1765 )
1766 })
1767 .collect::<BTreeMap<_, _>>();
1768 let census = match self.forwarding.route_census(None) {
1769 Ok(census) => census,
1770 Err(err) => {
1771 warn!(error = %err, "failed to read route census for capability deny enforcement");
1772 return;
1773 }
1774 };
1775
1776 for (target_module_id, routes) in census {
1777 let Some(target_manifest) = manifests.get(&target_module_id) else {
1778 continue;
1779 };
1780 let mut closed_routes = Vec::new();
1781 let mut goodbyes = Vec::new();
1782 for route in routes {
1783 let Principal::Reserved {
1784 module_id: opening_module_id,
1785 } = &route.principal
1786 else {
1787 continue;
1788 };
1789 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1790 continue;
1791 };
1792 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1793 continue;
1794 };
1795
1796 match self.forwarding.release_client_route(
1797 route.goodbye_target.connection_id,
1798 route.goodbye_target.channel,
1799 route.goodbye_target.epoch,
1800 ) {
1801 Ok(RouteRelease::Removed(module_goodbye)) => {
1802 warn!(
1803 opening_module_id,
1804 target_module_id,
1805 capability,
1806 "force-closing route because an attested capability deny edge now matches"
1807 );
1808 goodbyes.push(module_goodbye);
1809 goodbyes.push(route.goodbye_target.clone());
1810 closed_routes.push(route);
1811 }
1812 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1813 Err(err) => warn!(
1814 opening_module_id,
1815 target_module_id,
1816 capability,
1817 error = %err,
1818 "failed to force-close capability-denied route"
1819 ),
1820 }
1821 }
1822
1823 if closed_routes.is_empty() {
1824 continue;
1825 }
1826 send_route_control_pushes(
1827 &self.forwarding,
1828 closed_routes,
1829 ClientControlPush::RouteClosed {
1830 module_id: target_module_id,
1831 channels: Vec::new(),
1832 reason: RouteCloseReason::CapabilityDenied,
1833 drained: false,
1834 abandoned: 0,
1835 excluded_subscriptions: 0,
1836 terminal: Some(false),
1837 },
1838 );
1839 self.emit_route_goodbyes(goodbyes);
1842 }
1843 }
1844
1845 fn not_ready_reason(
1851 &self,
1852 registration: &crate::registry::ModuleRegistration,
1853 ) -> Option<NotReadyReason> {
1854 if !registration.ready {
1855 return Some(NotReadyReason {
1856 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1857 capability: None,
1858 });
1859 }
1860 self.first_unprovided_required_capability(registration)
1861 .map(|capability| NotReadyReason {
1862 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1863 capability: Some(capability),
1864 })
1865 }
1866
1867 fn first_unprovided_required_capability(
1885 &self,
1886 registration: &crate::registry::ModuleRegistration,
1887 ) -> Option<String> {
1888 let required = registration
1889 .manifest
1890 .capabilities
1891 .iter()
1892 .flat_map(|declarations| declarations.requires.iter())
1893 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1894 .map(|requirement| requirement.capability.as_str())
1895 .collect::<BTreeSet<_>>();
1896 if required.is_empty() {
1897 return None;
1898 }
1899 let module_id = registration.manifest.module_id.as_str();
1900 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1901 if required
1902 .iter()
1903 .any(|capability| verdict(capability).is_none())
1904 {
1905 self.refresh_capability_requirements();
1906 }
1907 required
1908 .into_iter()
1909 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1910 .map(str::to_string)
1911 }
1912
1913 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1914 self.capability_evaluator
1915 .statuses()
1916 .into_iter()
1917 .map(capability_requirement_status)
1918 .collect()
1919 }
1920
1921 fn deregister_connection(
1928 &self,
1929 connection_id: ConnectionId,
1930 reason: RegistrationEndReason,
1931 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1932 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
1933 spawn.end_primary_registration();
1934 }
1935 self.registry
1936 .deregister_connection_with_reason(connection_id, reason)
1937 }
1938
1939 fn rollback_unpublished_hello(
1944 &self,
1945 connection: ConnectionId,
1946 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1947 self.registry.deregister_connection_with_reason(
1948 connection,
1949 RegistrationEndReason::RegistrationFailed,
1950 )
1951 }
1952
1953 fn request_refusal_close(
1954 &self,
1955 connection: ConnectionId,
1956 sink: Option<&crate::FrameSink>,
1957 reason: CloseReason,
1958 ) {
1959 if let Some(sink) = sink {
1960 sink.terminate_ingress();
1961 }
1962 if self.claims.close_notifications_enabled() {
1963 self.forwarding.request_connection_close(connection, reason);
1964 }
1965 }
1966
1967 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1968 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1969 return None;
1970 }
1971 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1972 parse_client_control_request(&frame.body)
1973 else {
1974 return None;
1975 };
1976 Some(target_module_id(&target).to_string())
1977 }
1978
1979 pub(crate) fn route_open_capacity_refusal(
1980 &self,
1981 ctx: &RouteCtx,
1982 frame: &Frame,
1983 target_module_id: &str,
1984 in_flight: usize,
1985 limit: usize,
1986 ) -> Result<Frame, RouterError> {
1987 let caller = self.route_open_caller_from_frame(ctx, frame);
1990 self.route_open_admission_refusal_frame(
1991 ctx,
1992 &caller,
1993 frame,
1994 target_module_id,
1995 "open_admission_full",
1996 (in_flight, limit),
1997 format!(
1998 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1999 ),
2000 )
2001 }
2002
2003 fn route_open_target_capacity_refusal(
2004 &self,
2005 ctx: &RouteCtx,
2006 caller: &RouteOpenCaller,
2007 frame: &Frame,
2008 target_module_id: &str,
2009 in_flight: usize,
2010 ) -> Result<Frame, RouterError> {
2011 self.route_open_admission_refusal_frame(
2012 ctx,
2013 caller,
2014 frame,
2015 target_module_id,
2016 "target_binds_full",
2017 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
2018 format!(
2019 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2020 ),
2021 )
2022 }
2023
2024 #[allow(clippy::too_many_arguments)]
2032 fn route_open_admission_refusal_frame(
2033 &self,
2034 ctx: &RouteCtx,
2035 caller: &RouteOpenCaller,
2036 frame: &Frame,
2037 target_module_id: &str,
2038 reason: &'static str,
2039 (in_flight, limit): (usize, usize),
2040 message: impl Into<String>,
2041 ) -> Result<Frame, RouterError> {
2042 let code = error_codes::TARGET_UNAVAILABLE;
2043 self.counters.increment_route_open_refused(code);
2044 info!(
2045 target: "control",
2046 code,
2047 reason,
2048 module_id = ?target_module_id,
2049 connection_id = ctx.connection_id.get(),
2050 principal = %caller.principal,
2051 scope = %caller.scope,
2052 harness = %caller.harness,
2053 in_flight,
2054 limit,
2055 "route.open refused"
2056 );
2057 control_error_frame(frame, code, message.into())
2058 }
2059
2060 #[cfg(test)]
2066 pub fn handle_control(
2067 &self,
2068 connection_id: ConnectionId,
2069 frame: Frame,
2070 ) -> Result<Vec<Frame>, RouterError> {
2071 match frame.header.ty {
2072 FrameType::Ping => Ok(vec![pong(&frame)?]),
2073 FrameType::Hello => self.handle_hello(connection_id, None, frame),
2074 FrameType::Goodbye => self.handle_goodbye(connection_id),
2075 ty => Ok(vec![control_error_frame(
2076 &frame,
2077 "unsupported_control_frame",
2078 format!("unsupported channel-0 frame {ty:?}"),
2079 )?]),
2080 }
2081 }
2082
2083 pub async fn handle_control_frame(
2084 &self,
2085 ctx: &RouteCtx,
2086 frame: Frame,
2087 ) -> Result<Vec<Frame>, RouterError> {
2088 self.handle_control_frame_timed(ctx, frame, None).await
2089 }
2090
2091 #[cfg(any(test, feature = "test-support"))]
2092 pub fn with_test_operator_app_policy(
2093 mut self,
2094 policy: crate::operator_apps::TestOperatorAppPolicy,
2095 ) -> Self {
2096 self.operator_app_policy = policy.into();
2097 self
2098 }
2099
2100 #[cfg(all(windows, any(test, feature = "test-support")))]
2104 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
2105 self.windows_verify = Some(verify);
2106 self
2107 }
2108
2109 #[cfg(all(windows, any(test, feature = "test-support")))]
2111 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
2112 self.windows_flash = Some(flash);
2113 self
2114 }
2115
2116 #[cfg(all(windows, any(test, feature = "test-support")))]
2120 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
2121 self.windows_track = Some(track);
2122 self
2123 }
2124
2125 pub(crate) fn record_captured_peer(
2126 &self,
2127 connection: ConnectionId,
2128 peer: subc_os::local_peer::CapturedPeer,
2129 ) {
2130 self.record_observed_peer(connection, peer.observation().clone());
2131 let identity = self.operator_app_policy.config_identity(peer.clone());
2132 #[cfg(windows)]
2133 let identity = identity.or_else(|| self.config_windows_launch_identity(&peer));
2134 if let Some(identity) = identity {
2135 self.config_desktop_stamps
2138 .lock()
2139 .unwrap_or_else(|p| p.into_inner())
2140 .insert(connection, identity);
2141 }
2142 self.captured_peers
2143 .lock()
2144 .unwrap_or_else(|p| p.into_inner())
2145 .insert(connection, peer.clone());
2146 #[cfg(windows)]
2147 if let Some(stamp) = self.stamp_windows_launch(&peer) {
2148 self.operator_apps
2149 .lock()
2150 .unwrap_or_else(|p| p.into_inner())
2151 .insert(connection, stamp);
2152 }
2153 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
2154 self.operator_apps
2155 .lock()
2156 .unwrap_or_else(|p| p.into_inner())
2157 .insert(connection, stamp);
2158 }
2159 }
2160
2161 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
2162 self.local_connections
2163 .lock()
2164 .unwrap_or_else(|p| p.into_inner())
2165 .insert(connection);
2166 }
2167
2168 fn connection_operator_app(
2171 &self,
2172 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
2173 ) -> Option<subc_protocol::peer::OperatorApp> {
2174 stamp
2175 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
2176 .map(|stamp| stamp.app.clone())
2177 }
2178
2179 fn handle_connection_describe(
2180 &self,
2181 ctx: &RouteCtx,
2182 frame: Frame,
2183 ) -> Result<Vec<Frame>, RouterError> {
2184 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
2185 return Ok(vec![control_error_frame(
2186 &frame,
2187 "invalid_control_body",
2188 "malformed connection.describe body",
2189 )?]);
2190 }
2191 let local = self
2192 .local_connections
2193 .lock()
2194 .unwrap_or_else(|p| p.into_inner())
2195 .contains(&ctx.connection_id);
2196 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
2197 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
2198 Ok(vec![control_response_body_frame(
2199 &frame,
2200 &subc_control::ConnectionDescribeReply::new(local, operator_app),
2201 ops::CONNECTION_DESCRIBE,
2202 )?])
2203 }
2204
2205 pub(crate) fn record_observed_peer(
2206 &self,
2207 connection: ConnectionId,
2208 peer: subc_os::local_peer::ObservedPeer,
2209 ) {
2210 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
2211 let peer = match peer {
2212 OsPeer::Unobserved => ObservedPeer::unobserved(),
2213 OsPeer::Observed(peer) => {
2214 let observed = match peer.user {
2215 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
2216 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
2217 };
2218 observed
2219 .with_exe(
2220 peer.exe
2221 .and_then(|path| path.into_os_string().into_string().ok()),
2222 )
2223 .with_signing_id(peer.signing_id)
2224 }
2225 };
2226 self.observed_peers
2227 .lock()
2228 .unwrap_or_else(|p| p.into_inner())
2229 .insert(connection, peer);
2230 }
2231
2232 pub(crate) fn record_caller_endpoints(
2233 &self,
2234 connection: ConnectionId,
2235 endpoints: Option<(SocketAddr, SocketAddr)>,
2236 ) {
2237 let mut callers = self
2238 .caller_endpoints
2239 .lock()
2240 .unwrap_or_else(|p| p.into_inner());
2241 callers.remove(&connection);
2242 if let Some(endpoints) = endpoints {
2243 callers.insert(connection, endpoints);
2244 }
2245 }
2246
2247 #[cfg(test)]
2248 pub(crate) fn with_caller_lookup_for_test(
2249 mut self,
2250 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2251 ) -> Self {
2252 self.caller_lookup = Some(Arc::new(lookup));
2253 self
2254 }
2255
2256 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2257 let endpoints = self
2258 .caller_endpoints
2259 .lock()
2260 .unwrap_or_else(|p| p.into_inner())
2261 .get(&connection)
2262 .copied();
2263 #[cfg(test)]
2264 let lookup = self.caller_lookup.clone();
2265 tokio::task::spawn_blocking(move || {
2266 #[cfg(test)]
2267 if let Some(lookup) = lookup {
2268 return lookup(endpoints);
2269 }
2270 endpoints
2271 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2272 .unwrap_or_default()
2273 })
2274 .await
2275 .unwrap_or_default()
2276 }
2277
2278 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2279 frame.header.channel == 0
2280 && frame.header.ty == FrameType::Request
2281 && self
2282 .forwarding
2283 .module_endpoint_for_connection(ctx.connection_id)
2284 .is_ok_and(|endpoint| endpoint.is_none())
2285 && parse_client_control_request(&frame.body)
2286 .ok()
2287 .is_some_and(|request| privileged_operation(&request).is_some())
2288 }
2289
2290 pub(crate) async fn handle_control_frame_timed(
2291 &self,
2292 ctx: &RouteCtx,
2293 frame: Frame,
2294 dispatch_started_at: Option<StdInstant>,
2295 ) -> Result<Vec<Frame>, RouterError> {
2296 let first = ctx.egress.first_ingress_frame();
2297 self.handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
2298 .await
2299 }
2300
2301 pub(crate) async fn handle_control_frame_ingress(
2302 &self,
2303 ctx: &RouteCtx,
2304 frame: Frame,
2305 dispatch_started_at: Option<StdInstant>,
2306 first: bool,
2307 ) -> Result<Vec<Frame>, RouterError> {
2308 if ctx.egress.ingress_terminal() {
2309 return Ok(Vec::new());
2310 }
2311 #[cfg(test)]
2312 self.claims.record_dispatch(frame.header.corr);
2313 let claim_binding = self.claims.binding(ctx.connection_id);
2314 let mut prefix_attempt = (frame.header.ty == FrameType::Hello
2315 && claim_binding
2316 .as_ref()
2317 .is_some_and(|binding| binding.prefix_module_id().is_some()))
2318 .then(|| self.prefix_hello_attempt(ctx));
2319 if claim_binding
2320 .as_ref()
2321 .is_some_and(|binding| binding.prefix_module_id().is_some())
2322 {
2323 let registered = self
2324 .forwarding
2325 .module_endpoint_for_connection(ctx.connection_id)
2326 .map_err(RouterError::Forwarding)?
2327 .is_some();
2328 if !registered && frame.header.ty != FrameType::Hello {
2329 return self
2330 .refuse_claim_connection(ctx, &frame, "claim_hello_required")
2331 .await;
2332 }
2333 if registered && frame.header.ty == FrameType::Goodbye {
2334 ctx.egress.terminate_ingress();
2335 let result = self.handle_goodbye(ctx.connection_id);
2336 self.claims.connection_closed(ctx.connection_id);
2337 self.request_refusal_close(
2338 ctx.connection_id,
2339 Some(&ctx.egress),
2340 CloseReason::new("claim_registration_ended", "prefix registration ended"),
2341 );
2342 return result;
2343 }
2344 if registered && frame.header.ty == FrameType::Request {
2345 let allowed =
2346 serde_json::from_slice::<ControlOpProbe>(&frame.body).is_ok_and(|probe| {
2347 matches!(
2348 probe.op.as_str(),
2349 "catalog.update"
2350 | "catalog.update_role_declarations"
2351 | "scope.describe"
2352 | "scope.watch"
2353 | "supervisor.live_roots"
2354 | "activity.record"
2355 )
2356 });
2357 if !allowed {
2358 return Ok(vec![control_error_frame(
2359 &frame,
2360 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2361 "prefix registrations have only brokered module operations",
2362 )?]);
2363 }
2364 }
2365 if registered
2366 && frame.header.ty == FrameType::Push
2367 && !serde_json::from_slice::<ControlOpProbe>(&frame.body)
2368 .is_ok_and(|probe| probe.op == "route.status")
2369 {
2370 return Ok(vec![control_error_frame(
2371 &frame,
2372 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2373 "prefix registrations may push only route.status for routes they serve",
2374 )?]);
2375 }
2376 }
2377 match frame.header.ty {
2378 FrameType::Ping => Ok(vec![pong(&frame)?]),
2379 FrameType::Hello => {
2380 if claim_binding
2381 .as_ref()
2382 .is_some_and(|binding| binding.prefix_module_id().is_none())
2383 {
2384 return self
2385 .refuse_claim_connection(ctx, &frame, "claim_hello_not_allowed")
2386 .await;
2387 }
2388 let channel_spawn = self.supervisor.registration_spawn(ctx.connection_id);
2392 if let Some(spawn) = &channel_spawn {
2393 spawn.wait_launch().await;
2394 if !spawn.launch_ready() {
2397 ctx.egress.terminate_ingress();
2398 return Ok(Vec::new());
2399 }
2400 #[cfg(test)]
2401 self.pause_hello_ingress_for_test().await;
2402 } else if claim_binding.is_none() {
2403 if let Ok(hello) = serde_json::from_slice::<serde_json::Value>(&frame.body) {
2404 if let (Some(module), Some(nonce)) = (
2405 hello
2406 .get("manifest")
2407 .and_then(|value| value.get("module_id"))
2408 .and_then(|value| value.as_str()),
2409 hello.get("launch_nonce").and_then(|value| value.as_str()),
2410 ) {
2411 if !self
2412 .config_runtime
2413 .as_ref()
2414 .is_some_and(|owner| owner.module_is_gated(module))
2415 && !self.supervisor.wait_program_binding(module, nonce).await
2416 {
2417 self.request_refusal_close(
2418 ctx.connection_id,
2419 Some(&ctx.egress),
2420 CloseReason::new(
2421 "program_refused",
2422 "the spawned program binding failed",
2423 ),
2424 );
2425 return Ok(vec![control_error_frame(
2426 &frame,
2427 if self
2428 .config_runtime
2429 .as_ref()
2430 .is_some_and(|owner| owner.is_recovering())
2431 {
2432 "config_gated"
2433 } else {
2434 "program_refused"
2435 },
2436 "the spawned program binding failed",
2437 )?]);
2438 }
2439 }
2440 }
2441 }
2442 let handler = self.clone();
2443 let connection = ctx.connection_id;
2444 let egress = ctx.egress.clone();
2445 let correlation = frame.header.corr;
2446 let dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
2447 let span = tracing::Span::current();
2448 let hello_binding = claim_binding.clone();
2449 let replies = tokio::task::spawn_blocking(move || {
2450 tracing::dispatcher::with_default(&dispatcher, || {
2451 span.in_scope(|| {
2452 handler.handle_hello_checked(
2453 connection,
2454 Some(egress),
2455 frame,
2456 true,
2457 hello_binding,
2458 channel_spawn,
2459 )
2460 })
2461 })
2462 })
2463 .await
2464 .map_err(|error| RouterError::backend(0, correlation, error.to_string()))??;
2465 #[cfg(test)]
2466 if prefix_attempt.is_some() {
2467 self.claims.pause_prefix_commit_for_test(false).await;
2468 }
2469 if let Some(attempt) = &mut prefix_attempt {
2470 let accepted = !replies
2471 .iter()
2472 .any(|reply| reply.header.ty == FrameType::Error)
2473 && self
2474 .registry
2475 .get_module_by_connection(ctx.connection_id)
2476 .map_err(|err| RouterError::backend(0, correlation, err.to_string()))?
2477 .is_some();
2478 if accepted {
2479 attempt.accepted = true;
2480 } else {
2481 ctx.egress.terminate_ingress();
2482 for reply in replies {
2483 ctx.egress.send_flushed(reply).await?;
2484 }
2485 return Ok(Vec::new());
2486 }
2487 }
2488 Ok(replies)
2489 }
2490 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2491 FrameType::Cancel => {
2492 if self
2498 .agent_declarations
2499 .cancel(ctx.connection_id, frame.header.corr)
2500 || self
2501 .forwarding
2502 .operator_confirms()
2503 .cancel(ctx.connection_id, frame.header.corr)
2504 || self.cancel_config_approval(ctx.connection_id, frame.header.corr)
2505 || self
2506 .supervisor
2507 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2508 || self.cancel_activity_subscription(ctx, &frame)?
2509 || self.cancel_catalog_subscription(ctx.connection_id, frame.header.corr)
2510 || self.cancel_scope_subscription(ctx.connection_id, frame.header.corr)
2511 {
2512 Ok(Vec::new())
2513 } else {
2514 Ok(vec![control_error_frame(
2515 &frame,
2516 "unknown_subscription",
2517 "no subscription has this correlation id",
2518 )?])
2519 }
2520 }
2521 FrameType::Request => {
2522 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2525 let claim_present = op.as_ref().is_some_and(|probe| probe.op == "claim.present");
2526 if let Some(refusal) = self
2527 .recovery_request_refusal(
2528 ctx,
2529 &frame,
2530 op.as_ref().map(|probe| probe.op.as_str()),
2531 )
2532 .await?
2533 {
2534 if claim_present {
2535 self.reclaim_presented_expiry(&frame);
2536 return self
2537 .refuse_claim_connection(ctx, &frame, "config_gated")
2538 .await;
2539 }
2540 return Ok(vec![refusal]);
2541 }
2542 if claim_present {
2543 return self.handle_claim_present(ctx, &frame, first).await;
2544 }
2545 if op.as_ref().is_some_and(|probe| probe.op == "claim.mint") {
2546 return self.handle_claim_mint(ctx, &frame).await;
2547 }
2548 if let Some(refusal) = self.registration_operation_refusal(
2549 ctx,
2550 &frame,
2551 op.as_ref().map(|probe| probe.op.as_str()),
2552 )? {
2553 return Ok(vec![refusal]);
2554 }
2555 if op
2556 .as_ref()
2557 .is_some_and(|probe| probe.op == ops::CATALOG_WATCH)
2558 {
2559 return self.handle_catalog_watch(ctx, frame);
2560 }
2561 if op
2562 .as_ref()
2563 .is_some_and(|probe| probe.op == ops::SCOPE_WATCH)
2564 {
2565 return self.handle_scope_watch(ctx, frame);
2566 }
2567 if op
2568 .as_ref()
2569 .is_some_and(|probe| probe.op == MODULE_TO_SUBC_OP_CATALOG_UPDATE)
2570 && subc_protocol::provider_declarations::catalog_update_has_provider_declarations(&frame.body)
2571 .unwrap_or(false)
2572 {
2573 return Ok(vec![control_error_frame(
2574 &frame,
2575 error_codes::CATALOG_UPDATE_FROZEN_DECLARATIONS,
2576 "provider_declarations are fixed at HELLO; restart the module to change them",
2577 )?]);
2578 }
2579 if let Some(probe) = op
2580 .as_ref()
2581 .filter(|probe| config_gate::is_module_config_op(&probe.op))
2582 {
2583 return self
2584 .handle_module_config_request(ctx, frame, &probe.op)
2585 .await;
2586 }
2587 if op
2588 .as_ref()
2589 .is_some_and(|probe| config_gate::is_config_op(&probe.op))
2590 {
2591 return self.handle_config_gate_request(ctx, frame).await;
2592 }
2593 if op
2594 .as_ref()
2595 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2596 {
2597 return self.handle_activity_record(ctx, frame).await;
2598 }
2599 if op.as_ref().is_some_and(|probe| {
2600 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2601 }) {
2602 return self.handle_activity_read(ctx, frame).await;
2603 }
2604 if op
2605 .as_ref()
2606 .is_some_and(|probe| probe.op == "operator.confirm")
2607 {
2608 return self.handle_operator_confirm(ctx, frame);
2609 }
2610 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2611 return self
2612 .observe_control_wait(
2613 ctx,
2614 None,
2615 ops::APP_OPEN,
2616 self.handle_app_open(ctx, frame),
2617 )
2618 .await;
2619 }
2620 if op
2621 .as_ref()
2622 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2623 {
2624 return self.handle_connection_describe(ctx, frame);
2625 }
2626 if op
2627 .as_ref()
2628 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2629 {
2630 return self.handle_scope_served_by(ctx.connection_id, frame);
2631 }
2632 if self
2633 .forwarding
2634 .module_endpoint_for_connection(ctx.connection_id)
2635 .map_err(RouterError::Forwarding)?
2636 .is_some()
2637 {
2638 if !is_known_module_request_op(&frame.body) {
2639 return Ok(vec![control_error_frame(
2640 &frame,
2641 "unsupported_control_frame",
2642 "module-originated channel-0 REQUEST is not supported",
2643 )?]);
2644 }
2645 let request = match parse_module_control_request_from_module(&frame.body) {
2646 Ok(request) => request,
2647 Err((err, ControlRequestBodyError::UnknownOp)) => {
2648 return Ok(vec![control_error_frame(
2649 &frame,
2650 "unsupported_control_frame",
2651 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2652 )?])
2653 }
2654 Err((err, ControlRequestBodyError::InvalidBody)) => {
2655 return Ok(vec![control_error_frame(
2656 &frame,
2657 "invalid_control_body",
2658 format!("malformed module control body: {err}"),
2659 )?])
2660 }
2661 };
2662 let op = module_control_request_op(&request);
2663 let corr = frame.header.corr;
2664 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2665 if let ModuleControlRequestFromModule::AgentCommands {
2666 started,
2667 finished,
2668 known,
2669 } = request
2670 {
2671 return self
2672 .agent_declarations
2673 .enqueue(
2674 self,
2675 ctx,
2676 frame,
2677 crate::resource_totals::Declaration {
2678 started,
2679 finished,
2680 known,
2681 },
2682 )
2683 .await;
2684 }
2685 let result =
2686 self.handle_module_control_request(ctx.connection_id, frame, request);
2687 log_slow_control_dispatch(
2688 dispatch_started_at,
2689 op,
2690 ctx.connection_id,
2691 corr,
2692 None,
2693 );
2694 return result;
2695 }
2696
2697 if is_known_module_request_op(&frame.body) {
2698 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2699 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2700 let code = if is_agents {
2701 "not_supervised"
2702 } else if op
2703 .as_ref()
2704 .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2705 {
2706 error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2707 } else {
2708 "not_registered"
2709 };
2710 return Ok(vec![control_error_frame(
2711 &frame,
2712 code,
2713 "module-originated control requests require the module's own registered connection",
2714 )?]);
2715 }
2716
2717 let request = match parse_client_control_request(&frame.body) {
2718 Ok(request) => request,
2719 Err((err, ControlRequestBodyError::UnknownOp)) => {
2720 return Ok(vec![control_error_frame(
2721 &frame,
2722 "unknown_control_op",
2723 format!("unknown client control op: {err}"),
2724 )?])
2725 }
2726 Err((err, ControlRequestBodyError::InvalidBody)) => {
2727 return Ok(vec![control_error_frame(
2728 &frame,
2729 "invalid_control_body",
2730 format!("malformed client control body: {err}"),
2731 )?])
2732 }
2733 };
2734 let op = client_control_request_op(&request);
2735 let corr = frame.header.corr;
2736 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2737 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2738 #[cfg(test)]
2739 if let Some(delay) = self.control_dispatch_delay {
2740 tokio::time::sleep(delay).await;
2741 }
2742 let result = self
2743 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2744 .await;
2745 if let Some(timings) = route_open_timings.as_mut() {
2746 timings.finish();
2747 }
2748 log_slow_control_dispatch(
2749 dispatch_started_at,
2750 op,
2751 ctx.connection_id,
2752 corr,
2753 route_open_timings.as_ref(),
2754 );
2755 result
2756 }
2757 FrameType::Push => {
2758 let Some(endpoint) = self
2759 .forwarding
2760 .module_endpoint_for_connection(ctx.connection_id)
2761 .map_err(RouterError::Forwarding)?
2762 else {
2763 return Ok(vec![control_error_frame(
2764 &frame,
2765 "unsupported_control_frame",
2766 "client-originated channel-0 PUSH is not supported",
2767 )?]);
2768 };
2769 self.handle_status_update(endpoint, frame)
2770 }
2771 FrameType::Response | FrameType::Error
2772 if self
2773 .forwarding
2774 .module_endpoint_for_connection(ctx.connection_id)
2775 .map_err(RouterError::Forwarding)?
2776 .is_some() =>
2777 {
2778 self.handle_module_relay_response(ctx.connection_id, frame)
2779 }
2780 ty => Ok(vec![control_error_frame(
2781 &frame,
2782 "unsupported_control_frame",
2783 format!("unsupported channel-0 frame {ty:?}"),
2784 )?]),
2785 }
2786 }
2787
2788 #[cfg(test)]
2789 fn pause_cleanup_for_test(&self, stage: u8) {
2790 let mut slot = self.cleanup_pause.lock().unwrap_or_else(|p| p.into_inner());
2791 if slot.as_ref().is_some_and(|(at, _)| *at == stage) {
2792 let (_, (reached, release)) = slot.take().unwrap();
2793 drop(slot);
2794 let _ = reached.send(());
2795 let _ = release.recv_timeout(Duration::from_secs(5));
2796 }
2797 }
2798
2799 #[cfg(all(test, unix, feature = "test-support"))]
2802 pub(crate) fn after_nonce_attestation_for_test(&self, hook: impl FnOnce() + Send + 'static) {
2803 *self
2804 .after_nonce_attestation
2805 .lock()
2806 .unwrap_or_else(|p| p.into_inner()) = Some(Box::new(hook));
2807 }
2808
2809 #[cfg(test)]
2810 pub(super) fn pause_after_nonce_attestation_for_test(&self) {
2811 let hook = self
2812 .after_nonce_attestation
2813 .lock()
2814 .unwrap_or_else(|p| p.into_inner())
2815 .take();
2816 if let Some(hook) = hook {
2817 hook();
2818 }
2819 }
2820
2821 #[cfg(test)]
2822 async fn pause_hello_ingress_for_test(&self) {
2823 let pause = self
2824 .hello_ingress_pause
2825 .lock()
2826 .unwrap_or_else(|p| p.into_inner())
2827 .take();
2828 if let Some((reached, release)) = pause {
2829 let _ = reached.send(());
2830 let _ = release.await;
2831 }
2832 }
2833
2834 pub fn cleanup_connection(
2835 &self,
2836 connection_id: ConnectionId,
2837 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2838 self.agent_declarations.close(connection_id);
2852 self.close_catalog_subscriptions(connection_id);
2853 self.close_scope_subscriptions(connection_id);
2854 #[cfg(test)]
2855 self.pause_cleanup_for_test(0);
2856 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
2857 spawn.retire_registration_channel();
2858 }
2859 self.supervisor
2860 .forget_registration_connection(connection_id);
2861 #[cfg(test)]
2862 self.pause_cleanup_for_test(1);
2863 self.local_connections
2864 .lock()
2865 .unwrap_or_else(|p| p.into_inner())
2866 .remove(&connection_id);
2867 self.caller_endpoints
2868 .lock()
2869 .unwrap_or_else(|p| p.into_inner())
2870 .remove(&connection_id);
2871 self.build_class_warned
2872 .lock()
2873 .unwrap_or_else(|p| p.into_inner())
2874 .remove(&connection_id);
2875 self.observed_peers
2876 .lock()
2877 .unwrap_or_else(|p| p.into_inner())
2878 .remove(&connection_id);
2879 self.captured_peers
2880 .lock()
2881 .unwrap_or_else(|p| p.into_inner())
2882 .remove(&connection_id);
2883 self.config_desktop_stamps
2884 .lock()
2885 .unwrap_or_else(|p| p.into_inner())
2886 .remove(&connection_id);
2887 self.operator_apps
2888 .lock()
2889 .unwrap_or_else(|p| p.into_inner())
2890 .remove(&connection_id);
2891 self.forwarding
2892 .operator_confirms()
2893 .connection_closed(connection_id);
2894 let cleanup = self.cleanup_connection_with_end_reason(
2895 connection_id,
2896 RegistrationEndReason::ConnectionClosed,
2897 );
2898 self.claims.connection_closed(connection_id);
2901 cleanup
2902 }
2903
2904 fn cleanup_connection_with_end_reason(
2905 &self,
2906 connection_id: ConnectionId,
2907 requested_reason: RegistrationEndReason,
2908 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2909 #[cfg(test)]
2910 if let Some(pause) = self
2911 .activity_request_pause
2912 .lock()
2913 .unwrap()
2914 .as_ref()
2915 .filter(|pause| pause.connection == connection_id)
2916 {
2917 pause
2918 .cleanup_started
2919 .store(true, std::sync::atomic::Ordering::Release);
2920 }
2921 if let Some(runtime) = &self.config_runtime {
2922 runtime.disconnected(connection_id);
2923 }
2924 self.agent_declarations.close(connection_id);
2925 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2926 self.registry
2927 .get_module_by_connection(connection_id)?
2928 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2929 .and_then(|module| module.registration_end_reason().ok().flatten())
2930 .unwrap_or(requested_reason)
2931 } else {
2932 requested_reason
2933 };
2934 let crash_closed = self
2935 .registry
2936 .get_module_by_connection(connection_id)?
2937 .and_then(|registration| {
2938 self.forwarding
2939 .module_endpoint_for_connection(connection_id)
2940 .ok()
2941 .flatten()
2942 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2943 .map(|routes| (registration.manifest.module_id, routes))
2944 });
2945 let crash_closed = crash_closed.map(|(module_id, routes)| {
2946 let terminal = match self.supervisor.get(&module_id) {
2947 None => false,
2948 Some(module) => match module.will_recover_after_connection_loss() {
2949 Ok(will_recover) => !will_recover,
2950 Err(err) => {
2951 warn!(
2952 %module_id,
2953 error = %err,
2954 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2955 );
2956 false
2957 }
2958 },
2959 };
2960 let (reason, drained) = match self
2964 .forwarding
2965 .daemon_drain_result_for_connection(connection_id)
2966 {
2967 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2968 Ok(None) => (RouteCloseReason::Crash, false),
2969 Err(err) => {
2970 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2971 (RouteCloseReason::Crash, false)
2972 }
2973 };
2974 (module_id, routes, reason, drained, terminal)
2975 });
2976 let registrations = self.deregister_connection(connection_id, end_reason);
2977 let ended_authority = self
2980 .forwarding
2981 .target_authority_for_connection(connection_id)
2982 .ok()
2983 .flatten();
2984 let cleanup = if crash_closed.is_some() {
2985 self.forwarding.cleanup_connection_counted(connection_id)
2986 } else {
2987 self.forwarding
2993 .cleanup_connection(connection_id)
2994 .map(|released| crate::forwarding::ConnectionCleanup {
2995 released,
2996 abandoned_relays: 0,
2997 })
2998 };
2999 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
3004 let abandoned = cleanup
3005 .as_ref()
3006 .map_or(0, |cleanup| cleanup.abandoned_relays);
3007 send_route_control_pushes(
3008 &self.forwarding,
3009 routes,
3010 ClientControlPush::RouteClosed {
3011 module_id,
3012 channels: Vec::new(),
3013 reason,
3014 drained,
3015 abandoned,
3016 excluded_subscriptions: 0,
3017 terminal: Some(terminal),
3018 },
3019 );
3020 }
3021 if let Ok(cleanup) = cleanup {
3022 self.emit_route_goodbyes(cleanup.released);
3023 }
3024 if let Some(record) = ended_authority {
3025 self.change_spawn_authority(&record);
3026 }
3027 if matches!(®istrations, Ok(r) if !r.is_empty()) {
3031 crate::supervise::notify_registration_release();
3032 self.capability_evaluator.wake_deadline_loop();
3033 self.refresh_capability_requirements();
3034 }
3035 self.supervisor.remove_spawn_subscribers(connection_id);
3036 self.activity_streams
3037 .lock()
3038 .unwrap_or_else(|p| p.into_inner())
3039 .retain(|(id, _), task| {
3040 if *id == connection_id {
3041 task.abort();
3042 false
3043 } else {
3044 true
3045 }
3046 });
3047 self.hello_launch_nonces
3050 .lock()
3051 .unwrap_or_else(|poisoned| poisoned.into_inner())
3052 .forget(connection_id);
3053 self.scopes
3054 .write()
3055 .unwrap_or_else(|poisoned| poisoned.into_inner())
3056 .release_connection(connection_id);
3057 registrations
3058 }
3059
3060 pub(crate) fn handle_route_goodbye(
3061 &self,
3062 connection_id: ConnectionId,
3063 route_channel: u16,
3064 route_epoch: u32,
3065 ) -> Result<bool, RouterError> {
3066 debug!(
3067 connection_id = connection_id.get(),
3068 route_channel, route_epoch, "handling route GOODBYE"
3069 );
3070 let RouteRelease::Removed(released_route) = self
3071 .forwarding
3072 .release_client_route(connection_id, route_channel, route_epoch)
3073 .map_err(RouterError::Forwarding)?
3074 else {
3075 return Ok(false);
3076 };
3077 self.emit_route_goodbyes(vec![released_route]);
3078 Ok(true)
3079 }
3080
3081 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
3082 scope_expiry::ScopeExpiryContext::new(self).emit_route_goodbyes(released_routes);
3083 }
3084
3085 fn send_abandoned_route_bind_goodbye(
3097 &self,
3098 module_sink: &crate::FrameSink,
3099 negotiated_ver: u8,
3100 module_channel: u16,
3101 module_epoch: u32,
3102 ) {
3103 let frame = match Frame::build_with_version(
3104 negotiated_ver,
3105 FrameType::Goodbye,
3106 control_flags(),
3107 module_channel,
3108 module_epoch,
3109 0,
3110 Vec::new(),
3111 ) {
3112 Ok(frame) => frame,
3113 Err(err) => {
3114 warn!(
3115 route_channel = module_channel,
3116 error = %err,
3117 "failed to build GOODBYE for abandoned route.bind"
3118 );
3119 return;
3120 }
3121 };
3122 crate::forwarding::send_module_route_goodbye(
3123 &self.counters,
3124 module_sink,
3125 frame,
3126 None,
3127 "abandoned route.bind",
3128 );
3129 }
3130
3131 #[cfg(test)]
3132 fn handle_hello(
3133 &self,
3134 connection_id: ConnectionId,
3135 egress: Option<crate::router::FrameSink>,
3136 frame: Frame,
3137 ) -> Result<Vec<Frame>, RouterError> {
3138 let binding = self.claims.binding(connection_id);
3139 let channel_spawn = self.supervisor.registration_spawn(connection_id);
3140 self.handle_hello_checked(connection_id, egress, frame, false, binding, channel_spawn)
3141 }
3142
3143 fn handle_hello_checked(
3144 &self,
3145 connection_id: ConnectionId,
3146 sink: Option<crate::FrameSink>,
3147 frame: Frame,
3148 wait_for_gate: bool,
3149 prefix_binding: Option<claims::ClaimBinding>,
3150 channel_spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3151 ) -> Result<Vec<Frame>, RouterError> {
3152 let primary_sink = channel_spawn.as_ref().and_then(|_| sink.clone());
3153 #[cfg(test)]
3154 if prefix_binding.is_some() {
3155 self.claims.pause_prefix_hello_for_test(5);
3156 match self.claims.take_prefix_hello_failure_for_test() {
3157 Some("error") => {
3158 return Err(RouterError::backend(
3159 0,
3160 frame.header.corr,
3161 "injected HELLO failure",
3162 ))
3163 }
3164 Some("panic") => panic!("injected HELLO task panic"),
3165 _ => {}
3166 }
3167 }
3168 if let Some(binding) = &prefix_binding {
3169 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3170 let module = value
3171 .as_ref()
3172 .and_then(|value| value["manifest"]["module_id"].as_str());
3173 if !binding.admit_prefix_hello(module)
3174 || !self.prefix_binding_live(binding, connection_id)
3175 || !module.is_some_and(|module| {
3176 self.supervisor
3177 .prefix_claim_authorized(&binding.registration.module_id, module)
3178 })
3179 || binding.registration.check().is_err()
3180 {
3181 return Ok(vec![control_error_frame(
3182 &frame,
3183 "claim_prefix_refused",
3184 "HELLO must name the prefix claim's exact module id once",
3185 )?]);
3186 }
3187 }
3188 struct RefuseUnregisteredChannel {
3189 spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3190 sink: Option<crate::FrameSink>,
3191 }
3192 impl Drop for RefuseUnregisteredChannel {
3193 fn drop(&mut self) {
3194 if let Some(spawn) = &self.spawn {
3195 if !spawn.registered() {
3196 if let Some(sink) = &self.sink {
3197 sink.terminate_ingress();
3198 }
3199 spawn.retire_registration_channel();
3200 }
3201 }
3202 }
3203 }
3204 let _channel_attempt = RefuseUnregisteredChannel {
3205 spawn: channel_spawn.clone(),
3206 sink: sink.clone(),
3207 };
3208 if let Some(spawn) = &channel_spawn {
3209 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3210 if let Err(code) = spawn.admit_hello(
3211 value
3212 .as_ref()
3213 .and_then(|value| value["manifest"]["module_id"].as_str()),
3214 ) {
3215 return Ok(vec![control_error_frame(
3216 &frame,
3217 code,
3218 "registration channel HELLO refused",
3219 )?]);
3220 }
3221 }
3222 debug!(
3223 connection_id = connection_id.get(),
3224 corr = frame.header.corr,
3225 "handling HELLO"
3226 );
3227 if self
3231 .registry
3232 .get_module_by_connection(connection_id)
3233 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3234 .is_some()
3235 {
3236 return Ok(vec![control_error_frame(
3237 &frame,
3238 "invalid_hello",
3239 "connection is already registered as a module",
3240 )?]);
3241 }
3242 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
3243 Ok(value) => value,
3244 Err(err) => {
3245 return Ok(vec![control_error_frame(
3246 &frame,
3247 "invalid_hello",
3248 format!("malformed HELLO body: {err}"),
3249 )?])
3250 }
3251 };
3252 if let Err(detail) = catalog::validate_hello_declarations(&hello_value) {
3253 return Ok(vec![control_error_frame(
3254 &frame,
3255 error_codes::INVALID_PROVIDER_DECLARATIONS,
3256 detail,
3257 )?]);
3258 }
3259 if let Some(module) = hello_value
3260 .get("manifest")
3261 .and_then(|value| value.get("module_id"))
3262 .and_then(|value| value.as_str())
3263 {
3264 let recovering = self
3265 .config_runtime
3266 .as_ref()
3267 .is_some_and(|owner| owner.check_recovery().unwrap_or(true));
3268 if recovering
3272 && channel_spawn.as_ref().is_none_or(|spawn| {
3273 !self.supervisor.has_live_verified_channel_binding(spawn)
3274 || spawn.check().is_err()
3275 })
3276 && !(channel_spawn.is_none()
3277 && hello_value
3278 .get("launch_nonce")
3279 .and_then(|value| value.as_str())
3280 .is_some_and(|nonce| {
3281 self.supervisor.spawned_consumer_authorized(module, nonce)
3282 && self
3283 .supervisor
3284 .has_live_verified_program_binding(module, nonce)
3285 }))
3286 {
3287 return Ok(vec![control_error_frame(
3288 &frame,
3289 "config_gated",
3290 "recovery admits only previously bound supervised launches",
3291 )?]);
3292 }
3293 if self
3294 .config_runtime
3295 .as_ref()
3296 .is_some_and(|owner| owner.module_is_gated(module))
3297 && (channel_spawn
3298 .as_ref()
3299 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3300 || self.config_runtime.as_ref().is_some_and(|owner| {
3301 !recovering && owner.check_module_hold(module).is_err()
3302 }))
3303 {
3304 return Ok(vec![control_error_frame(
3305 &frame,
3306 "reserved_module",
3307 "a gated identity belongs only to its supervised child",
3308 )?]);
3309 }
3310 }
3311 if channel_spawn.is_none() {
3312 if let (Some(module), Some(nonce)) = (
3313 hello_value
3314 .get("manifest")
3315 .and_then(|value| value.get("module_id"))
3316 .and_then(|value| value.as_str()),
3317 hello_value
3318 .get("launch_nonce")
3319 .and_then(|value| value.as_str()),
3320 ) {
3321 let binding = self.supervisor.has_program_binding_record(module, nonce);
3322 let required = self
3323 .config_runtime
3324 .as_ref()
3325 .is_some_and(|owner| owner.module_is_gated(module));
3326 if let Some(error) = (binding || required)
3327 .then(|| {
3328 self.supervisor
3329 .check_current_program_binding(module, nonce)
3330 .err()
3331 })
3332 .flatten()
3333 {
3334 let code = if error.code == "program_refused" {
3335 "program_refused"
3336 } else {
3337 "config_delivery_deferred"
3338 };
3339 self.request_refusal_close(
3340 connection_id,
3341 sink.as_ref(),
3342 CloseReason::new(code, error.detail.clone()),
3343 );
3344 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3345 }
3346 }
3347 }
3348 if let Err(err) = validate_hello_display_name(&hello_value) {
3349 return Ok(vec![control_error_frame(
3350 &frame,
3351 "invalid_manifest",
3352 err.to_string(),
3353 )?]);
3354 }
3355 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
3356 return Ok(vec![control_error_frame(
3357 &frame,
3358 "invalid_capability_grammar",
3359 err.to_string(),
3360 )?]);
3361 }
3362 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
3363 return Ok(vec![control_error_frame(
3364 &frame,
3365 "invalid_manifest",
3366 err.to_string(),
3367 )?]);
3368 }
3369 if let Err(err) = validate_hello_event_declarations(&hello_value) {
3370 return Ok(vec![control_error_frame(
3371 &frame,
3372 "invalid_event_declaration",
3373 err.to_string(),
3374 )?]);
3375 }
3376 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
3377 return Ok(vec![control_error_frame(
3378 &frame,
3379 "invalid_manifest",
3380 err.to_string(),
3381 )?]);
3382 }
3383 if let Some(provenance) = hello_value
3384 .get("manifest")
3385 .and_then(|manifest| manifest.get("provenance"))
3386 {
3387 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
3388 return Ok(vec![control_error_frame(
3389 &frame,
3390 "invalid_manifest",
3391 format!("malformed manifest provenance: {err}"),
3392 )?]);
3393 }
3394 }
3395 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
3396 let module_id = hello_value["manifest"]["module_id"]
3397 .as_str()
3398 .unwrap_or("<unknown>");
3399 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
3400 }
3401 if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
3402 let module_id = hello_value
3403 .get("manifest")
3404 .and_then(|m| m.get("module_id"))
3405 .and_then(serde_json::Value::as_str)
3406 .unwrap_or("unknown");
3407 warn!(module_id, %reason, "dropping operation_deferral from HELLO");
3408 }
3409 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
3410 let module_id = hello_value["manifest"]["module_id"]
3411 .as_str()
3412 .unwrap_or("<unknown>");
3413 warn!(module_id, %reason, "dropping flow_operations from HELLO");
3414 }
3415 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
3416 let module_id = hello_value["manifest"]["module_id"]
3417 .as_str()
3418 .unwrap_or("<unknown>");
3419 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
3420 }
3421 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
3422 Ok(hello) => hello,
3423 Err(err) => {
3424 return Ok(vec![control_error_frame(
3425 &frame,
3426 "invalid_hello",
3427 format!("malformed HELLO body: {err}"),
3428 )?])
3429 }
3430 };
3431
3432 if hello.protocol_ver != hello.manifest.protocol_ver {
3433 return Ok(vec![control_error_frame(
3434 &frame,
3435 "invalid_manifest",
3436 format!(
3437 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
3438 hello.protocol_ver, hello.manifest.protocol_ver
3439 ),
3440 )?]);
3441 }
3442
3443 if hello.manifest.module_id.trim().is_empty() {
3444 return Ok(vec![control_error_frame(
3445 &frame,
3446 "invalid_manifest",
3447 "manifest module_id must not be empty",
3448 )?]);
3449 }
3450
3451 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
3452 Ok(negotiated_ver) => negotiated_ver,
3453 Err(message) => {
3454 return Ok(vec![control_error_frame(
3455 &frame,
3456 "version_unsupported",
3457 message,
3458 )?])
3459 }
3460 };
3461
3462 let _publication_gate = if let Some(owner) = &self.config_runtime {
3468 let gate = owner
3469 .module_gate(&hello.manifest.module_id)
3470 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3471 Some(if wait_for_gate {
3472 gate.blocking_lock_owned()
3473 } else {
3474 match gate.try_lock_owned() {
3475 Ok(guard) => guard,
3476 Err(_) => {
3477 return Ok(vec![control_error_frame(
3478 &frame,
3479 "config_delivery_deferred",
3480 "module admission is busy",
3481 )?])
3482 }
3483 }
3484 })
3485 } else {
3486 None
3487 };
3488 #[cfg(test)]
3489 if let Some(hold) = &self.hello_before_publication_hold {
3490 hold.wait();
3491 hold.wait();
3492 }
3493 let module = &hello.manifest.module_id;
3494 let required = self
3495 .config_runtime
3496 .as_ref()
3497 .is_some_and(|owner| owner.module_is_gated(module));
3498 let recovering = self
3499 .config_runtime
3500 .as_ref()
3501 .is_some_and(|owner| owner.is_recovering());
3502 if recovering
3510 && channel_spawn.as_ref().is_none_or(|spawn| {
3511 !self.supervisor.has_live_verified_channel_binding(spawn) || spawn.check().is_err()
3512 })
3513 && !(channel_spawn.is_none()
3514 && hello.launch_nonce.as_deref().is_some_and(|nonce| {
3515 self.supervisor.spawned_consumer_authorized(module, nonce)
3516 && self
3517 .supervisor
3518 .has_live_verified_program_binding(module, nonce)
3519 }))
3520 {
3521 return Ok(vec![control_error_frame(
3522 &frame,
3523 "config_gated",
3524 "recovery admits only previously bound supervised launches",
3525 )?]);
3526 }
3527 if required
3528 && (channel_spawn
3529 .as_ref()
3530 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3531 || self
3532 .config_runtime
3533 .as_ref()
3534 .is_some_and(|owner| !recovering && owner.check_module_hold(module).is_err()))
3535 {
3536 return Ok(vec![control_error_frame(
3537 &frame,
3538 "reserved_module",
3539 "gated admission requires the current supervised child's registration channel and verified image binding",
3540 )?]);
3541 }
3542 if let Some(spawn) = &channel_spawn {
3543 if let Err(code) = spawn.check() {
3544 return Ok(vec![control_error_frame(
3545 &frame,
3546 code,
3547 "registration launch is no longer live",
3548 )?]);
3549 }
3550 } else if let Some(nonce) = hello.launch_nonce.as_deref() {
3551 if required || self.supervisor.has_program_binding_record(module, nonce) {
3552 if let Err(error) = self.supervisor.check_current_program_binding(module, nonce) {
3553 let code = if error.code == "program_refused" {
3554 "program_refused"
3555 } else {
3556 "config_delivery_deferred"
3557 };
3558 self.request_refusal_close(
3559 connection_id,
3560 sink.as_ref(),
3561 CloseReason::new(code, error.detail.clone()),
3562 );
3563 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3564 }
3565 }
3566 }
3567
3568 let swap_admission = match &channel_spawn {
3576 Some(spawn) if spawn.is_candidate() => SwapHelloAdmission::Candidate,
3577 Some(_) => SwapHelloAdmission::NotSwapping,
3578 None if prefix_binding.is_some() => SwapHelloAdmission::NotSwapping,
3579 None => self
3580 .supervisor
3581 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref()),
3582 };
3583 if swap_admission == SwapHelloAdmission::ProtocolNone {
3584 warn!(
3585 module_id = %hello.manifest.module_id,
3586 connection_id = connection_id.get(),
3587 "HELLO refused: the module_id belongs to a protocol:none module, which never registers"
3588 );
3589 return Ok(vec![control_error_frame(
3592 &frame,
3593 "swap_token_invalid",
3594 format!(
3595 "module_id '{}' is a protocol:none module; a protocol:none module may not register",
3596 hello.manifest.module_id
3597 ),
3598 )?]);
3599 }
3600 if swap_admission == SwapHelloAdmission::ChannelRequired {
3601 warn!(
3602 module_id = %hello.manifest.module_id,
3603 connection_id = connection_id.get(),
3604 "HELLO refused: the module_id is gated and the HELLO did not arrive over its registration channel"
3605 );
3606 return Ok(vec![control_error_frame(
3609 &frame,
3610 "swap_token_invalid",
3611 format!(
3612 "module_id '{}' is gated; it must register over its supervised registration channel",
3613 hello.manifest.module_id
3614 ),
3615 )?]);
3616 }
3617 if swap_admission == SwapHelloAdmission::Refused {
3618 warn!(
3619 module_id = %hello.manifest.module_id,
3620 connection_id = connection_id.get(),
3621 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
3622 );
3623 return Ok(vec![control_error_frame(
3624 &frame,
3625 "swap_token_invalid",
3626 format!(
3627 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
3628 hello.manifest.module_id
3629 ),
3630 )?]);
3631 }
3632 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
3633
3634 let verification = if swap_candidate && channel_spawn.is_none() {
3639 ReservedHelloVerification {
3640 nonce_verified: true,
3641 prefix_owner: None,
3642 }
3643 } else {
3644 let verification = if let Some(binding) = &prefix_binding {
3645 if !self.supervisor.prefix_claim_authorized(
3646 &binding.registration.module_id,
3647 &hello.manifest.module_id,
3648 ) {
3649 return Ok(vec![control_error_frame(
3650 &frame,
3651 "claim_prefix_refused",
3652 "prefix ownership changed",
3653 )?]);
3654 }
3655 Ok(ReservedHelloVerification {
3656 nonce_verified: true,
3657 prefix_owner: Some(binding.registration.module_id.clone()),
3658 })
3659 } else if channel_spawn.is_some() {
3660 self.supervisor
3661 .channel_hello_verification(&hello.manifest.module_id)
3662 } else {
3663 self.supervisor.reserved_hello_verification(
3664 &hello.manifest.module_id,
3665 hello.launch_nonce.as_deref(),
3666 )
3667 };
3668 match verification {
3669 Ok(verified) => verified,
3670 Err(rejection) => {
3671 let message = match rejection {
3672 ReservedHelloRejection::Exact { module_id } => format!(
3673 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
3674 ),
3675 ReservedHelloRejection::Prefix {
3676 prefix,
3677 owner_module_id,
3678 } => format!(
3679 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
3680 hello.manifest.module_id
3681 ),
3682 };
3683 return Ok(vec![control_error_frame(
3684 &frame,
3685 "reserved_module",
3686 message,
3687 )?]);
3688 }
3689 }
3690 };
3691
3692 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
3693 &hello.manifest.module_id,
3694 hello.manifest.capabilities.as_ref(),
3695 );
3696 if let Some(refusal) = reserved_capability_refusals.first() {
3697 let capability = refusal.capability.clone();
3698 let bound_module = refusal.claimants[0].clone();
3699 log_duplicate_claim_events(reserved_capability_refusals);
3700 return Ok(vec![control_error_frame(
3701 &frame,
3702 "reserved_capability",
3703 format!(
3704 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3705 capability, bound_module, hello.manifest.module_id
3706 ),
3707 )?]);
3708 }
3709
3710 if self
3713 .forwarding
3714 .connection_has_client_routes(connection_id)
3715 .map_err(RouterError::Forwarding)?
3716 {
3717 return Ok(vec![control_error_frame(
3718 &frame,
3719 "invalid_hello",
3720 "connection has open client routes and cannot also register as a module",
3721 )?]);
3722 }
3723
3724 self.hello_launch_nonces
3732 .lock()
3733 .unwrap_or_else(|poisoned| poisoned.into_inner())
3734 .record(
3735 connection_id,
3736 if prefix_binding.is_some() || channel_spawn.is_some() {
3737 None
3738 } else {
3739 hello.launch_nonce.as_deref()
3740 },
3741 );
3742 let control_ops = effective_module_control_ops(hello.control_ops);
3743 let prefix_submodule = verification.prefix_owner.is_some();
3744 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
3747 #[cfg(test)]
3748 struct PublicationFinished(Arc<std::sync::Barrier>);
3749 #[cfg(test)]
3750 impl Drop for PublicationFinished {
3751 fn drop(&mut self) {
3752 self.0.wait();
3753 }
3754 }
3755 #[cfg(test)]
3756 let _publication_finished = self.hello_after_launch_check_hold.as_ref().map(|hold| {
3757 hold.wait();
3758 hold.wait();
3759 PublicationFinished(hold.clone())
3760 });
3761 let publication_spawn = channel_spawn
3762 .as_ref()
3763 .or_else(|| prefix_binding.as_ref().map(|binding| &binding.registration));
3764 #[cfg(test)]
3765 if prefix_binding.is_some() {
3766 self.claims.pause_prefix_hello_for_test(7);
3767 }
3768 let _registration_publication = match publication_spawn {
3769 Some(spawn) => Some(match spawn.begin_publication() {
3770 Ok(guard) => guard,
3771 Err(code) => {
3772 return Ok(vec![control_error_frame(
3773 &frame,
3774 code,
3775 "registration ended before HELLO publication",
3776 )?])
3777 }
3778 }),
3779 None => None,
3780 };
3781 let (_prefix_live, _prefix_ownership) = if let Some(binding) = &prefix_binding {
3782 let live = self.claims.lock_prefix_binding(binding, connection_id);
3783 let ownership = self.supervisor.prefix_claim_publication(
3784 &binding.registration.module_id,
3785 &hello.manifest.module_id,
3786 );
3787 if live.is_none()
3788 || ownership.is_none()
3789 || !self.claim_primary_registered(binding.spawn, &binding.registration)
3790 || binding.prefix_module_id() != Some(hello.manifest.module_id.as_str())
3791 {
3792 return Ok(vec![control_error_frame(
3793 &frame,
3794 "claim_prefix_refused",
3795 "prefix authority ended before publication",
3796 )?]);
3797 }
3798 (live, ownership)
3799 } else {
3800 (None, None)
3801 };
3802 if let Some(spawn) = publication_spawn {
3803 self.forwarding
3804 .bind_spawn(connection_id, spawn)
3805 .map_err(RouterError::Forwarding)?;
3806 }
3807 let hello_authority = match (&channel_spawn, &prefix_binding) {
3817 (Some(spawn), _) => Some(spawn.authority.clone()),
3818 (None, None) => hello.launch_nonce.as_deref().and_then(|nonce| {
3819 self.supervisor
3820 .spawned_consumer_record(&hello.manifest.module_id, nonce)
3821 }),
3822 (None, Some(_)) => None,
3823 }
3824 .map(|record| HelloAuthority {
3825 record,
3826 requested: hello
3827 .manifest
3828 .provider_declarations
3829 .clone()
3830 .unwrap_or_default(),
3831 froze: Default::default(),
3832 });
3833 #[cfg(test)]
3834 if prefix_binding.is_some() {
3835 self.claims.pause_prefix_hello_for_test(8);
3836 }
3837 if swap_candidate {
3838 let module_id = hello.manifest.module_id.clone();
3839 let mut differing_declarations = Default::default();
3840 let reply = self.register_swap_candidate(
3841 connection_id,
3842 sink,
3843 &frame,
3844 hello.manifest,
3845 negotiated_ver,
3846 control_ops,
3847 hello_ack,
3848 hello_authority.as_ref(),
3849 &mut differing_declarations,
3850 #[cfg(windows)]
3851 hello.launch_nonce.as_deref(),
3852 )?;
3853 if let Some(spawn) = &channel_spawn {
3854 if self
3855 .registry
3856 .get_module_by_connection(connection_id)
3857 .ok()
3858 .flatten()
3859 .is_some()
3860 {
3861 if let Some(sink) = primary_sink {
3862 spawn.bind_primary_sink(sink);
3863 }
3864 spawn.hello_accepted();
3865 }
3866 } else if self
3867 .registry
3868 .get_module_by_connection(connection_id)
3869 .ok()
3870 .flatten()
3871 .is_some()
3872 {
3873 self.supervisor
3874 .retire_unused_registration_channel(&module_id, true);
3875 }
3876 if self
3877 .registry
3878 .get_module_by_connection(connection_id)
3879 .ok()
3880 .flatten()
3881 .is_some()
3882 {
3883 self.file_development_widening(
3884 hello_authority.map(|authority| authority.record),
3885 differing_declarations,
3886 );
3887 }
3888 return Ok(reply);
3889 }
3890 let registration = match self
3897 .registry
3898 .register_unpublished_verified_with_control_ops(
3899 hello.manifest,
3900 negotiated_ver,
3901 connection_id,
3902 control_ops,
3903 verification.nonce_verified,
3904 ) {
3905 Ok(registration) => registration,
3906 Err(RegistryError::DuplicateModuleId { module_id }) => {
3907 return Ok(vec![control_error_frame(
3908 &frame,
3909 "duplicate_module_id",
3910 format!(
3911 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
3912 ),
3913 )?])
3914 }
3915 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3916 return Ok(vec![control_error_frame(
3917 &frame,
3918 "invalid_module_id",
3919 err.to_string(),
3920 )?])
3921 }
3922 Err(err) => {
3923 return Ok(vec![control_error_frame(
3924 &frame,
3925 "registry_error",
3926 err.to_string(),
3927 )?])
3928 }
3929 };
3930 let differing_declarations =
3931 match self.admit_hello_authority(connection_id, hello_authority.as_ref()) {
3932 Ok(differing) => differing,
3933 Err(err) => {
3934 if matches!(
3935 self.rollback_unpublished_hello(connection_id),
3936 Ok(r) if !r.is_empty()
3937 ) {
3938 crate::supervise::notify_registration_release();
3939 }
3940 return Err(RouterError::Forwarding(err));
3941 }
3942 };
3943 #[cfg(test)]
3944 self.hold_between_registry_and_forwarding();
3945
3946 let mut reply = Vec::new();
3947 let publication = self
3950 .registry
3951 .publish_registration_after(connection_id, || {
3952 if let Some(sink) = sink {
3953 let concurrency = manifest_concurrency(®istration.manifest);
3969 self.forwarding.register_module_connection_acked(
3970 connection_id,
3971 registration.manifest.module_id.clone(),
3972 negotiated_ver,
3973 concurrency,
3974 sink,
3975 crate::forwarding::ModuleHelloAck {
3976 frame: hello_ack,
3977 verified_prefix_owner: verification.prefix_owner,
3978 },
3979 )?;
3980 } else {
3981 reply.push(hello_ack);
3984 }
3985 #[cfg(test)]
3986 self.hold_between_ack_and_publication();
3987 Ok(())
3988 })
3989 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3990 let published = match publication {
3991 Ok(published) => published,
3992 Err(err) => {
3993 if matches!(
4000 self.rollback_unpublished_hello(connection_id),
4001 Ok(r) if !r.is_empty()
4002 ) {
4003 crate::supervise::notify_registration_release();
4004 }
4005 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
4006 return Ok(vec![control_error_frame(
4007 &frame,
4008 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
4009 "invalid_hello"
4010 } else {
4011 forwarding_error_code(&err)
4012 },
4013 err.to_string(),
4014 )?]);
4015 }
4016 };
4017 if !published {
4018 debug!(
4019 module_id = %registration.manifest.module_id,
4020 connection_id = connection_id.get(),
4021 "registration was removed before it could be published"
4022 );
4023 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
4024 return Ok(reply);
4025 }
4026
4027 drop(_prefix_ownership);
4028 drop(_prefix_live);
4029
4030 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
4039 info!(
4040 module_id = %registration.manifest.module_id,
4041 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
4042 );
4043 }
4044 if registration.manifest.registration_channel == Some(false)
4045 && self
4046 .supervisor
4047 .module_is_gated(®istration.manifest.module_id)
4048 && self
4049 .config_runtime
4050 .as_ref()
4051 .and_then(|owner| owner.registration_declaration(®istration.manifest.module_id))
4052 .as_deref()
4053 == Some("declared")
4054 {
4055 warn!(module_id = %registration.manifest.module_id,
4056 "HELLO registration_channel is false despite the approved channel declaration; registration remains admitted");
4057 }
4058
4059 self.apply_registration_capabilities(®istration);
4060
4061 self.supervisor
4062 .record_activity_registration(®istration.manifest.module_id);
4063
4064 info!(
4065 module_id = %registration.manifest.module_id,
4066 module_version = %registration.manifest.module_version,
4067 negotiated_ver,
4068 routable_provider = manifest_provides_routable_role(®istration.manifest),
4069 connection_id = connection_id.get(),
4070 "module registered"
4071 );
4072
4073 if let Some(spawn) = &channel_spawn {
4074 if let Some(sink) = primary_sink {
4075 spawn.bind_primary_sink(sink);
4076 }
4077 spawn.hello_accepted();
4078 } else if !prefix_submodule {
4079 self.supervisor
4080 .retire_unused_registration_channel(®istration.manifest.module_id, false);
4081 }
4082 self.file_development_widening(
4083 hello_authority.map(|authority| authority.record),
4084 differing_declarations,
4085 );
4086 #[cfg(windows)]
4087 if let Some(nonce) = hello.launch_nonce.as_deref() {
4088 self.supervisor
4089 .program_hello_accepted(®istration.manifest.module_id, nonce);
4090 }
4091
4092 #[cfg(test)]
4093 if let Some(hold) = &self.hello_inside_publication_hold {
4094 hold.wait();
4095 hold.wait();
4096 }
4097
4098 Ok(reply)
4099 }
4100
4101 #[allow(clippy::too_many_arguments)]
4118 fn register_swap_candidate(
4119 &self,
4120 connection_id: ConnectionId,
4121 sink: Option<crate::FrameSink>,
4122 frame: &Frame,
4123 manifest: ModuleManifest,
4124 negotiated_ver: u8,
4125 control_ops: Vec<String>,
4126 hello_ack: Frame,
4127 hello_authority: Option<&HelloAuthority>,
4128 differing_declarations: &mut ProviderDeclarations,
4129 #[cfg(windows)] launch_nonce: Option<&str>,
4130 ) -> Result<Vec<Frame>, RouterError> {
4131 let module_id = manifest.module_id.clone();
4132 self.supervisor
4135 .with_swap_candidate_admission(&module_id, || {
4136 let reply = self.publish_swap_candidate(
4137 connection_id,
4138 sink,
4139 frame,
4140 manifest,
4141 negotiated_ver,
4142 control_ops,
4143 hello_ack,
4144 hello_authority,
4145 differing_declarations,
4146 #[cfg(windows)]
4147 launch_nonce,
4148 );
4149 let admitted = self
4150 .registry
4151 .get_candidate(&module_id)
4152 .ok()
4153 .flatten()
4154 .is_some_and(|candidate| candidate.connection_id == connection_id);
4155 (reply, admitted)
4156 })
4157 .unwrap_or_else(|| {
4158 Ok(vec![control_error_frame(
4159 frame,
4160 "swap_token_invalid",
4161 "swap ended before candidate HELLO publication",
4162 )?])
4163 })
4164 }
4165
4166 #[allow(clippy::too_many_arguments)]
4167 fn publish_swap_candidate(
4168 &self,
4169 connection_id: ConnectionId,
4170 sink: Option<crate::FrameSink>,
4171 frame: &Frame,
4172 manifest: ModuleManifest,
4173 negotiated_ver: u8,
4174 control_ops: Vec<String>,
4175 hello_ack: Frame,
4176 hello_authority: Option<&HelloAuthority>,
4177 differing_declarations: &mut ProviderDeclarations,
4178 #[cfg(windows)] launch_nonce: Option<&str>,
4179 ) -> Result<Vec<Frame>, RouterError> {
4180 let module_id = manifest.module_id.clone();
4181 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
4182 manifest,
4183 negotiated_ver,
4184 connection_id,
4185 control_ops,
4186 ) {
4187 Ok(registration) => registration,
4188 Err(RegistryError::DuplicateModuleId { module_id }) => {
4189 return Ok(vec![control_error_frame(
4190 frame,
4191 "duplicate_module_id",
4192 format!(
4193 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
4194 ),
4195 )?])
4196 }
4197 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
4198 return Ok(vec![control_error_frame(
4199 frame,
4200 "invalid_module_id",
4201 err.to_string(),
4202 )?])
4203 }
4204 Err(err) => {
4205 return Ok(vec![control_error_frame(
4206 frame,
4207 "registry_error",
4208 err.to_string(),
4209 )?])
4210 }
4211 };
4212 *differing_declarations = match self.admit_hello_authority(connection_id, hello_authority) {
4213 Ok(differing) => differing,
4214 Err(err) => {
4215 if matches!(
4216 self.rollback_unpublished_hello(connection_id),
4217 Ok(r) if !r.is_empty()
4218 ) {
4219 crate::supervise::notify_registration_release();
4220 }
4221 return Err(RouterError::Forwarding(err));
4222 }
4223 };
4224 #[cfg(test)]
4225 self.hold_between_registry_and_forwarding();
4226 let mut reply = Vec::new();
4227 let publication = self
4228 .registry
4229 .publish_registration_after(connection_id, || {
4230 if let Some(sink) = sink {
4231 let concurrency = manifest_concurrency(®istration.manifest);
4235 self.forwarding.register_candidate_module_connection_acked(
4236 connection_id,
4237 module_id.clone(),
4238 negotiated_ver,
4239 concurrency,
4240 sink,
4241 hello_ack,
4242 )?;
4243 } else {
4244 reply.push(hello_ack);
4245 }
4246 #[cfg(test)]
4247 self.hold_between_ack_and_publication();
4248 Ok(())
4249 })
4250 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4251 let published = match publication {
4252 Ok(published) => published,
4253 Err(err) => {
4254 if matches!(
4255 self.rollback_unpublished_hello(connection_id),
4256 Ok(r) if !r.is_empty()
4257 ) {
4258 crate::supervise::notify_registration_release();
4259 }
4260 self.refuse_hello_authority(connection_id, hello_authority);
4261 return Ok(vec![control_error_frame(
4262 frame,
4263 forwarding_error_code(&err),
4264 err.to_string(),
4265 )?]);
4266 }
4267 };
4268 if !published {
4269 debug!(
4270 module_id = %module_id,
4271 connection_id = connection_id.get(),
4272 "swap candidate registration was removed before it could be published"
4273 );
4274 self.refuse_hello_authority(connection_id, hello_authority);
4275 return Ok(reply);
4276 }
4277 info!(
4278 module_id = %module_id,
4279 module_version = %registration.manifest.module_version,
4280 negotiated_ver,
4281 ready = registration.ready,
4282 connection_id = connection_id.get(),
4283 "swap candidate registered; not routable until cutover"
4284 );
4285 #[cfg(windows)]
4286 if let Some(nonce) = launch_nonce {
4287 self.supervisor.program_hello_accepted(&module_id, nonce);
4288 }
4289 Ok(reply)
4290 }
4291
4292 fn admit_hello_authority(
4298 &self,
4299 connection: ConnectionId,
4300 authority: Option<&HelloAuthority>,
4301 ) -> Result<ProviderDeclarations, ForwardingError> {
4302 let Some(authority) = authority else {
4303 return Ok(ProviderDeclarations::new());
4304 };
4305 let froze = authority
4306 .record
4307 .freeze_declarations(authority.requested.clone());
4308 authority
4309 .froze
4310 .store(froze.is_some(), std::sync::atomic::Ordering::Relaxed);
4311 self.forwarding
4312 .bind_target_authority(connection, authority.record.clone())
4313 .inspect_err(|_| self.refuse_hello_authority(connection, Some(authority)))?;
4314 Ok(froze.unwrap_or_default())
4315 }
4316
4317 fn refuse_hello_authority(&self, connection: ConnectionId, authority: Option<&HelloAuthority>) {
4322 let Some(authority) = authority else {
4323 return;
4324 };
4325 if let Err(err) = self.forwarding.unbind_target_authority(connection) {
4326 warn!(error = %err, "failed to unbind a refused HELLO from its spawn");
4327 }
4328 if authority
4329 .froze
4330 .swap(false, std::sync::atomic::Ordering::Relaxed)
4331 {
4332 authority.record.unfreeze_refused_declarations();
4333 }
4334 }
4335
4336 fn build_hello_ack(
4337 &self,
4338 frame: &Frame,
4339 negotiated_ver: u8,
4340 module_id: &str,
4341 ) -> Result<Frame, RouterError> {
4342 let ack = ModuleHelloAckBody {
4343 negotiated_ver,
4344 subc_ops: module_subc_ops(),
4345 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
4346 storage: self
4347 .storage_config
4348 .as_ref()
4349 .map(|cfg| cfg.descriptor_for(module_id)),
4350 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4351 };
4352 let body = serde_json::to_vec(&ack).map_err(|err| {
4353 RouterError::backend(
4354 0,
4355 frame.header.corr,
4356 format!("failed to encode HELLO_ACK: {err}"),
4357 )
4358 })?;
4359
4360 Frame::build_with_version(
4361 negotiated_ver,
4362 FrameType::HelloAck,
4363 control_flags(),
4364 0,
4365 0,
4366 frame.header.corr,
4367 body,
4368 )
4369 .map_err(RouterError::FrameBuild)
4370 }
4371
4372 async fn handle_client_control_request(
4373 &self,
4374 ctx: &RouteCtx,
4375 frame: Frame,
4376 request: ClientControlRequest,
4377 route_open_timings: &mut Option<RouteOpenTimings>,
4378 ) -> Result<Vec<Frame>, RouterError> {
4379 if let Some(owner) = &self.config_runtime {
4380 let module = match &request {
4381 ClientControlRequest::SupervisorRestart { module_id, .. }
4382 | ClientControlRequest::SupervisorReload { module_id }
4383 | ClientControlRequest::SupervisorSwap { module_id, .. }
4384 | ClientControlRequest::SupervisorSetEnabled {
4385 module_id,
4386 enabled: true,
4387 } => Some(module_id.as_str()),
4388 _ => None,
4389 };
4390 let rescan = matches!(
4391 &request,
4392 ClientControlRequest::SupervisorRescan { preview: false }
4393 | ClientControlRequest::SupervisorRescanChecked { preview: false, .. }
4394 );
4395 let held = if module.is_some() || rescan {
4396 let module = module.map(str::to_owned);
4397 owner
4398 .on_blocking(move |owner| owner.checked_admission_hold(module.as_deref()))
4399 .await
4400 } else {
4401 Ok(())
4402 };
4403 if let Err(error) = held {
4404 if error.code != "config_delivery_deferred" {
4405 let code = if error.code == "store_tampered" {
4406 "store_tampered"
4407 } else {
4408 "config_gated"
4409 };
4410 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
4411 }
4412 }
4413 }
4414 if let Some(operation) = privileged_operation(&request) {
4415 let caller = self.resolve_caller(ctx.connection_id).await;
4416 let op = client_control_request_op(&request);
4417 let text = prompt_text(operation, &caller);
4418 match self.privileged_operator_prompt {
4419 PrivilegedOperatorPrompt::Log => {
4420 info!(target: "subc_daemon::privileged_prompt", operation = op,
4421 target = ?operation, caller = %caller.description(), summary = %text,
4422 "privileged request would prompt the operator");
4423 }
4424 PrivilegedOperatorPrompt::Enforce => {
4425 let key =
4426 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
4427 let stamp = self
4428 .operator_apps
4429 .lock()
4430 .unwrap_or_else(|p| p.into_inner())
4431 .get(&ctx.connection_id)
4432 .cloned();
4433 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
4434 &self.operator_app_policy,
4435 &self.launch_table,
4436 stamp.as_ref(),
4437 )
4438 .then(|| self.shortcut_context(stamp, None, None));
4439 let outcome = self
4440 .forwarding
4441 .operator_confirms()
4442 .approve_privileged_with_shortcut(
4443 ctx,
4444 &frame,
4445 op,
4446 text,
4447 caller.description(),
4448 key,
4449 shortcut,
4450 )
4451 .await;
4452 if outcome.code != "confirmed" {
4453 return Ok(vec![outcome.frame(&frame)]);
4454 }
4455 }
4456 }
4457 }
4458 match request {
4459 ClientControlRequest::DaemonShutdownIntent { intent } => {
4460 if intent.as_str() != Some("restart") {
4461 return Ok(vec![control_error_frame(
4462 &frame,
4463 "invalid_params",
4464 "intent must be 'restart'",
4465 )?]);
4466 }
4467 if let Some(owner) = &self.config_runtime {
4472 match owner
4473 .on_blocking(|owner| owner.assign_daemon_restart())
4474 .await
4475 {
4476 Ok(Some(generation)) => {
4477 info!(
4478 generation,
4479 "daemon restart will warm up approved generation"
4480 )
4481 }
4482 Ok(None) => {}
4483 Err(error) => warn!(
4484 %error,
4485 "daemon restart intent not recorded; the next start boots the last active generation"
4486 ),
4487 }
4488 }
4489 let response = ClientControlResponse::DaemonShutdownIntent {
4490 intent: "restart".to_owned(),
4491 expires_at_ms: self.supervisor.announce_daemon_restart(),
4492 };
4493 Ok(vec![control_response_body_frame(
4494 &frame,
4495 &response,
4496 "ClientControlResponse::DaemonShutdownIntent",
4497 )?])
4498 }
4499 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
4500 ClientControlRequest::CatalogList { module_id } => {
4501 self.handle_catalog_list(frame, module_id)
4502 }
4503 ClientControlRequest::RouteOpen {
4504 target,
4505 identity,
4506 consumer_identity,
4507 consumer_capabilities,
4508 role_versions,
4509 busy_refusals,
4510 admission_facts,
4511 scope,
4512 ..
4513 } => {
4514 let extensions = match serde_json::from_slice::<subc_control::RouteOpenRequestExt>(
4515 &frame.body,
4516 ) {
4517 Ok(extensions) => extensions,
4518 Err(err) => {
4519 return Ok(vec![control_error_frame(
4520 &frame,
4521 "invalid_control_body",
4522 format!("malformed route.open extensions: {err}"),
4523 )?]);
4524 }
4525 };
4526 self.handle_route_open(
4527 ctx,
4528 frame,
4529 RouteOpenRequest {
4530 target,
4531 identity,
4532 consumer_identity,
4533 consumer_capabilities,
4534 role_versions,
4535 busy_refusals,
4536 admission_facts,
4537 scope,
4538 accepts_close_reasons: extensions.accepts_close_reasons,
4539 },
4540 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
4541 )
4542 .await
4543 }
4544 ClientControlRequest::RoutePoll {
4545 route_channel,
4546 route_epoch,
4547 kind,
4548 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
4549 ClientControlRequest::SupervisorList {} => {
4550 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
4551 self.agent_declarations.refuse(
4552 self,
4553 ctx,
4554 control_error_frame(
4555 &frame,
4556 "busy",
4557 "supervisor list work is at capacity; retry",
4558 )?,
4559 );
4560 return Ok(Vec::new());
4561 };
4562 let handler = self.clone();
4564 let sink = ctx.egress.clone();
4565 tokio::spawn(
4566 async move {
4567 let _permit = permit;
4568 let result = handler.handle_supervisor_list(frame.clone()).await;
4569 let replies = match result {
4570 Ok(replies) => replies,
4571 Err(error) => {
4572 control_error_frame(&frame, "backend_error", error.to_string())
4573 .into_iter()
4574 .collect()
4575 }
4576 };
4577 for reply in replies {
4578 if !matches!(
4579 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
4580 .await,
4581 Ok(Ok(()))
4582 ) {
4583 break;
4584 }
4585 }
4586 }
4587 .with_current_subscriber(),
4588 );
4589 Ok(Vec::new())
4590 }
4591 ClientControlRequest::SupervisorSpawnSnapshot {} => {
4592 self.handle_supervisor_spawn_snapshot(frame)
4593 }
4594 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
4595 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
4596 }
4597 ClientControlRequest::SupervisorRestart {
4598 module_id,
4599 drain_timeout_ms,
4600 } => {
4601 self.log_supervisor_request_received(
4602 ctx,
4603 frame.header.corr,
4604 ops::SUPERVISOR_RESTART,
4605 Some(&module_id),
4606 None,
4607 )?;
4608 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4609 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
4610 .await
4611 }
4612 ClientControlRequest::SupervisorSwap {
4613 module_id,
4614 ready_timeout_ms,
4615 } => {
4616 self.log_supervisor_request_received(
4617 ctx,
4618 frame.header.corr,
4619 ops::SUPERVISOR_SWAP,
4620 Some(&module_id),
4621 None,
4622 )?;
4623 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4624 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
4625 .await
4626 }
4627 ClientControlRequest::SupervisorReload { module_id } => {
4628 self.log_supervisor_request_received(
4629 ctx,
4630 frame.header.corr,
4631 ops::SUPERVISOR_RELOAD,
4632 Some(&module_id),
4633 None,
4634 )?;
4635 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4636 self.handle_supervisor_reload(frame, module_id, requester)
4637 .await
4638 }
4639 ClientControlRequest::SupervisorRescan { preview } => {
4640 if !preview {
4641 self.log_supervisor_request_received(
4642 ctx,
4643 frame.header.corr,
4644 ops::SUPERVISOR_RESCAN,
4645 None,
4646 None,
4647 )?;
4648 }
4649 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4650 let operation = self.handle_supervisor_rescan(
4651 frame,
4652 if preview {
4653 RescanMode::Preview
4654 } else {
4655 RescanMode::Execute
4656 },
4657 requester,
4658 );
4659 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
4660 .await
4661 }
4662 ClientControlRequest::SupervisorRescanChecked {
4663 preview,
4664 candidate,
4665 expect,
4666 } => {
4667 self.observe_control_wait(
4668 ctx,
4669 None,
4670 ops::SUPERVISOR_RESCAN,
4671 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
4672 )
4673 .await
4674 }
4675 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
4676 self.log_supervisor_request_received(
4677 ctx,
4678 frame.header.corr,
4679 ops::SUPERVISOR_RELEASE_RESERVED,
4680 Some(&module_id),
4681 None,
4682 )?;
4683 self.handle_supervisor_release_reserved(frame, module_id)
4684 .await
4685 }
4686 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
4687 self.log_supervisor_request_received(
4688 ctx,
4689 frame.header.corr,
4690 ops::SUPERVISOR_SET_ENABLED,
4691 Some(&module_id),
4692 Some(enabled),
4693 )?;
4694 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4695 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
4696 .await
4697 }
4698 ClientControlRequest::SupervisorHealthProbe { module_id } => {
4699 self.handle_supervisor_health_probe(ctx, frame, module_id)
4700 .await
4701 }
4702 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
4703 ClientControlRequest::SupervisorRoutes { module_id } => {
4704 self.handle_supervisor_routes(frame, module_id)
4705 }
4706 ClientControlRequest::SupervisorProvenance { module_id } => {
4707 self.handle_supervisor_provenance(ctx, frame, module_id)
4708 .await
4709 }
4710 ClientControlRequest::SupervisorStderrTail {
4711 module_id,
4712 max_lines,
4713 max_bytes,
4714 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
4715 ClientControlRequest::SupervisorTerminals { module_id } => {
4716 self.handle_supervisor_terminals(ctx, frame, module_id)
4717 .await
4718 }
4719 }
4720 }
4721
4722 fn log_supervisor_request_received(
4723 &self,
4724 ctx: &RouteCtx,
4725 corr: u64,
4726 op: &'static str,
4727 module_id: Option<&str>,
4728 enabled: Option<bool>,
4729 ) -> Result<(), RouterError> {
4730 let caller = self.supervisor_requester(ctx, corr)?;
4731 let caller = principal_label(&caller);
4732 info!(
4735 target: "control",
4736 op,
4737 module_id,
4738 enabled,
4739 connection_id = ctx.connection_id.get(),
4740 caller = %caller,
4741 "supervisor request received"
4742 );
4743 Ok(())
4744 }
4745
4746 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
4747 Ok(self
4748 .registry
4749 .get_module_by_connection(ctx.connection_id)
4750 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4751 .map(|registration| Principal::Reserved {
4752 module_id: registration.manifest.module_id,
4753 })
4754 .unwrap_or(Principal::Direct))
4755 }
4756
4757 fn handle_module_control_request(
4758 &self,
4759 connection_id: ConnectionId,
4760 frame: Frame,
4761 request: ModuleControlRequestFromModule,
4762 ) -> Result<Vec<Frame>, RouterError> {
4763 match request {
4764 ModuleControlRequestFromModule::RouteAuthorize {
4765 principal,
4766 scope,
4767 target,
4768 expected,
4769 } => self.handle_route_authorize(
4770 connection_id,
4771 frame,
4772 principal,
4773 scope,
4774 target,
4775 expected,
4776 ),
4777 ModuleControlRequestFromModule::CatalogUpdate {
4778 provides,
4779 capabilities,
4780 ready,
4781 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
4782 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
4783 self.handle_catalog_update_role_declarations(
4784 connection_id,
4785 frame,
4786 role_declarations,
4787 )
4788 }
4789 ModuleControlRequestFromModule::LiveRoots {} => {
4790 let registered = self
4791 .registry
4792 .get_module_by_connection(connection_id)
4793 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4794 let Some(registration) = registered else {
4795 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
4796 };
4797 let response = self
4798 .forwarding
4799 .live_roots(®istration.manifest.module_id)
4800 .map_err(RouterError::Forwarding)?;
4801 Ok(vec![control_response_body_frame(
4802 &frame,
4803 &response,
4804 "ModuleControlResponseToModule::LiveRoots",
4805 )?])
4806 }
4807 ModuleControlRequestFromModule::AgentCommands {
4808 started,
4809 finished,
4810 known,
4811 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
4812 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
4813 self.handle_scope_sync(connection_id, frame, generation, scopes)
4814 }
4815 ModuleControlRequestFromModule::ScopeApply {
4816 generation,
4817 upsert,
4818 end,
4819 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
4820 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
4821 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
4822 }
4823 _ => Ok(vec![control_error_frame(
4824 &frame,
4825 "unsupported_control_frame",
4826 "unsupported module control operation",
4827 )?]),
4828 }
4829 }
4830
4831 fn handle_scope_served_by(
4832 &self,
4833 connection_id: ConnectionId,
4834 frame: Frame,
4835 ) -> Result<Vec<Frame>, RouterError> {
4836 let registered = self
4837 .registry
4838 .get_module_by_connection(connection_id)
4839 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4840 let Some(registered) = registered else {
4841 return Ok(vec![control_error_frame(
4842 &frame,
4843 "not_registered",
4844 "scope.served_by requires an active module registration owned by this connection",
4845 )?]);
4846 };
4847 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
4848 {
4849 Ok(request) => request,
4850 Err(err) => {
4851 return Ok(vec![control_error_frame(
4852 &frame,
4853 "invalid_control_body",
4854 format!("malformed scope.served_by body: {err}"),
4855 )?])
4856 }
4857 };
4858 let answer = self
4859 .scopes
4860 .read()
4861 .unwrap_or_else(|poisoned| poisoned.into_inner())
4862 .served_by(
4863 ®istered.manifest.module_id,
4864 &request.scope_ref,
4865 request.scope_epoch,
4866 &request.module_id,
4867 );
4868 let reply = subc_control::ScopeServedByReply::new(
4869 answer,
4870 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4871 );
4872 Ok(vec![control_response_body_frame(
4873 &frame,
4874 &reply,
4875 "ScopeServedByReply",
4876 )?])
4877 }
4878
4879 fn handle_operator_confirm(
4880 &self,
4881 ctx: &RouteCtx,
4882 frame: Frame,
4883 ) -> Result<Vec<Frame>, RouterError> {
4884 use crate::operator_confirm::{audit, Outcome};
4885 let registration = self
4886 .registry
4887 .get_module_by_connection(ctx.connection_id)
4888 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4889 let Some(registration) = registration else {
4890 let outcome = Outcome::refusal("not_registered");
4891 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4892 return Ok(vec![outcome.frame(&frame)]);
4893 };
4894 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
4895 Ok(request) => request,
4896 Err(_) => {
4897 let outcome = Outcome::refusal("invalid_control_body");
4898 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4899 return Ok(vec![outcome.frame(&frame)]);
4900 }
4901 };
4902 let declared = matches!(
4903 request.kind,
4904 Some(subc_protocol::session::OperatorConfirmKind::Operation)
4905 ) && request.operation.as_ref().is_some_and(|operation| {
4906 registration
4907 .manifest
4908 .confirm_operations
4909 .as_ref()
4910 .is_some_and(|operations| operations.contains(operation))
4911 });
4912 let module_id = registration.manifest.module_id;
4913 let nonce = self
4917 .hello_launch_nonces
4918 .lock()
4919 .unwrap_or_else(|p| p.into_inner())
4920 .nonce(ctx.connection_id)
4921 .map(str::to_owned);
4922 let nonce_proven = self
4925 .confirm_channel_proven(ctx.connection_id, &module_id)
4926 .unwrap_or_else(|| {
4927 nonce.as_deref().is_some_and(|nonce| {
4928 self.supervisor
4929 .spawned_consumer_authorized(&module_id, nonce)
4930 })
4931 });
4932 let confirms = self.forwarding.operator_confirms();
4933 self.forwarding
4934 .with_operator_route(
4935 ctx.connection_id,
4936 request.route_channel,
4937 request.route_epoch,
4938 |binding| {
4939 let shortcut = binding
4940 .filter(|binding| {
4941 nonce_proven
4942 && crate::operator_confirm::valid_summary(&request.summary)
4943 && declared
4944 && crate::operator_confirm::ShortcutContext::route_eligible(
4945 &self.operator_app_policy,
4946 &self.launch_table,
4947 binding,
4948 )
4949 })
4950 .map(|binding| {
4951 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
4952 &self.operator_app_policy,
4953 binding,
4954 );
4955 let stamp = if phone.is_some() {
4956 None
4957 } else {
4958 binding.operator_app.clone()
4959 };
4960 self.shortcut_context(
4961 stamp,
4962 phone,
4963 Some((request.route_channel, request.route_epoch)),
4964 )
4965 });
4966 confirms.admit_with_shortcut(
4967 ctx,
4968 frame,
4969 module_id,
4970 nonce_proven,
4971 request,
4972 binding,
4973 shortcut,
4974 )
4975 },
4976 )
4977 .map_err(RouterError::Forwarding)
4978 }
4979
4980 fn shortcut_context(
4981 &self,
4982 stamp: Option<crate::operator_apps::OperatorAppStamp>,
4983 phone_fingerprint: Option<String>,
4984 route: Option<(u16, u32)>,
4985 ) -> crate::operator_confirm::ShortcutContext {
4986 crate::operator_confirm::ShortcutContext {
4987 ledger: self.activity.clone(),
4988 forwarding: self.forwarding.clone(),
4989 launches: self.launch_table.clone(),
4990 stamp,
4991 phone_fingerprint,
4992 route,
4993 connections: self.operator_apps.clone(),
4994 #[cfg(test)]
4995 hooks: Default::default(),
4996 }
4997 }
4998
4999 #[cfg(any(test, feature = "test-support"))]
5001 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
5002 self.operator_app_policy.phone_enabled = enabled;
5003 self
5004 }
5005
5006 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
5009 let mut scopes = ScopeRecords::from(scopes);
5010 for (index, record) in scopes.records.iter_mut().enumerate() {
5011 if let Some(refusal) = Self::canonicalize_workdir(record) {
5012 scopes.refusals.insert(index, refusal);
5013 }
5014 }
5015 scopes
5016 }
5017
5018 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
5019 let workdir = record.workdir.as_mut()?;
5020 let path = std::path::Path::new(workdir);
5021 if !path.is_absolute() {
5022 return Some((
5023 error_codes::INVALID_WORKDIR,
5024 "workdir must be absolute".to_string(),
5025 ));
5026 }
5027 match ProjectRootId::from_path_allowing_missing(path) {
5028 Ok(root) => match root.as_path().to_str() {
5029 Some(canonical) => {
5030 *workdir = canonical.to_string();
5031 None
5032 }
5033 None => Some((
5034 error_codes::INVALID_WORKDIR,
5035 "canonical workdir is not UTF-8".to_string(),
5036 )),
5037 },
5038 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
5039 }
5040 }
5041
5042 fn handle_scope_sync(
5046 &self,
5047 connection_id: ConnectionId,
5048 frame: Frame,
5049 generation: u64,
5050 scopes: Vec<ScopeRecord>,
5051 ) -> Result<Vec<Frame>, RouterError> {
5052 self.handle_scope_change(connection_id, frame, generation, scopes, None)
5053 }
5054
5055 fn handle_scope_change(
5056 &self,
5057 connection_id: ConnectionId,
5058 frame: Frame,
5059 generation: u64,
5060 scopes: Vec<ScopeRecord>,
5061 end: Option<Vec<ScopeEnd>>,
5062 ) -> Result<Vec<Frame>, RouterError> {
5063 let is_apply = end.is_some();
5064 let Some(registration) = self
5065 .registry
5066 .get_module_by_connection(connection_id)
5067 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5068 else {
5069 return Ok(vec![control_error_frame(
5070 &frame,
5071 "not_registered",
5072 "scope.sync and scope.apply require an active module registration owned by this connection",
5073 )?]);
5074 };
5075 let owner = registration.manifest.module_id;
5076 let scopes = Self::prepare_scope_records(scopes);
5077 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
5078 let is_current_launch = |connection: ConnectionId| {
5079 self.connection_is_current_launch(&owner, connection, current_nonce.as_deref())
5080 };
5081 let mut table = self
5086 .scopes
5087 .write()
5088 .unwrap_or_else(|poisoned| poisoned.into_inner());
5089 let outcome = match end {
5090 Some(end) => table.apply(
5091 &owner,
5092 connection_id,
5093 is_current_launch,
5094 generation,
5095 scopes,
5096 end,
5097 ),
5098 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
5099 };
5100 let drained = match &outcome {
5101 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
5102 Err(_) => Vec::new(),
5103 };
5104 let has_deadlines = outcome.is_ok() && table.has_deadlines();
5105 drop(table);
5106 if has_deadlines {
5107 self.scope_deadline_added.notify_one();
5108 }
5109 match outcome {
5110 Ok(applied) => {
5111 let counts = ScopeOutcomeCounts::of(&applied.results);
5112 info!(
5113 owner = %owner,
5114 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5115 generation,
5116 records = applied.results.len(),
5117 created = counts.created,
5118 replaced = counts.replaced,
5119 updated = counts.updated,
5120 unchanged = counts.unchanged,
5121 refused = counts.refused,
5122 ended = applied.ended.len(),
5123 tag_changes = applied.tag_changes.len(),
5124 routes_closed = drained.len(),
5125 "scope change accepted"
5126 );
5127 for refused in applied
5133 .results
5134 .iter()
5135 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
5136 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
5137 {
5138 warn!(
5139 owner = %owner,
5140 generation,
5141 scope_ref = %refused.scope_ref,
5142 scope_epoch = refused.scope_epoch,
5143 code = refused.code.as_deref().unwrap_or(""),
5144 "scope record refused"
5145 );
5146 }
5147 self.close_scope_drained_routes(drained);
5148 let response = if is_apply {
5149 ModuleControlResponseToModule::ScopeApply {
5150 generation,
5151 results: applied.results,
5152 end_results: applied.end_results,
5153 ended: applied.ended,
5154 }
5155 } else {
5156 ModuleControlResponseToModule::ScopeSync {
5157 generation,
5158 results: applied.results,
5159 ended: applied.ended,
5160 }
5161 };
5162 Ok(vec![control_response_body_frame(
5163 &frame,
5164 &response,
5165 "scope change response",
5166 )?])
5167 }
5168 Err(refusal) => {
5169 info!(
5170 owner = %owner,
5171 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5172 generation,
5173 code = refusal.code,
5174 "scope change refused"
5175 );
5176 Ok(vec![control_error_frame(
5177 &frame,
5178 refusal.code,
5179 refusal.message,
5180 )?])
5181 }
5182 }
5183 }
5184
5185 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
5187 scope_expiry::ScopeExpiryContext::new(self).close_scope_drained_routes(drained);
5188 }
5189
5190 fn handle_route_authorize(
5196 &self,
5197 connection_id: ConnectionId,
5198 frame: Frame,
5199 principal: Principal,
5200 scope: subc_protocol::scope::RouteAuthorizeScope,
5201 target: String,
5202 expected: subc_protocol::scope::RouteAuthorizeExpected,
5203 ) -> Result<Vec<Frame>, RouterError> {
5204 use crate::scopes::admission::{admit_route, AdmissionMode};
5205 use subc_protocol::scope::{
5206 RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
5207 };
5208 let registry = self
5209 .registry
5210 .route_admission_snapshot()
5211 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5212 let owner_configured = match &scope.owner {
5213 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5214 _ => false,
5215 };
5216 let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
5217 let answer = {
5218 let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
5219 let permitted = registry
5220 .iter()
5221 .find(|r| r.connection_id == connection_id)
5222 .is_some_and(|r| {
5223 scopes.is_federated_owner(&Principal::Reserved {
5224 module_id: r.manifest.module_id.clone(),
5225 }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
5226 caps.provides
5227 .iter()
5228 .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
5229 })
5230 });
5231 if !permitted {
5232 drop(scopes);
5233 return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
5234 "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
5235 }
5236 admit_route(
5237 &scopes,
5238 ®istry,
5239 &principal,
5240 &target,
5241 Some(&selector),
5242 owner_configured,
5243 AdmissionMode::Authorize(&expected),
5244 )
5245 };
5246 let response = match answer {
5247 Ok(admission) => {
5248 let mut response = RouteAuthorizeResponse::new(
5249 RouteAuthorizeOutcome::Authorized,
5250 "route admission authorized",
5251 );
5252 response.content_version = admission.content_version;
5253 response
5254 }
5255 Err(refusal) => refusal.response(),
5256 };
5257 Ok(vec![control_response_body_frame(
5258 &frame,
5259 &response,
5260 "RouteAuthorizeResponse",
5261 )?])
5262 }
5263
5264 fn handle_scope_describe(
5267 &self,
5268 connection_id: ConnectionId,
5269 frame: Frame,
5270 owner: Principal,
5271 scope_ref: String,
5272 ) -> Result<Vec<Frame>, RouterError> {
5273 let registered = self
5274 .registry
5275 .get_module_by_connection(connection_id)
5276 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5277 let Some(registered) = registered else {
5278 return Ok(vec![control_error_frame(
5279 &frame,
5280 "not_registered",
5281 "scope.describe requires an active module registration owned by this connection",
5282 )?]);
5283 };
5284 let scopes = self
5285 .scopes
5286 .read()
5287 .unwrap_or_else(|poisoned| poisoned.into_inner());
5288 if !scopes.reader_permitted(&owner, registered.manifest.capabilities.as_ref()) {
5289 return Ok(vec![control_error_frame(
5290 &frame,
5291 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
5292 "scope.describe of the federated owner requires scope-federated/v1",
5293 )?]);
5294 }
5295 let mut description = scopes.describe(&owner, &scope_ref);
5296 drop(scopes);
5297 if owner
5298 != (Principal::Reserved {
5299 module_id: registered.manifest.module_id.clone(),
5300 })
5301 {
5302 if let Some(facts) = description
5303 .stamp
5304 .as_mut()
5305 .and_then(|stamp| stamp.attributes.federated.as_mut())
5306 {
5307 facts.stamp_required = false;
5308 }
5309 }
5310 if !registered
5313 .manifest
5314 .capabilities
5315 .as_ref()
5316 .is_some_and(|caps| {
5317 caps.provides
5318 .iter()
5319 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5320 })
5321 {
5322 if let Some(stamp) = description.stamp.as_mut() {
5323 stamp.attributes.on_behalf_of_agent = None;
5324 }
5325 }
5326 if !registered
5327 .manifest
5328 .capabilities
5329 .as_ref()
5330 .is_some_and(|caps| {
5331 caps.provides
5332 .iter()
5333 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5334 })
5335 {
5336 if let Some(stamp) = description.stamp.as_mut() {
5337 stamp.attributes.flow_version = None;
5338 stamp.attributes.flow_code_hash = None;
5339 stamp.attributes.flow_install_id = None;
5340 }
5341 }
5342 let owner_configured = match &owner {
5343 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5348 _ => false,
5349 };
5350 let response = ModuleControlResponseToModule::ScopeDescribe {
5351 status: description.status,
5352 scope_epoch: description.scope_epoch,
5353 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
5354 owner_synced: description.owner_synced,
5355 owner_configured,
5356 scope: description.stamp,
5357 };
5358 Ok(vec![control_response_body_frame(
5359 &frame,
5360 &response,
5361 "ModuleControlResponseToModule::ScopeDescribe",
5362 )?])
5363 }
5364
5365 fn handle_catalog_update_role_declarations(
5366 &self,
5367 connection_id: ConnectionId,
5368 frame: Frame,
5369 role_declarations: serde_json::Value,
5370 ) -> Result<Vec<Frame>, RouterError> {
5371 if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
5372 return Ok(vec![control_error_frame(
5373 &frame,
5374 "invalid_role_declarations",
5375 err.to_string(),
5376 )?]);
5377 }
5378 let declarations = serde_json::from_value(role_declarations)
5379 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5380 if self
5381 .registry
5382 .replace_role_declarations_for_connection(connection_id, declarations)
5383 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5384 .is_none()
5385 {
5386 return Ok(vec![control_error_frame(
5387 &frame,
5388 "not_registered",
5389 "catalog.update_role_declarations requires an active module registration owned by this connection",
5390 )?]);
5391 }
5392 control_response_body_frame(
5393 &frame,
5394 &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
5395 "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
5396 )
5397 .map(|frame| vec![frame])
5398 }
5399
5400 fn handle_catalog_update(
5401 &self,
5402 connection_id: ConnectionId,
5403 frame: Frame,
5404 provides: Vec<ProviderRole>,
5405 capabilities: Option<CapabilityDeclarations>,
5406 ready: Option<bool>,
5407 ) -> Result<Vec<Frame>, RouterError> {
5408 self.refresh_capability_requirements();
5409 let Some(registration) = self
5410 .registry
5411 .get_module_by_connection(connection_id)
5412 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5413 else {
5414 return Ok(vec![control_error_frame(
5415 &frame,
5416 "not_registered",
5417 "catalog.update requires an active module registration owned by this connection",
5418 )?]);
5419 };
5420
5421 if let Some(message) =
5422 catalog_update_frozen_field_message(®istration.manifest, &provides)
5423 {
5424 return Ok(vec![control_error_frame(
5425 &frame,
5426 "catalog_update_frozen_field",
5427 message,
5428 )?]);
5429 }
5430
5431 let mut candidate = registration.manifest.clone();
5432 candidate.provides = provides.clone();
5433 candidate.capabilities = capabilities
5434 .clone()
5435 .or_else(|| registration.manifest.capabilities.clone());
5436 if let Some(rich_ui) = &candidate.rich_ui {
5437 if let Err(err) = rich_ui.validate() {
5438 return Ok(vec![control_error_frame(
5439 &frame,
5440 "invalid_manifest",
5441 err.to_string(),
5442 )?]);
5443 }
5444 }
5445 if let Err(err) = candidate.validate_capability_grammar() {
5446 return Ok(vec![control_error_frame(
5447 &frame,
5448 "invalid_capability_grammar",
5449 err.to_string(),
5450 )?]);
5451 }
5452
5453 let mut conflicts = self
5456 .capability_evaluator
5457 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
5458 if let Some(conflict) = conflicts.first() {
5459 let message = format!(
5460 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
5461 conflict.capability, conflict.claimants[0], candidate.module_id
5462 );
5463 for conflict in &mut conflicts {
5464 conflict.source = DuplicateClaimSource::CatalogUpdate;
5465 }
5466 log_duplicate_claim_events(conflicts);
5467 return Ok(vec![control_error_frame(
5468 &frame,
5469 "reserved_capability",
5470 message,
5471 )?]);
5472 }
5473
5474 let updated = match self.registry.replace_catalog_for_connection(
5475 connection_id,
5476 provides,
5477 capabilities,
5478 ready,
5479 ) {
5480 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => {
5481 return Ok(vec![control_error_frame(
5482 &frame,
5483 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5484 "scope-federated/v1 is fixed at HELLO; restart the module to change it",
5485 )?])
5486 }
5487 Err(crate::registry::RegistryError::CallerAttestationCapabilityChanged) => {
5488 return Ok(vec![control_error_frame(
5489 &frame,
5490 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5491 "caller-attestation/v1 is fixed at HELLO; restart the module to change it",
5492 )?])
5493 }
5494 result => {
5495 result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5496 }
5497 };
5498 if updated.is_none() {
5499 return Ok(vec![control_error_frame(
5500 &frame,
5501 "not_registered",
5502 "catalog.update requires an active module registration owned by this connection",
5503 )?]);
5504 }
5505 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
5506 log_duplicate_claim_events(
5507 self.capability_evaluator
5508 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
5509 );
5510 }
5511 if capability_census_trigger(
5512 registration.manifest.capabilities.as_ref(),
5513 updated
5514 .as_ref()
5515 .and_then(|entry| entry.manifest.capabilities.as_ref()),
5516 ) {
5517 self.enforce_capability_denies();
5518 }
5519 self.refresh_capability_requirements();
5520
5521 let response = ModuleControlResponseToModule::CatalogUpdate {};
5522 control_response_body_frame(
5523 &frame,
5524 &response,
5525 "ModuleControlResponseToModule::CatalogUpdate",
5526 )
5527 .map(|frame| vec![frame])
5528 }
5529
5530 pub(crate) fn with_activity_ledger_config(
5531 mut self,
5532 run_dir: &Path,
5533 incarnation: String,
5534 config: crate::activity::ActivityConfig,
5535 ) -> Self {
5536 self.activity = Some(crate::activity::ActivityLedger::open_config(
5537 run_dir.join("activity.jsonl"),
5538 incarnation,
5539 config,
5540 ));
5541 self.supervisor.set_activity(self.activity.clone());
5542 self
5543 }
5544
5545 pub(crate) async fn configure_activity(
5546 &self,
5547 config: crate::activity::ActivityConfig,
5548 ) -> Result<(), &'static str> {
5549 match &self.activity {
5550 Some(ledger) => ledger.configure(config).await,
5551 None => Ok(()),
5552 }
5553 }
5554
5555 fn cancel_activity_subscription(
5556 &self,
5557 ctx: &RouteCtx,
5558 frame: &Frame,
5559 ) -> Result<bool, RouterError> {
5560 let task = self
5561 .activity_streams
5562 .lock()
5563 .unwrap_or_else(|p| p.into_inner())
5564 .remove(&(ctx.connection_id, frame.header.corr));
5565 let Some(task) = task else {
5566 return Ok(false);
5567 };
5568 task.abort();
5569 let end = Frame::build_with_version(
5570 response_version(frame),
5571 FrameType::StreamEnd,
5572 control_flags(),
5573 0,
5574 0,
5575 frame.header.corr,
5576 Vec::new(),
5577 )
5578 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
5579 let sink = ctx.egress.clone();
5580 tokio::spawn(async move {
5581 let _ = sink.send(end).await;
5582 });
5583 Ok(true)
5584 }
5585
5586 async fn handle_activity_read(
5587 &self,
5588 ctx: &RouteCtx,
5589 frame: Frame,
5590 ) -> Result<Vec<Frame>, RouterError> {
5591 #[cfg(test)]
5592 {
5593 let pause = self
5594 .activity_request_pause
5595 .lock()
5596 .unwrap()
5597 .clone()
5598 .filter(|pause| pause.connection == ctx.connection_id);
5599 if let Some(pause) = pause {
5600 let _cancelled = ActivityRequestCancellation(pause.clone());
5601 pause.started.notify_one();
5602 std::future::pending::<()>().await;
5603 }
5604 }
5605 use subc_control::{
5606 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
5607 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
5608 };
5609 #[derive(Deserialize)]
5610 struct Identity {
5611 #[serde(default)]
5612 consumer_identity: Option<ConsumerIdentity>,
5613 }
5614 let mut bound_spawn = None;
5618 let legacy = match self.activity_reader(ctx.connection_id) {
5619 channel_operations::ActivityReader::Module { module_id, spawn } => {
5620 bound_spawn = Some(spawn);
5621 Some(Ok(Principal::Reserved { module_id }))
5622 }
5623 channel_operations::ActivityReader::Refused => {
5625 return Ok(vec![control_error_frame(
5626 &frame,
5627 ACTIVITY_READ_FORBIDDEN,
5628 "helper, relay and retired connections cannot read activity",
5629 )?]);
5630 }
5631 channel_operations::ActivityReader::Legacy => None,
5632 };
5633 let principal = match legacy {
5637 Some(principal) => principal,
5638 None => match self
5639 .forwarding
5640 .module_id_for_connection(ctx.connection_id)
5641 .map_err(RouterError::Forwarding)?
5642 {
5643 Some(module_id) => Ok(Principal::Reserved { module_id }),
5644 None => {
5645 let identity: Identity = match serde_json::from_slice(&frame.body) {
5646 Ok(identity) => identity,
5647 Err(_) => {
5648 return Ok(vec![control_error_frame(
5649 &frame,
5650 ACTIVITY_MALFORMED,
5651 "malformed activity read",
5652 )?])
5653 }
5654 };
5655 self.route_open_principal(
5656 ctx.connection_id,
5657 &frame,
5658 identity.consumer_identity,
5659 )?
5660 }
5661 },
5662 };
5663 let admitted = matches!(principal, Ok(Principal::Direct))
5664 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
5665 if !admitted {
5666 return Ok(vec![control_error_frame(
5667 &frame,
5668 ACTIVITY_READ_FORBIDDEN,
5669 "activity reads require an operator or trusted mirror principal",
5670 )?]);
5671 }
5672 let Some(ledger) = &self.activity else {
5673 return Ok(vec![control_error_frame(
5674 &frame,
5675 ACTIVITY_UNAVAILABLE,
5676 "activity ledger unavailable",
5677 )?]);
5678 };
5679 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
5680 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
5681 {
5682 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
5683 Ok(request) => ledger.query(request).await,
5684 Err(_) => Err(ACTIVITY_MALFORMED),
5685 };
5686 return match result {
5687 Ok(reply) => Ok(vec![control_response_body_frame(
5688 &frame,
5689 &reply,
5690 ops::ACTIVITY_QUERY,
5691 )?]),
5692 Err(code) => Ok(vec![control_error_frame(
5693 &frame,
5694 code,
5695 "activity query refused",
5696 )?]),
5697 };
5698 }
5699 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
5700 Ok(request) => request,
5701 Err(_) => {
5702 return Ok(vec![control_error_frame(
5703 &frame,
5704 ACTIVITY_MALFORMED,
5705 "malformed activity subscription",
5706 )?])
5707 }
5708 };
5709 let mut subscription = match ledger.subscribe(request.since).await {
5710 Ok(subscription) => subscription,
5711 Err(code) => {
5712 return Ok(vec![control_error_frame(
5713 &frame,
5714 code,
5715 "activity subscription refused",
5716 )?])
5717 }
5718 };
5719 let key = (ctx.connection_id, frame.header.corr);
5720 let sink = ctx.egress.clone();
5721 let mut streams = self
5722 .activity_streams
5723 .lock()
5724 .unwrap_or_else(|p| p.into_inner());
5725 streams.retain(|_, task| !task.is_finished());
5726 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
5727 return Ok(vec![control_error_frame(
5728 &frame,
5729 "duplicate_subscription",
5730 "activity subscription correlation id already in use",
5731 )?]);
5732 }
5733 let stream = async move {
5734 loop {
5735 let entry = if let Some(entry) = subscription.replay.pop_front() {
5736 Some(entry)
5737 } else {
5738 subscription.receiver.recv().await
5739 };
5740 let Some(entry) = entry else {
5741 break;
5742 };
5743 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
5744 return;
5745 };
5746 let Ok(next) = Frame::build_with_version(
5747 response_version(&frame),
5748 FrameType::StreamData,
5749 control_flags(),
5750 0,
5751 0,
5752 frame.header.corr,
5753 body,
5754 ) else {
5755 return;
5756 };
5757 if sink.send(next).await.is_err() {
5758 return;
5759 }
5760 }
5761 if let Some(code) = subscription.terminal_code() {
5764 if let Ok(end) =
5765 control_error_frame(&frame, code, "activity subscriber fell behind")
5766 {
5767 let _ = sink.send(end).await;
5768 }
5769 }
5770 };
5771 let task = tokio::spawn(async move {
5775 match bound_spawn {
5776 Some(spawn) => {
5777 tokio::select! {
5778 () = stream => {}
5779 () = spawn.wait_closed() => {}
5780 }
5781 }
5782 None => stream.await,
5783 }
5784 });
5785 streams.insert(key, task.abort_handle());
5786 Ok(Vec::new())
5787 }
5788
5789 async fn handle_activity_record(
5790 &self,
5791 ctx: &RouteCtx,
5792 frame: Frame,
5793 ) -> Result<Vec<Frame>, RouterError> {
5794 use subc_control::{
5795 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
5796 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
5797 };
5798 let result = async {
5799 let module = self
5800 .forwarding
5801 .module_id_for_connection(ctx.connection_id)
5802 .ok()
5803 .flatten()
5804 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
5805 let request: ActivityRecordRequest =
5806 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
5807 crate::activity::validate(&module, &request)?;
5808 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
5809 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
5810 let at_ms = std::time::SystemTime::now()
5811 .duration_since(std::time::UNIX_EPOCH)
5812 .unwrap_or_default()
5813 .as_millis() as u64;
5814 ledger.record(module, request, actor, at_ms).await
5815 }
5816 .await;
5817 match result {
5818 Ok(entry) => Ok(vec![control_response_body_frame(
5819 &frame,
5820 &ActivityRecordResponse { seq: entry.seq },
5821 "activity.record",
5822 )?]),
5823 Err(code) => Ok(vec![control_error_frame(
5824 &frame,
5825 code,
5826 "activity recording refused",
5827 )?]),
5828 }
5829 }
5830
5831 fn activity_actor(
5832 &self,
5833 connection: ConnectionId,
5834 module: &str,
5835 on: Option<&subc_control::ActivityOn>,
5836 ) -> Result<subc_control::ActivityActor, &'static str> {
5837 use crate::forwarding::{DataRoute, DataRouteState};
5838 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
5839 let principal = |id: &str| Principal::Reserved {
5840 module_id: id.to_owned(),
5841 };
5842 let mut actor = ActivityActor {
5843 principal: principal(module),
5844 scope_owner: None,
5845 agent_id: None,
5846 run_id: None,
5847 flow_id: None,
5848 project_root: None,
5849 workdir: None,
5850 project_id: None,
5851 };
5852 let scope = match on {
5853 None => return Ok(actor),
5854 Some(ActivityOn::Scope(scope)) => {
5855 let selector = subc_protocol::scope::ScopeSelector::new(
5856 principal(module),
5857 scope.scope_ref.clone(),
5858 Some(scope.epoch),
5859 );
5860 self.scopes
5861 .read()
5862 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5863 .attribute(&principal(module), module, &selector, true)
5864 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5865 }
5866 Some(ActivityOn::Route(route)) => {
5867 let DataRoute::Module(DataRouteState::Bound(binding)) = self
5868 .forwarding
5869 .lookup_data_route(connection, route.channel, route.epoch)
5870 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5871 else {
5872 return Err(ACTIVITY_ACTOR_UNKNOWN);
5873 };
5874 if binding.module_id != module {
5875 return Err(ACTIVITY_ACTOR_UNKNOWN);
5876 }
5877 actor.principal = binding.principal.clone();
5878 actor.project_root = binding
5879 .project_root
5880 .as_ref()
5881 .map(|root| root.as_path().to_path_buf());
5882 let Some(bound) = &binding.scope else {
5883 return Ok(actor);
5884 };
5885 let owner = principal(&bound.owner);
5886 let selector = subc_protocol::scope::ScopeSelector::new(
5887 owner.clone(),
5888 bound.scope_ref.clone(),
5889 Some(bound.tag.scope_epoch),
5890 );
5891 let admitted = self
5892 .scopes
5893 .read()
5894 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5895 .attribute(&owner, module, &selector, true)
5896 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
5897 if admitted.tag != bound.tag {
5898 return Err(ACTIVITY_ACTOR_UNKNOWN);
5899 }
5900 admitted
5901 }
5902 };
5903 actor.scope_owner = Some(scope.stamp.owner);
5904 actor.agent_id = scope.stamp.attributes.agent_id;
5905 actor.run_id = scope.stamp.attributes.run_id;
5906 actor.flow_id = scope.stamp.attributes.flow_id;
5907 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
5908 actor.project_id = scope.stamp.project_id;
5909 Ok(actor)
5910 }
5911
5912 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5913 self.refresh_capability_requirements();
5914 let mut counters = self.counters.snapshot();
5920 let losses = self.resource_tracker.loss_diagnostics();
5921 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
5922 if let Some(obj) = counters.as_object_mut() {
5923 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
5924 }
5925 }
5926 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
5927 obj.insert(
5928 "activity_unparseable_lines".into(),
5929 ledger.read_errors().into(),
5930 );
5931 obj.insert(
5932 "activity_lifecycle_append_failures".into(),
5933 ledger.lifecycle_failures().into(),
5934 );
5935 obj.insert(
5936 "activity_started_digest_skips".into(),
5937 ledger.lifecycle_digest_skips().into(),
5938 );
5939 }
5940 if let (Ok((connections_with_routes, max)), Some(obj)) = (
5941 self.forwarding.client_route_concentration(),
5942 counters.as_object_mut(),
5943 ) {
5944 obj.insert(
5945 "client_connections_with_routes".into(),
5946 connections_with_routes.into(),
5947 );
5948 obj.insert("max_routes_on_one_connection".into(), max.into());
5949 }
5950 if let (Some(open_breakers), Some(obj)) = (
5956 self.route_bind_breakers.open_snapshot(),
5957 counters.as_object_mut(),
5958 ) {
5959 obj.insert("route_bind_breakers_open".into(), open_breakers);
5960 }
5961 if let (Some(reason), Some(obj)) = (
5966 self.local_endpoint_unavailable.clone(),
5967 counters.as_object_mut(),
5968 ) {
5969 obj.insert("local_endpoint_unavailable".into(), reason.into());
5970 }
5971 let response = ClientControlResponse::ServerDescribe {
5972 protocol_ver: PROTOCOL_VERSION,
5973 subc_ops: subc_ops(),
5974 capabilities: self.describe_capabilities(),
5975 connected_clients: self.connected_clients.count(),
5976 counters: Some(counters),
5977 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
5978 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
5979 capability_requirements: self.capability_requirement_statuses(),
5980 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
5981 config_path: self
5982 .rescan
5983 .as_ref()
5984 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
5985 };
5986 Ok(vec![control_response_body_frame(
5987 &frame,
5988 &response,
5989 "ClientControlResponse::ServerDescribe",
5990 )?])
5991 }
5992
5993 fn handle_catalog_list(
5994 &self,
5995 frame: Frame,
5996 module_id: Option<String>,
5997 ) -> Result<Vec<Frame>, RouterError> {
5998 let (generation, modules) = self
5999 .registry
6000 .list_modules_with_nonce_verification()
6001 .map_err(|err| {
6002 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
6003 })?;
6004 let entries = modules
6005 .into_iter()
6006 .filter(|(registration, _)| {
6007 module_id
6008 .as_deref()
6009 .map(|wanted| registration.manifest.module_id == wanted)
6010 .unwrap_or(true)
6011 })
6012 .map(|(registration, nonce_verified)| {
6013 let entry = self
6014 .catalog_discovery_entry(®istration, module_id.is_some())
6015 .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
6016 let not_ready = self.not_ready_reason(®istration);
6017 let mut entry = entry;
6018 entry.roles = registration.manifest.provides;
6019 entry.control_ops = registration.control_ops;
6020 Ok(entry
6021 .with_ready(not_ready.is_none())
6022 .with_display_name(registration.manifest.display_name)
6023 .with_not_ready(not_ready)
6024 .with_module_version(Some(registration.manifest.module_version))
6025 .with_provenance(registration.manifest.provenance)
6026 .with_capabilities(registration.manifest.capabilities)
6027 .with_self_signals(registration.manifest.self_signals)
6028 .with_events(registration.manifest.events)
6029 .with_operation_delivery(registration.manifest.operation_delivery)
6030 .with_operation_payload(registration.manifest.operation_payload)
6031 .with_flow_operations(registration.manifest.flow_operations)
6032 .with_operation_deferral(registration.manifest.operation_deferral)
6033 .with_confirm_operations(registration.manifest.confirm_operations)
6034 .with_rich_ui(registration.manifest.rich_ui)
6035 .with_role_declarations(registration.manifest.role_declarations)
6036 .with_nonce_verified(nonce_verified))
6037 })
6038 .collect::<Result<Vec<_>, RouterError>>()?;
6039 let response = ClientControlResponse::CatalogList {
6040 generation,
6041 modules: entries,
6042 subc_ops: subc_ops(),
6043 };
6044 Ok(vec![control_response_body_frame(
6045 &frame,
6046 &response,
6047 "ClientControlResponse::CatalogList",
6048 )?])
6049 }
6050
6051 fn route_open_caller_from_frame(&self, ctx: &RouteCtx, frame: &Frame) -> RouteOpenCaller {
6055 let Ok(ClientControlRequest::RouteOpen {
6056 identity,
6057 consumer_identity,
6058 scope,
6059 ..
6060 }) = parse_client_control_request(&frame.body)
6061 else {
6062 return RouteOpenCaller::unknown();
6063 };
6064 let principal = self
6065 .attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref())
6066 .map(|attested| attested.principal);
6067 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
6068 }
6069
6070 fn route_open_principal(
6071 &self,
6072 connection_id: ConnectionId,
6073 frame: &Frame,
6074 consumer_identity: Option<ConsumerIdentity>,
6075 ) -> Result<Result<Principal, Frame>, RouterError> {
6076 let attested = self
6077 .attest_route_open_consumer(connection_id, consumer_identity.as_ref())
6078 .map(|attested| attested.principal);
6079 self.route_open_principal_attested(frame, consumer_identity, attested)
6080 }
6081
6082 fn route_open_principal_attested(
6086 &self,
6087 frame: &Frame,
6088 consumer_identity: Option<ConsumerIdentity>,
6089 attested: Option<Principal>,
6090 ) -> Result<Result<Principal, Frame>, RouterError> {
6091 if let Some(principal) = attested {
6092 return Ok(Ok(principal));
6093 }
6094 Ok(Err(control_error_frame(
6095 frame,
6096 "bad_consumer_identity",
6097 consumer_identity.map_or_else(
6098 || "connection claim is no longer authorized for routes".to_string(),
6099 |identity| format!(
6100 "consumer_identity for module_id '{}' conflicts with its connection claim or did not match a supervised launch nonce",
6101 identity.module_id
6102 ),
6103 ),
6104 )?))
6105 }
6106
6107 #[allow(clippy::too_many_arguments)]
6114 fn route_open_refusal_frame(
6115 &self,
6116 ctx: &RouteCtx,
6117 caller: &RouteOpenCaller,
6118 frame: &Frame,
6119 module_id: &str,
6120 reason: &'static str,
6121 code: &'static str,
6122 message: impl Into<String>,
6123 ) -> Result<Frame, RouterError> {
6124 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
6125 let detail = if matches!(
6126 reason,
6127 "registration_not_active"
6128 | "supervisor_not_live"
6129 | "no_forwarding_connection"
6130 | "reloading"
6131 ) {
6132 let expected_back = (reason == "reloading").then_some(true);
6135 self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
6136 } else {
6137 None
6138 };
6139 control_error_body_frame(
6140 frame,
6141 ErrorBody {
6142 code: code.to_string(),
6143 message: message.into(),
6144 detail,
6145 },
6146 )
6147 }
6148
6149 fn route_open_recovery_detail(
6150 &self,
6151 module_id: &str,
6152 corr: u64,
6153 detail: Option<serde_json::Value>,
6154 expected_back: Option<bool>,
6155 ) -> Result<Option<serde_json::Value>, RouterError> {
6156 let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
6157 return Ok(detail);
6158 };
6159 let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
6160 detail["state"] = serde_json::json!(status.state.to_string());
6161 if let Some(expected_back) = expected_back {
6162 detail["expected_back"] = serde_json::json!(expected_back);
6163 } else if let Some(module) = self.supervisor.get(module_id) {
6164 match module.will_recover_after_connection_loss() {
6165 Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
6166 Err(err) => warn!(
6167 %module_id,
6168 error = %err,
6169 "failed to read route.open recovery verdict; omitting expected_back"
6170 ),
6171 }
6172 }
6173 Ok(Some(detail))
6174 }
6175
6176 #[allow(clippy::too_many_arguments)]
6185 fn route_bind_rejection(
6186 &self,
6187 ctx: &RouteCtx,
6188 caller: &RouteOpenCaller,
6189 frame: &Frame,
6190 target_module_id: &str,
6191 body: ErrorBody,
6192 daemon_authored: bool,
6193 ) -> Result<Vec<Frame>, RouterError> {
6194 if daemon_authored {
6200 let scope_code = match body.code.as_str() {
6201 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
6202 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
6203 _ => None,
6204 };
6205 if let Some(code) = scope_code {
6206 self.observe_route_open_refusal(
6207 ctx,
6208 caller,
6209 target_module_id,
6210 "scope_changed_before_commit",
6211 code,
6212 );
6213 return Ok(vec![control_error_body_frame(frame, body)?]);
6214 }
6215 }
6216 self.counters
6217 .increment_route_open_refused("module_rejected");
6218 info!(
6219 target: "control",
6220 code = "module_rejected",
6221 module_code = ?body.code,
6222 module_id = ?target_module_id,
6223 connection_id = ctx.connection_id.get(),
6224 principal = %caller.principal,
6225 scope = %caller.scope,
6226 harness = %caller.harness,
6227 "route.open refused"
6228 );
6229 let refusal = if daemon_authored {
6230 control_error_body_frame(frame, body)?
6231 } else {
6232 control_relayed_error_body_frame(frame, body)?
6233 };
6234 Ok(vec![refusal])
6235 }
6236
6237 #[allow(clippy::too_many_arguments)]
6255 fn route_open_breaker_refusal_frame(
6256 &self,
6257 ctx: &RouteCtx,
6258 caller: &RouteOpenCaller,
6259 frame: &Frame,
6260 module_id: &str,
6261 consecutive_timeouts: u32,
6262 retry_in: Duration,
6263 probe_in_flight: bool,
6264 ) -> Result<Frame, RouterError> {
6265 self.counters
6266 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
6267 debug!(
6268 target: "control",
6269 code = "module_timeout",
6270 module_id = ?module_id,
6271 connection_id = ctx.connection_id.get(),
6272 principal = %caller.principal,
6273 scope = %caller.scope,
6274 harness = %caller.harness,
6275 consecutive_timeouts,
6276 retry_in_ms = retry_in.as_millis() as u64,
6277 probe_in_flight,
6278 "route.open refused by open bind-relay breaker"
6279 );
6280 let detail = if probe_in_flight {
6287 "one test route is already being tried; retry once it settles".to_string()
6288 } else {
6289 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
6290 };
6291 control_error_frame(
6292 frame,
6293 "module_timeout",
6294 format!(
6295 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
6296 timed out in a row); {detail}; its established routes are unaffected"
6297 ),
6298 )
6299 }
6300
6301 fn observe_route_open_refusal(
6314 &self,
6315 ctx: &RouteCtx,
6316 caller: &RouteOpenCaller,
6317 module_id: &str,
6318 reason: &'static str,
6319 code: &'static str,
6320 ) {
6321 self.counters.increment_route_open_refused(code);
6322 info!(
6323 target: "control",
6324 code,
6325 reason,
6326 module_id = ?module_id,
6327 connection_id = ctx.connection_id.get(),
6328 principal = %caller.principal,
6329 scope = %caller.scope,
6330 harness = %caller.harness,
6331 "route.open refused"
6332 );
6333 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
6334 self.route_outages.record_not_serving(module_id, reason);
6335 }
6336 }
6337
6338 fn observe_route_open_accept(
6383 &self,
6384 ctx: &RouteCtx,
6385 module_id: &str,
6386 principal: &str,
6387 scope: &str,
6388 ) {
6389 self.route_outages.record_accepted(module_id);
6390 self.counters.increment_route_open_accepted(principal);
6391 info!(
6392 target: "control",
6393 principal,
6394 module_id,
6395 connection_id = ctx.connection_id.get(),
6396 scope = %scope,
6397 "route.open accepted"
6398 );
6399 }
6400
6401 fn supervised_absent_route_open_refusal_frame(
6402 &self,
6403 ctx: &RouteCtx,
6404 caller: &RouteOpenCaller,
6405 frame: &Frame,
6406 module_id: &str,
6407 code: &'static str,
6408 status: &crate::supervise::ModuleStatus,
6409 ) -> Result<Frame, RouterError> {
6410 self.counters.increment_route_open_refused(code);
6411 info!(
6412 target: "control",
6413 code,
6414 reason = "supervised_not_registered",
6415 module_id = ?module_id,
6416 connection_id = ctx.connection_id.get(),
6417 principal = %caller.principal,
6418 scope = %caller.scope,
6419 harness = %caller.harness,
6420 state = %status.state,
6421 enabled = status.enabled,
6422 live = status.live,
6423 "route.open refused"
6424 );
6425 self.route_outages
6429 .record_not_serving(module_id, "supervised_not_registered");
6430 let message = if let Some(pid) = status.waiting_for_previous_pid {
6431 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
6432 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
6433 })
6434 } else {
6435 format!(
6436 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
6437 status.state, status.enabled, status.live
6438 )
6439 };
6440 let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
6444 control_error_body_frame(
6445 frame,
6446 ErrorBody {
6447 code: code.to_string(),
6448 message,
6449 detail: self.route_open_recovery_detail(
6450 module_id,
6451 frame.header.corr,
6452 None,
6453 expected_back,
6454 )?,
6455 },
6456 )
6457 }
6458
6459 async fn handle_route_open(
6460 &self,
6461 ctx: &RouteCtx,
6462 frame: Frame,
6463 request: RouteOpenRequest,
6464 timings: &mut RouteOpenTimings,
6465 ) -> Result<Vec<Frame>, RouterError> {
6466 let RouteOpenRequest {
6467 target,
6468 mut identity,
6469 consumer_identity,
6470 consumer_capabilities,
6471 role_versions,
6472 busy_refusals,
6473 admission_facts,
6474 scope,
6475 accepts_close_reasons,
6476 } = request;
6477 let target_module_id = target_module_id(&target).to_string();
6478 timings.module_id.clone_from(&target_module_id);
6479 if self
6480 .registry
6481 .get_module_by_connection(ctx.connection_id)
6482 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6483 .is_some()
6484 {
6485 return Ok(vec![control_error_frame(
6486 &frame,
6487 "invalid_request",
6488 "module connections cannot open client routes",
6489 )?]);
6490 }
6491 debug!(
6492 connection_id = ctx.connection_id.get(),
6493 corr = frame.header.corr,
6494 module_id = %target_module_id,
6495 "handling route.open"
6496 );
6497
6498 let attested =
6504 self.attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref());
6505 let mut caller = RouteOpenCaller::new(
6506 attested.as_ref().map(|attested| &attested.principal),
6507 scope.as_ref(),
6508 &identity,
6509 );
6510
6511 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
6517 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
6518 self.observe_route_open_refusal(
6519 ctx,
6520 &caller,
6521 &target_module_id,
6522 "invalid_role_versions",
6523 error_codes::INVALID_REQUEST,
6524 );
6525 return Ok(vec![control_error_body_frame(
6526 &frame,
6527 ErrorBody {
6528 code: error_codes::INVALID_REQUEST.to_string(),
6529 message: error.to_string(),
6530 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
6531 },
6532 )?]);
6533 }
6534
6535 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
6536 if status.waiting_for_previous_pid.is_some() {
6537 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6538 ctx,
6539 &caller,
6540 &frame,
6541 &target_module_id,
6542 "module_warming",
6543 &status,
6544 )?]);
6545 }
6546 }
6547
6548 let admission_registry = self
6567 .registry
6568 .route_admission_snapshot()
6569 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6570 let Some(registration) =
6571 crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
6572 else {
6573 if let Some((status, warming)) =
6574 self.supervisor_status(&target_module_id, frame.header.corr)?
6575 {
6576 if status.protocol == ModuleProtocol::None {
6583 return Ok(vec![self.route_open_refusal_frame(
6584 ctx,
6585 &caller,
6586 &frame,
6587 &target_module_id,
6588 "protocol_none",
6589 error_codes::MODULE_NO_PROTOCOL,
6590 format!(
6591 "module_id '{target_module_id}' is declared protocol: none; \
6592 it speaks no subc wire and serves no routes"
6593 ),
6594 )?]);
6595 }
6596 let code = if warming {
6597 "module_warming"
6598 } else {
6599 "target_unavailable"
6600 };
6601 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6602 ctx,
6603 &caller,
6604 &frame,
6605 &target_module_id,
6606 code,
6607 &status,
6608 )?]);
6609 }
6610 if let Some(removed_ago_ms) =
6611 self.supervisor.removal_tombstone_age_ms(&target_module_id)
6612 {
6613 return Ok(vec![self.route_open_refusal_frame(
6614 ctx,
6615 &caller,
6616 &frame,
6617 &target_module_id,
6618 "removed",
6619 error_codes::MODULE_REMOVED,
6620 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
6621 )?]);
6622 }
6623 return Ok(vec![self.route_open_refusal_frame(
6624 ctx,
6625 &caller,
6626 &frame,
6627 &target_module_id,
6628 "not_registered",
6629 error_codes::UNKNOWN_MODULE,
6630 format!("module_id '{target_module_id}' is not registered"),
6631 )?]);
6632 };
6633
6634 if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
6638 .is_err()
6639 {
6640 self.counters
6641 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
6642 info!(
6643 target: "control",
6644 code = error_codes::MODULE_WARMING,
6645 module_id = ?target_module_id,
6646 connection_id = ctx.connection_id.get(),
6647 principal = %caller.principal,
6648 scope = %caller.scope,
6649 harness = %caller.harness,
6650 reason = "declared_not_ready",
6651 "route.open refused"
6652 );
6653 self.route_outages
6656 .record_not_serving(&target_module_id, "declared_not_ready");
6657 return Ok(vec![control_error_body_frame(
6660 &frame,
6661 ErrorBody {
6662 code: error_codes::MODULE_WARMING.to_string(),
6663 message: format!(
6664 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
6665 ),
6666 detail: self.route_open_recovery_detail(
6667 &target_module_id,
6668 frame.header.corr,
6669 Some(serde_json::json!({ "reason": "declared_not_ready" })),
6670 Some(true),
6671 )?,
6672 },
6673 )?]);
6674 }
6675
6676 if let Some(capability) = self.first_unprovided_required_capability(registration) {
6695 self.counters
6696 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
6697 info!(
6698 target: "control",
6699 code = error_codes::MODULE_WARMING,
6700 module_id = ?target_module_id,
6701 connection_id = ctx.connection_id.get(),
6702 principal = %caller.principal,
6703 scope = %caller.scope,
6704 harness = %caller.harness,
6705 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6706 capability = %capability,
6707 "route.open refused"
6708 );
6709 return Ok(vec![control_error_body_frame(
6710 &frame,
6711 ErrorBody {
6712 code: error_codes::MODULE_WARMING.to_string(),
6713 message: format!(
6714 "module_id '{target_module_id}' requires capability '{capability}', \
6715 which no registered module provides; retry"
6716 ),
6717 detail: Some(serde_json::json!({
6718 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6719 "capability": capability,
6720 })),
6721 },
6722 )?]);
6723 }
6724
6725 if !target_has_required_role(&target, ®istration.manifest.provides) {
6726 return Ok(vec![self.route_open_refusal_frame(
6727 ctx,
6728 &caller,
6729 &frame,
6730 &target_module_id,
6731 "role_not_provided",
6732 "target_unavailable",
6733 format!("module_id '{target_module_id}' does not provide the requested target"),
6734 )?]);
6735 }
6736
6737 if registration.state != ChannelState::Active {
6738 return Ok(vec![self.route_open_refusal_frame(
6739 ctx,
6740 &caller,
6741 &frame,
6742 &target_module_id,
6743 "registration_not_active",
6744 "target_unavailable",
6745 format!("module_id '{target_module_id}' is not active"),
6746 )?]);
6747 }
6748
6749 if self
6750 .forwarding
6751 .module_is_draining(&target_module_id)
6752 .map_err(RouterError::Forwarding)?
6753 {
6754 return Ok(vec![self.route_open_refusal_frame(
6755 ctx,
6756 &caller,
6757 &frame,
6758 &target_module_id,
6759 "reloading",
6760 "module_reloading",
6761 format!("module_id '{target_module_id}' is reloading"),
6762 )?]);
6763 }
6764
6765 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
6766 process_liveness.process_live(&target_module_id) == Some(false)
6767 }) {
6768 if process_liveness.process_replacing(&target_module_id) {
6775 return Ok(vec![self.route_open_refusal_frame(
6776 ctx,
6777 &caller,
6778 &frame,
6779 &target_module_id,
6780 "reloading",
6781 "module_reloading",
6782 format!("module_id '{target_module_id}' is reloading"),
6783 )?]);
6784 }
6785 return Ok(vec![self.route_open_refusal_frame(
6786 ctx,
6787 &caller,
6788 &frame,
6789 &target_module_id,
6790 "supervisor_not_live",
6791 "target_unavailable",
6792 format!("module_id '{target_module_id}' is not live"),
6793 )?]);
6794 }
6795
6796 if !self
6797 .forwarding
6798 .has_live_module_connection(&target_module_id)
6799 .map_err(RouterError::Forwarding)?
6800 {
6801 return Ok(vec![self.route_open_refusal_frame(
6802 ctx,
6803 &caller,
6804 &frame,
6805 &target_module_id,
6806 "no_forwarding_connection",
6807 "target_unavailable",
6808 format!("module_id '{target_module_id}' has no live forwarding connection"),
6809 )?]);
6810 }
6811
6812 if let Some(error) =
6813 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
6814 {
6815 self.observe_route_open_refusal(
6816 ctx,
6817 &caller,
6818 &target_module_id,
6819 "op_not_allowed",
6820 "op_not_allowed",
6821 );
6822 return Ok(vec![error]);
6823 }
6824
6825 let principal = match (&attested, consumer_identity.is_some()) {
6826 (None, false) => {
6829 self.observe_route_open_refusal(
6830 ctx,
6831 &caller,
6832 &target_module_id,
6833 "registration_channel_closed",
6834 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6835 );
6836 return Ok(vec![control_error_frame(
6837 &frame,
6838 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6839 "the claim binding's launch has ended",
6840 )?]);
6841 }
6842 _ => match self.route_open_principal_attested(
6843 &frame,
6844 consumer_identity,
6845 attested.as_ref().map(|attested| attested.principal.clone()),
6846 )? {
6847 Ok(principal) => principal,
6848 Err(error) => {
6849 self.observe_route_open_refusal(
6850 ctx,
6851 &caller,
6852 &target_module_id,
6853 "bad_consumer_identity",
6854 "bad_consumer_identity",
6855 );
6856 return Ok(vec![error]);
6857 }
6858 },
6859 };
6860 let attested = attested.expect("an attested principal was resolved above");
6861 timings.principal = principal_label(&principal);
6862
6863 let Ok(mut caller_capture) = attested.capture() else {
6866 self.observe_route_open_refusal(
6867 ctx,
6868 &caller,
6869 &target_module_id,
6870 "registration_channel_closed",
6871 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6872 );
6873 return Ok(vec![control_error_frame(
6874 &frame,
6875 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6876 "the opener's launch has ended",
6877 )?]);
6878 };
6879 let caller_module_id = match &attested.spawn {
6883 Some((record, _)) => Some(record.module_id().to_owned()),
6884 None => match &principal {
6885 Principal::Reserved { module_id } => Some(module_id.clone()),
6886 _ => None,
6887 },
6888 };
6889
6890 let build_class = identity.build_class.take().filter(|build_class| {
6893 if caller_module_id.as_deref() == Some("callosum") {
6894 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
6895 } else {
6896 if self
6897 .build_class_warned
6898 .lock()
6899 .unwrap_or_else(|p| p.into_inner())
6900 .insert(ctx.connection_id)
6901 {
6902 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
6903 "ignoring build_class from a non-Callosum opener");
6904 }
6905 false
6906 }
6907 });
6908
6909 if let Some(opening_registration) = self.caller_registration(&attested, &admission_registry)
6913 {
6914 let opening_module_id = opening_registration.manifest.module_id.as_str();
6915 if let Some(capability) =
6916 denied_capability(&opening_registration.manifest, ®istration.manifest)
6917 {
6918 warn!(
6919 opening_module_id,
6920 target_module_id,
6921 capability,
6922 "refusing route.open because an attested capability deny edge matches"
6923 );
6924 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6925 &frame,
6926 &target_module_id,
6927 "capability_deny_edge",
6928 "capability_forbidden",
6929 format!(
6930 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
6931 ),
6932 )?]);
6933 }
6934 }
6935
6936 if admission_facts.is_some() {
6937 let carrier_matches = caller_module_id.is_some()
6938 && self.admission_facts_carrier_module_id.as_deref() == caller_module_id.as_deref();
6939 if !carrier_matches {
6940 return Ok(vec![self.route_open_refusal_frame(
6941 ctx,
6942 &caller,
6943 &frame,
6944 &target_module_id,
6945 "admission_facts_carrier_not_permitted",
6946 "admission_facts_not_permitted",
6947 "admission facts may only be carried by the configured reserved module",
6948 )?]);
6949 }
6950
6951 let target_allowed = self
6952 .admission_facts_targets
6953 .as_ref()
6954 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
6955 if !target_allowed {
6956 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6957 &frame,
6958 &target_module_id,
6959 "admission_facts_target_not_listed",
6960 "admission_facts_target_not_allowed",
6961 format!(
6962 "admission facts are not permitted for target module_id '{target_module_id}'"
6963 ),
6964 )?]);
6965 }
6966
6967 }
6971
6972 let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
6978 None => (None, None, false),
6979 Some(selector) => {
6980 let owner_configured = match &selector.owner {
6981 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
6987 _ => false,
6988 };
6989 let mut scopes = self
6990 .scopes
6991 .write()
6992 .unwrap_or_else(|poisoned| poisoned.into_inner());
6993 let admitted = crate::scopes::admission::admit_route(
6994 &scopes,
6995 &admission_registry,
6996 &principal,
6997 &target_module_id,
6998 Some(selector),
6999 owner_configured,
7000 crate::scopes::admission::AdmissionMode::Open,
7001 )
7002 .map(|route| {
7003 let admission = route.scope.expect("scoped admission");
7004 let bound = scopes.reserve_bind(
7005 &admission,
7006 &principal,
7007 ctx.connection_id,
7008 selector.hold_presence,
7009 );
7010 (admission, bound)
7011 });
7012 match admitted {
7013 Ok((admission, bound)) => {
7014 (Some(bound), Some(admission.stamp), admission.stamp_required)
7015 }
7016 Err(refusal) => {
7017 return Ok(vec![self.route_open_refusal_frame(
7018 ctx,
7019 &caller,
7020 &frame,
7021 &target_module_id,
7022 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7023 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7024 refusal.message,
7025 )?]);
7026 }
7027 }
7028 }
7029 };
7030 if let Some(stamp) = &scope_stamp {
7033 caller.scope = route_open_scope_label(stamp);
7034 }
7035 if scope_stamp.as_ref().is_some_and(scope_acts_for_another) {
7038 caller_capture = caller_capture.map(crate::spawn_authority::CallerCapture::relayed);
7039 }
7040
7041 timings.enter(RouteOpenPhase::ProjectRoot);
7049 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
7050 Ok(project) => project,
7051 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
7052 };
7053 timings.enter(RouteOpenPhase::Admission);
7054 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
7055 Ok(root) => root,
7056 Err(err) => {
7057 return Ok(vec![control_error_frame(
7058 &frame,
7059 "invalid_project_root",
7060 err.to_string(),
7061 )?])
7062 }
7063 };
7064 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
7070
7071 let _concurrency_guard = match self
7082 .route_bind_concurrency
7083 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
7084 {
7085 Ok(guard) => guard,
7086 Err(in_flight) => {
7087 return Ok(vec![self.route_open_target_capacity_refusal(
7088 ctx,
7089 &caller,
7090 &frame,
7091 &target_module_id,
7092 in_flight,
7093 )?]);
7094 }
7095 };
7096
7097 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
7100 RouteBindAdmission::Admitted { guard, probe } => {
7101 if probe {
7102 info!(
7103 module_id = %target_module_id,
7104 connection_id = ctx.connection_id.get(),
7105 "route.bind breaker half-open: admitting one probe"
7106 );
7107 }
7108 guard
7109 }
7110 RouteBindAdmission::Refused {
7111 consecutive_timeouts,
7112 retry_in,
7113 probe_in_flight,
7114 } => {
7115 return Ok(vec![self.route_open_breaker_refusal_frame(
7116 ctx,
7117 &caller,
7118 &frame,
7119 &target_module_id,
7120 consecutive_timeouts,
7121 retry_in,
7122 probe_in_flight,
7123 )?]);
7124 }
7125 };
7126
7127 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
7132 let relay_deadline = Instant::now() + route_bind_relay_timeout;
7133 let operator_app = self
7134 .operator_apps
7135 .lock()
7136 .unwrap_or_else(|p| p.into_inner())
7137 .get(&ctx.connection_id)
7138 .cloned();
7139 let production_peer_build =
7140 build_class.as_deref() == Some(subc_protocol::BUILD_CLASS_PRODUCTION);
7141 timings.enter(RouteOpenPhase::EgressReservation);
7142 let pending = match self
7143 .forwarding
7144 .begin_route_bind_relay_for(
7145 ctx.connection_id,
7146 ctx.egress.clone(),
7147 response_version(&frame),
7148 frame.header.corr,
7149 &target_module_id,
7150 principal.clone(),
7151 bound_scope,
7152 project_root,
7153 identity.harness.clone(),
7154 build_class,
7155 operator_app.clone(),
7156 crate::forwarding::RouteAuthority {
7157 incarnation: self.supervisor.daemon_incarnation(),
7158 caller: caller_capture
7159 .as_ref()
7160 .map(|caller| caller.authority.pair()),
7161 },
7162 relay_deadline,
7163 busy_refusals == Some(true),
7164 accepts_close_reasons,
7165 )
7166 .await
7167 {
7168 Ok(pending) => pending,
7169 Err(err) => {
7170 return Ok(vec![self.route_open_refusal_frame(
7171 ctx,
7172 &caller,
7173 &frame,
7174 &target_module_id,
7175 "relay_reservation_failed",
7176 forwarding_error_code(&err),
7177 err.to_string(),
7178 )?])
7179 }
7180 };
7181 timings.enter(RouteOpenPhase::Admission);
7182 let crate::forwarding::PendingRouteBindRelay {
7183 endpoint,
7184 module_sink,
7185 negotiated_ver,
7186 client_channel,
7187 client_epoch,
7188 module_channel,
7189 module_epoch,
7190 corr: relay_corr,
7191 receiver,
7192 } = pending;
7193 let mut reservation =
7194 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
7195
7196 let relay_registration = self
7201 .registry
7202 .get_module_by_connection(endpoint.connection_id)
7203 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7204 let relay_capabilities = relay_registration
7205 .as_ref()
7206 .and_then(|registration| registration.manifest.capabilities.as_ref());
7207 if let Some(selector) = &scope {
7208 if let Err(refusal) = self
7211 .scopes
7212 .read()
7213 .unwrap_or_else(|poisoned| poisoned.into_inner())
7214 .check_bind_proof(&principal, selector)
7215 {
7216 reservation.release_and_disarm();
7217 return Ok(vec![self.route_open_refusal_frame(
7218 ctx,
7219 &caller,
7220 &frame,
7221 &target_module_id,
7222 refusal.code,
7223 refusal.code,
7224 refusal.message,
7225 )?]);
7226 }
7227 }
7228 if let Some(stamp) = scope_stamp.as_ref() {
7229 if let Err(refusal) = crate::scopes::check_target_required_stamp(
7230 stamp_required,
7231 &target_module_id,
7232 relay_capabilities,
7233 )
7234 .and_then(|()| {
7235 crate::scopes::check_target_flow_support(
7236 stamp,
7237 &target_module_id,
7238 relay_capabilities,
7239 )
7240 })
7241 .and_then(|()| {
7242 crate::scopes::check_target_agent_run_support(
7243 stamp,
7244 &target_module_id,
7245 relay_capabilities,
7246 )
7247 }) {
7248 reservation.release_and_disarm();
7249 return Ok(vec![self.route_open_refusal_frame(
7250 ctx,
7251 &caller,
7252 &frame,
7253 &target_module_id,
7254 refusal.code,
7255 refusal.code,
7256 refusal.message,
7257 )?]);
7258 }
7259 }
7260
7261 if scope_stamp
7264 .as_ref()
7265 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
7266 && !crate::scopes::target_supports_federated(relay_capabilities)
7267 {
7268 scope_stamp = None;
7270 }
7271 if !relay_capabilities.is_some_and(|caps| {
7272 caps.provides
7273 .iter()
7274 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
7275 }) {
7276 if let Some(stamp) = scope_stamp.as_mut() {
7277 stamp.workdir = None;
7278 stamp.project_id = None;
7279 }
7280 }
7281 let worker_run = scope_stamp
7286 .as_ref()
7287 .is_some_and(crate::scopes::is_worker_run)
7288 && relay_capabilities.is_some_and(|caps| {
7289 caps.provides
7290 .iter()
7291 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
7292 });
7293 if !worker_run
7294 && !relay_capabilities.is_some_and(|caps| {
7295 caps.provides
7296 .iter()
7297 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
7298 })
7299 {
7300 if let Some(stamp) = scope_stamp.as_mut() {
7301 stamp.attributes.on_behalf_of_agent = None;
7302 }
7303 }
7304 if !relay_capabilities.is_some_and(|caps| {
7305 caps.provides
7306 .iter()
7307 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
7308 }) {
7309 if let Some(stamp) = scope_stamp.as_mut() {
7310 stamp.attributes.flow_version = None;
7311 stamp.attributes.flow_code_hash = None;
7312 stamp.attributes.flow_install_id = None;
7313 }
7314 }
7315 let call_mode = if worker_run
7316 || relay_capabilities
7317 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
7318 {
7319 Some(derive_call_mode(scope_stamp.as_ref()))
7320 } else {
7321 if let Some(stamp) = scope_stamp.as_mut() {
7322 stamp.expires_at_ms = None;
7323 }
7324 None
7325 };
7326
7327 debug!(
7328 connection_id = ctx.connection_id.get(),
7329 client_channel,
7330 client_epoch,
7331 module_channel,
7332 module_epoch,
7333 "reserved route handle pair"
7334 );
7335 let route_principal_label = principal_label(&principal);
7338 let relay = ModuleControlRequest::RouteBind {
7339 route_channel: module_channel,
7340 epoch: module_epoch,
7341 target,
7342 identity,
7343 principal: Some(principal),
7344 consumer_capabilities,
7345 role_versions,
7346 admission_facts,
7347 scope: scope_stamp,
7348 call_mode,
7349 };
7350 let peer = RouteBindPeer::new()
7351 .with_observed_peer(
7352 relay_capabilities
7353 .is_some_and(|caps| {
7354 caps.provides
7355 .iter()
7356 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
7357 })
7358 .then(|| {
7359 self.observed_peers
7360 .lock()
7361 .unwrap_or_else(|p| p.into_inner())
7362 .get(&ctx.connection_id)
7363 .cloned()
7364 .unwrap_or_else(ObservedPeer::unobserved)
7365 }),
7366 )
7367 .with_operator_app(
7368 self.connection_operator_app(operator_app.as_ref())
7369 .filter(|_| {
7370 relay_capabilities.is_some_and(|caps| {
7371 caps.provides
7372 .iter()
7373 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
7374 })
7375 }),
7376 )
7377 .with_caller(caller_for_target(
7378 caller_capture.as_ref(),
7379 relay_capabilities,
7380 &target_module_id,
7381 ))
7382 .with_peer_build(
7383 (production_peer_build
7384 && relay_capabilities.is_some_and(|caps| {
7385 caps.provides.iter().any(|cap| cap == PEER_BUILD_CAPABILITY)
7386 }))
7387 .then(|| subc_protocol::BUILD_CLASS_PRODUCTION.to_owned()),
7388 );
7389 let relay_body = crate::bind_compat::encode_bind(
7390 &relay,
7391 &peer,
7392 &legacy_bind_project,
7393 relay_capabilities,
7394 )
7395 .map_err(|err| {
7396 RouterError::backend(
7397 0,
7398 frame.header.corr,
7399 format!("failed to encode route.bind request: {err}"),
7400 )
7401 })?;
7402 let relay_frame = Frame::build_with_version(
7403 negotiated_ver,
7404 FrameType::Request,
7405 control_flags(),
7406 0,
7407 0,
7408 relay_corr,
7409 relay_body,
7410 )
7411 .map_err(RouterError::FrameBuild)?;
7412
7413 timings.enter(RouteOpenPhase::BindSend);
7414 let sent = module_sink.send(relay_frame).await;
7415 timings.enter(RouteOpenPhase::Admission);
7416 if let Err(err) = sent {
7417 reservation.release_and_disarm();
7418 return Ok(vec![self.route_open_refusal_frame(
7419 ctx,
7420 &caller,
7421 &frame,
7422 &target_module_id,
7423 "relay_send_failed",
7424 "target_unavailable",
7425 err.to_string(),
7426 )?]);
7427 }
7428
7429 #[cfg(test)]
7430 if let Some(hold) = &self.route_bind_relay_sent_hold {
7431 hold.wait().await;
7435 hold.wait().await;
7436 }
7437
7438 if self
7443 .forwarding
7444 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
7445 .map_err(RouterError::Forwarding)?
7446 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
7447 {
7448 self.send_abandoned_route_bind_goodbye(
7449 &module_sink,
7450 negotiated_ver,
7451 module_channel,
7452 module_epoch,
7453 );
7454 }
7455
7456 timings.enter(RouteOpenPhase::BindAckWait);
7457 let outcome = timeout_at(relay_deadline, receiver).await;
7458 timings.enter(RouteOpenPhase::Admission);
7459 match outcome {
7460 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
7461 reservation.disarm();
7462 if breaker.record_accepted() {
7463 info!(
7464 module_id = %target_module_id,
7465 "route.bind breaker closed: the probe was accepted"
7466 );
7467 }
7468 self.observe_route_open_accept(
7469 ctx,
7470 &target_module_id,
7471 &route_principal_label,
7472 &caller.scope,
7473 );
7474 Ok(Vec::new())
7475 }
7476 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
7477 reservation.release_and_disarm();
7478 breaker.record_inconclusive();
7482 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
7483 }
7484 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
7485 reservation.release_and_disarm();
7486 breaker.record_inconclusive();
7487 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
7488 }
7489 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
7490 reservation.release_and_disarm();
7491 breaker.record_inconclusive();
7492 tracing::warn!(
7499 module_id = %target_module_id,
7500 "route.bind relay abandoned: {message}"
7501 );
7502 Ok(vec![self.route_open_refusal_frame(
7503 ctx,
7504 &caller,
7505 &frame,
7506 &target_module_id,
7507 "relay_abandoned",
7508 "target_unavailable",
7509 message,
7510 )?])
7511 }
7512 Ok(Err(_)) => {
7513 reservation.release_and_disarm();
7514 breaker.record_inconclusive();
7515 Ok(vec![self.route_open_refusal_frame(
7516 ctx,
7517 &caller,
7518 &frame,
7519 &target_module_id,
7520 "relay_waiter_canceled",
7521 "target_unavailable",
7522 "route.bind relay waiter was canceled before the module responded",
7523 )?])
7524 }
7525 Err(_) => {
7526 reservation.release_and_disarm();
7527 if let Some(opened) = breaker.record_timeout(
7531 self.route_bind_breaker_threshold,
7532 self.route_bind_breaker_cooldown,
7533 ) {
7534 warn!(
7535 module_id = %target_module_id,
7536 consecutive_timeouts = opened.consecutive_timeouts,
7537 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
7538 reopened_after_probe = opened.reopened_after_probe,
7539 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
7540 );
7541 }
7542 tracing::warn!(
7550 module_id = %target_module_id,
7551 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
7552 "route.bind relay timed out: module did not ack within budget"
7553 );
7554 Ok(vec![self.route_open_refusal_frame(
7555 ctx,
7556 &caller,
7557 &frame,
7558 &target_module_id,
7559 "relay_timed_out",
7560 "module_timeout",
7561 format!(
7562 "module_id '{target_module_id}' did not answer route.bind within {:?}",
7563 route_bind_relay_timeout
7564 ),
7565 )?])
7566 }
7567 }
7568 }
7569
7570 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7571 let response = ClientControlResponse::SupervisorSpawnSnapshot {
7572 snapshot: self.supervisor.spawn_snapshot(),
7573 };
7574 Ok(vec![control_response_body_frame(
7575 &frame,
7576 &response,
7577 "ClientControlResponse::SupervisorSpawnSnapshot",
7578 )?])
7579 }
7580
7581 fn handle_supervisor_spawn_subscribe(
7582 &self,
7583 ctx: &RouteCtx,
7584 frame: Frame,
7585 since: Option<SpawnCursor>,
7586 ) -> Result<Vec<Frame>, RouterError> {
7587 match self.supervisor.subscribe_spawns(
7588 ctx.connection_id,
7589 frame.header.corr,
7590 response_version(&frame),
7591 since,
7592 ctx.egress.clone(),
7593 ) {
7594 Ok(()) => Ok(Vec::new()),
7595 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
7596 Ok(vec![control_error_body_frame(
7597 &frame,
7598 ErrorBody {
7599 code: "spawn_cursor_incarnation_mismatch".to_string(),
7600 message: "spawn cursor belongs to a different daemon incarnation"
7601 .to_string(),
7602 detail: Some(serde_json::json!({
7603 "current_daemon_incarnation": current
7604 })),
7605 },
7606 )?])
7607 }
7608 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
7609 &frame,
7610 ErrorBody {
7611 code: "spawn_cursor_too_old".to_string(),
7612 message: "spawn cursor predates the retained event ring".to_string(),
7613 detail: Some(serde_json::json!({
7614 "oldest_retained_cursor": oldest
7615 })),
7616 },
7617 )?]),
7618 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
7619 0,
7620 frame.header.corr,
7621 format!("failed to open supervisor spawn subscription: {error}"),
7622 )),
7623 }
7624 }
7625
7626 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7627 let handler = self.clone();
7630 let runtime = tokio::runtime::Handle::current();
7631 let subscriber = tracing::dispatcher::get_default(Clone::clone);
7634 let corr = frame.header.corr;
7635 tokio::task::spawn_blocking(move || {
7636 runtime.block_on(
7637 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
7638 .with_subscriber(subscriber),
7639 )
7640 })
7641 .await
7642 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
7643 }
7644
7645 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7646 use crate::config_gate::runtime::ApprovedDaemonView;
7647 let publication = self
7648 .config_runtime
7649 .as_ref()
7650 .and_then(|owner| owner.polled_snapshot().ok());
7651 let approved = publication
7652 .as_ref()
7653 .map(|view| view.approved_daemon.clone())
7654 .unwrap_or_else(|| {
7655 if self.config_runtime.is_some() {
7656 ApprovedDaemonView::Unavailable
7657 } else {
7658 ApprovedDaemonView::NotStamped
7659 }
7660 });
7661 let generation = self
7662 .registry
7663 .generation()
7664 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7665 let supervised = self.supervisor.list();
7666 let mut statuses = BTreeMap::new();
7667 let mut not_running = BTreeSet::new();
7668 for module in &supervised {
7669 let status = module.status_for_control("list").map_err(|err| {
7670 RouterError::backend(
7671 0,
7672 frame.header.corr,
7673 format!("failed to read supervisor status: {err}"),
7674 )
7675 })?;
7676 if !status.enabled || !status.process_alive {
7677 not_running.insert(module.module_id().to_owned());
7678 }
7679 statuses.insert(module.module_id().to_owned(), status);
7680 }
7681 let mut process_resources = BTreeMap::new();
7684 for module in &supervised {
7685 process_resources.insert(
7686 module.module_id().to_owned(),
7687 crate::subtree_resources::read_process(module.clone()).await,
7688 );
7689 }
7690 let mut image_readings = BTreeMap::new();
7693 for module in &supervised {
7694 image_readings.insert(
7695 module.module_id().to_owned(),
7696 module.running_image_agreement().await,
7697 );
7698 }
7699 let readings = self.resource_tracker.list(supervised.clone());
7700 let mut modules = Vec::new();
7701 for module in supervised {
7702 let status = statuses
7703 .remove(module.module_id())
7704 .expect("each supervised module has a status snapshot");
7705 let (configured, _) = module.configuration().map_err(|err| {
7706 RouterError::backend(
7707 0,
7708 frame.header.corr,
7709 format!("failed to read module configuration: {err}"),
7710 )
7711 })?;
7712 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
7713 RouterError::backend(
7714 0,
7715 frame.header.corr,
7716 format!("failed to read retry time: {err}"),
7717 )
7718 })?;
7719 let image = image_readings
7720 .remove(module.module_id())
7721 .expect("each supervised module has an image snapshot");
7722 let resources = process_resources.remove(module.module_id());
7723 let mut verdict =
7724 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
7725 verdict.spec = Some(launch_spec_agreement(
7726 &configured,
7727 status.process_alive,
7728 status.launch_spec_digest.as_ref(),
7729 ));
7730 let pending_reload = Some(verdict);
7731 let pending_entry = match &approved {
7732 ApprovedDaemonView::Approved { config, .. } => config
7733 .modules
7734 .iter()
7735 .find(|m| m.module_id == module.module_id()),
7736 _ => None,
7737 };
7738 let pending = self
7739 .pending_module_fields(
7740 &module,
7741 pending_entry,
7742 false,
7743 Self::registration_channel_in_force(
7744 publication
7745 .as_ref()
7746 .and_then(|view| view.registration_declaration(module.module_id()))
7747 .as_deref(),
7748 pending_entry.map(|entry| entry.registration_channel),
7749 ),
7750 )
7751 .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
7752 let script = configured.script.as_ref().map(|entry| {
7753 let approved = self
7754 .config_runtime
7755 .as_ref()
7756 .and_then(|owner| owner.approved_script(&status.module_id, &entry.path));
7757 crate::supervise::script::status(entry, approved.as_ref())
7758 });
7759 let configured_enabled = match &approved {
7760 ApprovedDaemonView::Approved {
7761 config: approved, ..
7762 } => approved
7763 .modules
7764 .iter()
7765 .find(|module| module.module_id == status.module_id)
7766 .map(|module| module.enabled),
7767 ApprovedDaemonView::NotStamped => self
7768 .capability_evaluator
7769 .configured_enabled(&status.module_id),
7770 ApprovedDaemonView::Unavailable => None,
7771 };
7772 let admission_refused = status
7773 .admission_refused
7774 .clone()
7775 .filter(|_| status.enabled && !status.process_alive)
7776 .filter(|_| {
7777 publication
7778 .as_ref()
7779 .and_then(|view| view.registration_declaration(&status.module_id))
7780 .as_deref()
7781 == Some("undeclared")
7782 });
7783 let mut entry = SupervisorEntry::new(
7786 status.module_id,
7787 status.state.to_string(),
7788 status.enabled,
7789 status.live,
7790 status.health.status,
7791 )
7792 .with_configured_enabled(configured_enabled)
7793 .with_script(script)
7794 .with_admission_refused(admission_refused)
7795 .with_next_retry_at_ms(next_retry_at_ms)
7796 .with_launch_nonce_env(Some(!cfg!(unix)))
7797 .with_launch_nonce_source(status.launch_nonce_source)
7798 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
7799 .with_protocol(status.protocol)
7800 .with_pending_reload(pending_reload)
7801 .with_last_probe_ms(status.health.last_probe_ms)
7802 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
7803 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
7804 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
7805 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
7806 .with_restart_count(Some(status.restart_count))
7807 .with_max_restarts(Some(status.max_restarts))
7808 .with_lifetime_restarts(Some(status.lifetime_restarts))
7809 .with_spawn_generation(Some(status.spawn_generation))
7810 .with_restart_window_secs(Some(status.restart_window.as_secs()))
7811 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
7812 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
7813 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
7814 .with_resources(resources);
7815 entry.pending = pending;
7816 modules.push(entry);
7817 }
7818 for module in &mut modules {
7819 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
7820 module.tree = tree.clone();
7821 module.cpu_last_hour = hour.clone();
7822 }
7823 module.tree_unavailable = readings
7824 .reasons
7825 .get(&module.module_id)
7826 .map(|reason| (*reason).to_owned());
7827 if module.tree.is_none() {
7828 module.tree_unavailable = Some(
7829 if not_running.contains(&module.module_id) {
7830 "not_running"
7831 } else if let Some(reason) = readings.reasons.get(&module.module_id) {
7832 reason
7833 } else {
7834 "not_yet_measured"
7835 }
7836 .to_owned(),
7837 );
7838 }
7839 }
7840 let response = ClientControlResponse::SupervisorList {
7841 generation,
7842 modules,
7843 daemon: readings.daemon,
7844 agent_commands: readings.agent_commands,
7845 };
7846 Ok(vec![control_response_body_frame(
7847 &frame,
7848 &response,
7849 "ClientControlResponse::SupervisorList",
7850 )?])
7851 }
7852
7853 fn handle_supervisor_stderr_tail(
7854 &self,
7855 frame: Frame,
7856 module_id: String,
7857 max_lines: Option<u32>,
7858 max_bytes: Option<u32>,
7859 ) -> Result<Vec<Frame>, RouterError> {
7860 let Some(module) = self.supervisor.get(&module_id) else {
7861 return Ok(vec![control_error_frame(
7862 &frame,
7863 "unknown_module",
7864 format!("module_id '{module_id}' is not supervised"),
7865 )?]);
7866 };
7867
7868 let snapshot = module.stderr_tail(
7869 max_lines.map(|value| value as usize),
7870 max_bytes.map(|value| value as usize),
7871 );
7872
7873 let response = ClientControlResponse::SupervisorStderrTail {
7874 module_id,
7875 tail: StderrTail {
7876 capture: match snapshot.capture {
7877 CaptureState::Captured => StderrCaptureState::Captured,
7878 CaptureState::Incomplete { reason } => {
7879 StderrCaptureState::Incomplete { reason }
7880 }
7881 CaptureState::NotCaptured { reason } => {
7882 StderrCaptureState::NotCaptured { reason }
7883 }
7884 },
7885 entries: snapshot
7886 .entries
7887 .into_iter()
7888 .map(|entry| match entry {
7889 TailEntry::Line {
7890 text,
7891 truncated,
7892 at_ms,
7893 } => StderrTailEntry::Line {
7894 text,
7895 truncated,
7896 at_ms,
7897 },
7898 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
7899 })
7900 .collect(),
7901 dropped_lines: snapshot.dropped_lines,
7902 },
7903 };
7904 Ok(vec![control_response_body_frame(
7905 &frame,
7906 &response,
7907 "ClientControlResponse::SupervisorStderrTail",
7908 )?])
7909 }
7910
7911 async fn observe_control_wait<F: std::future::Future>(
7912 &self,
7913 ctx: &RouteCtx,
7914 target: Option<&str>,
7915 kind: &'static str,
7916 future: F,
7917 ) -> F::Output {
7918 self.counters
7919 .observe_reader_wait(
7920 || {
7921 let module = self
7922 .forwarding
7923 .module_id_for_connection(ctx.connection_id)
7924 .ok()
7925 .flatten();
7926 let principal = if module.is_some() { "module" } else { "direct" };
7927 (
7928 self.resolve_reader_wait_target(kind, target)
7929 .or(module)
7930 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
7931 principal.into(),
7932 )
7933 },
7934 kind,
7935 future,
7936 )
7937 .await
7938 }
7939
7940 pub(crate) fn resolve_reader_wait_target(
7943 &self,
7944 op: &str,
7945 selector: Option<&str>,
7946 ) -> Option<String> {
7947 let selector = selector?;
7948 match op {
7949 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
7950 .supervisor
7951 .get(selector)
7952 .map(|module| module.module_id().to_owned()),
7953 ops::SUPERVISOR_HEALTH_PROBE => self
7954 .registry
7955 .get_module(selector)
7956 .ok()
7957 .flatten()
7958 .map(|registration| registration.manifest.module_id),
7959 _ => None,
7960 }
7961 }
7962
7963 async fn handle_supervisor_terminals(
7964 &self,
7965 ctx: &RouteCtx,
7966 frame: Frame,
7967 module_id: String,
7968 ) -> Result<Vec<Frame>, RouterError> {
7969 let Some(module) = self.supervisor.get(&module_id) else {
7970 return Ok(vec![control_error_frame(
7971 &frame,
7972 "unknown_module",
7973 format!("module_id '{module_id}' is not supervised"),
7974 )?]);
7975 };
7976
7977 let terminals = self
7980 .observe_control_wait(
7981 ctx,
7982 Some(module.module_id()),
7983 ops::SUPERVISOR_TERMINALS,
7984 module.read_durable_terminal_history(),
7985 )
7986 .await
7987 .map_err(|error| {
7988 RouterError::backend(
7989 0,
7990 frame.header.corr,
7991 format!("failed to read terminal history: {error}"),
7992 )
7993 })?;
7994 let response = ClientControlResponse::SupervisorTerminals {
7995 module_id,
7996 terminals,
7997 };
7998 Ok(vec![control_response_body_frame(
7999 &frame,
8000 &response,
8001 "ClientControlResponse::SupervisorTerminals",
8002 )?])
8003 }
8004
8005 fn handle_supervisor_routes(
8006 &self,
8007 frame: Frame,
8008 module_id: Option<String>,
8009 ) -> Result<Vec<Frame>, RouterError> {
8010 let modules = self
8011 .forwarding
8012 .route_census(module_id.as_deref())
8013 .map_err(RouterError::Forwarding)?
8014 .into_iter()
8015 .map(|(module_id, routes)| SupervisorRouteModule {
8016 module_id,
8017 routes: routes
8018 .into_iter()
8019 .map(|route| SupervisorRoute {
8020 consumer: match route.principal {
8021 Principal::Reserved { module_id } => {
8022 SupervisorRouteConsumer::Reserved { module_id }
8023 }
8024 Principal::Direct | Principal::Unverified => {
8025 SupervisorRouteConsumer::Direct {
8026 connection_id: route.goodbye_target.connection_id.get(),
8027 }
8028 }
8029 },
8030 age_ms: Instant::now()
8031 .saturating_duration_since(route.bound_at)
8032 .as_millis()
8033 .try_into()
8034 .unwrap_or(u64::MAX),
8035 draining: route.draining,
8036 drain_reason: route.drain_reason,
8037 })
8038 .collect(),
8039 })
8040 .collect();
8041 let response = ClientControlResponse::SupervisorRoutes { modules };
8042 Ok(vec![control_response_body_frame(
8043 &frame,
8044 &response,
8045 "ClientControlResponse::SupervisorRoutes",
8046 )?])
8047 }
8048
8049 fn running_registration_method(
8050 &self,
8051 module: &str,
8052 ) -> Option<subc_control::RegistrationMethod> {
8053 let registration = self.registry.get_module(module).ok()??;
8054 let connection = registration.connection_id;
8055 let channel = self.supervisor.registration_spawn(connection).is_some();
8056 let nonce = if channel {
8057 None
8058 } else {
8059 self.hello_launch_nonces
8060 .lock()
8061 .ok()?
8062 .nonce(connection)
8063 .map(str::to_owned)
8064 };
8065 if !channel && nonce.is_none() {
8066 return None;
8067 }
8068 self.supervisor
8069 .registration_snapshot_is_current_live(
8070 &self.registry,
8071 module,
8072 connection,
8073 nonce.as_deref(),
8074 )
8075 .then_some(if channel {
8076 subc_control::RegistrationMethod::Channel
8077 } else {
8078 subc_control::RegistrationMethod::Nonce
8079 })
8080 }
8081
8082 async fn handle_supervisor_provenance(
8083 &self,
8084 ctx: &RouteCtx,
8085 frame: Frame,
8086 module_id: Option<String>,
8087 ) -> Result<Vec<Frame>, RouterError> {
8088 let targeted = module_id.is_some();
8089 let selected = if let Some(module_id) = module_id {
8090 let Some(module) = self.supervisor.get(&module_id) else {
8091 return Ok(vec![control_error_frame(
8092 &frame,
8093 "unknown_module",
8094 format!("module_id '{module_id}' is not supervised"),
8095 )?]);
8096 };
8097 vec![module]
8098 } else {
8099 self.supervisor.list()
8100 };
8101
8102 let target = targeted.then(|| selected[0].module_id());
8103 let declarations = if let Some(owner) = &self.config_runtime {
8104 owner
8105 .on_blocking(|owner| owner.registration_declarations())
8106 .await
8107 .ok()
8108 } else {
8109 None
8110 };
8111 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
8112 let mut modules = Vec::with_capacity(selected.len());
8113 let mut extensions = Vec::with_capacity(selected.len());
8114 for module in &selected {
8115 let (status, observed_image) = module
8116 .status_and_running_image_agreement()
8117 .await
8118 .map_err(|err| {
8119 RouterError::backend(
8120 0,
8121 frame.header.corr,
8122 format!("failed to read supervisor status: {err}"),
8123 )
8124 })?;
8125 let module_declared = self
8126 .registry
8127 .get_module(&status.module_id)
8128 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
8129 .and_then(|registration| registration.manifest.provenance)
8130 .map(|build| ModuleDeclaredProvenance::Reported { build })
8131 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
8132 #[cfg(test)]
8133 let running_image = match &self.provenance_probe_override {
8134 Some(result) => result.clone(),
8135 None => observed_image,
8136 };
8137 #[cfg(not(test))]
8138 let running_image = observed_image;
8139 let registration = self.running_registration_method(&status.module_id);
8140 let mut extension = subc_control::SupervisorModuleProvenanceExt::new()
8141 .with_registration(registration);
8142 extension.registration_declaration = declarations
8143 .as_ref()
8144 .and_then(|entries| entries.get(&status.module_id))
8145 .cloned();
8146 if status.process_alive && status.registration_declaration.is_some() {
8147 extension.registration_declaration = status.registration_declaration.clone();
8148 }
8149 extensions.push(extension);
8150 modules.push(SupervisorModuleProvenance {
8151 module_id: status.module_id,
8152 module_declared,
8153 daemon_observed: SupervisorObservedProcess {
8154 launch_nonce_source: status.launch_nonce_source,
8155 pid: status.pid,
8156 spawned_at_ms: status.spawned_at_ms,
8157 spawned_from: status.spawned_from,
8158 running_image,
8159 },
8160 });
8161 }
8162 let daemon = SupervisorDaemonProvenance {
8163 daemon_build: self.daemon_provenance.build.clone(),
8164 daemon_observed: DaemonObservedProcess {
8165 pid: self.daemon_provenance.pid,
8166 started_at_ms: self
8167 .daemon_provenance
8168 .start_clock
8169 .map(|clock| clock.started_at_ms())
8170 .or(self.daemon_provenance.started_at_ms),
8171 running_image: self
8172 .daemon_provenance
8173 .probe
8174 .observe(
8175 self.daemon_provenance.pid,
8176 self.daemon_provenance.executable_path.as_deref(),
8177 self.daemon_provenance.executable_identity.clone(),
8178 self.daemon_provenance.process_start_time,
8179 )
8180 .await,
8181 },
8182 };
8183 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
8184 let mut response = serde_json::to_value(response)
8185 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
8186 for (entry, extension) in response["modules"]
8187 .as_array_mut()
8188 .expect("provenance modules are an array")
8189 .iter_mut()
8190 .zip(extensions)
8191 {
8192 extension.write_to(entry).map_err(|error| {
8193 RouterError::backend(0, frame.header.corr, error.to_string())
8194 })?;
8195 }
8196 Ok(vec![control_response_body_frame(
8197 &frame,
8198 &response,
8199 "ClientControlResponse::SupervisorProvenance",
8200 )?])
8201 })
8202 .await
8203 }
8204
8205 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
8206 self.refresh_capability_requirements();
8207 let publication = self
8208 .config_runtime
8209 .as_ref()
8210 .and_then(|owner| owner.polled_snapshot().ok());
8211 let generation = self
8212 .registry
8213 .generation()
8214 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
8215 let modules = self
8216 .supervisor
8217 .list()
8218 .into_iter()
8219 .map(|module| {
8220 let status = module.status_for_control("health").map_err(|err| {
8221 RouterError::backend(
8222 0,
8223 frame.header.corr,
8224 format!("failed to read supervisor health: {err}"),
8225 )
8226 })?;
8227 let module_id = status.module_id;
8228 let capability_detail = self
8229 .capability_evaluator
8230 .required_problem_detail(&module_id);
8231 let declaration_warning = self.registry.get_module(&module_id).ok().flatten()
8232 .filter(|registration| registration.manifest.registration_channel == Some(false))
8233 .filter(|_| publication.as_ref().and_then(|view| view.registration_declaration(&module_id))
8234 .as_deref().map(|next| if status.process_alive { status.registration_declaration.as_deref().unwrap_or(next) } else { next }) == Some("declared"))
8235 .map(|_| format!("{module_id}: HELLO registration_channel is false despite the approved channel declaration"));
8236 let capability_detail = append_capability_problem_detail(capability_detail, declaration_warning);
8237 Ok(SupervisorHealthEntry {
8238 module_id,
8239 status: status.health.status,
8240 detail: append_capability_problem_detail(
8241 status.health.detail,
8242 capability_detail,
8243 ),
8244 metrics: status.health.metrics,
8245 consecutive_failures: status.health.consecutive_failures,
8246 late_answer_count: status.health.late_answer_count,
8247 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
8248 last_action: status.health.last_action,
8249 last_action_ms: status.health.last_action_ms,
8250 last_probe_ms: status.health.last_probe_ms,
8251 })
8252 })
8253 .collect::<Result<Vec<_>, RouterError>>()?;
8254 let response = ClientControlResponse::SupervisorHealth {
8255 generation,
8256 modules,
8257 };
8258 Ok(vec![control_response_body_frame(
8259 &frame,
8260 &response,
8261 "ClientControlResponse::SupervisorHealth",
8262 )?])
8263 }
8264
8265 async fn handle_supervisor_restart(
8266 &self,
8267 frame: Frame,
8268 module_id: String,
8269 drain_timeout_ms: Option<u64>,
8270 requester: Principal,
8271 ) -> Result<Vec<Frame>, RouterError> {
8272 let operation_lock = self.supervisor.operation_lock();
8273 let _operation_guard = operation_lock.lock().await;
8274 let Some(module) = self.supervisor.get(&module_id) else {
8275 return Ok(vec![control_error_frame(
8276 &frame,
8277 "unknown_module",
8278 format!("module_id '{module_id}' is not supervised"),
8279 )?]);
8280 };
8281
8282 self.route_outages.mark_operator_action(&module_id);
8283 if let Err(err) = module
8284 .restart_requested(drain_timeout_ms, Some(requester))
8285 .await
8286 {
8287 self.route_outages
8288 .operator_action_ended_unrefused(&module_id);
8289 let (code, message) = match err {
8290 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8291 crate::supervise::SuperviseError::Disabled { .. } => {
8292 ("module_disabled", err.to_string())
8293 }
8294 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8295 ("swap_in_progress", err.to_string())
8296 }
8297 _ => (
8298 "target_unavailable",
8299 format!("failed to restart module_id '{module_id}': {err}"),
8300 ),
8301 };
8302 return Ok(vec![control_error_frame(&frame, code, message)?]);
8303 }
8304
8305 let response = ClientControlResponse::SupervisorAck {
8306 module_id,
8307 applied: true,
8308 };
8309 Ok(vec![control_response_body_frame(
8310 &frame,
8311 &response,
8312 "ClientControlResponse::SupervisorAck",
8313 )?])
8314 }
8315
8316 async fn handle_supervisor_swap(
8320 &self,
8321 frame: Frame,
8322 module_id: String,
8323 ready_timeout_ms: Option<u64>,
8324 requester: Principal,
8325 ) -> Result<Vec<Frame>, RouterError> {
8326 let module = {
8333 let operation_lock = self.supervisor.operation_lock();
8334 let _operation_guard = operation_lock.lock().await;
8335 self.supervisor.get(&module_id)
8336 };
8337 let Some(module) = module else {
8338 return Ok(vec![control_error_frame(
8339 &frame,
8340 "unknown_module",
8341 format!("module_id '{module_id}' is not supervised"),
8342 )?]);
8343 };
8344
8345 self.route_outages.mark_operator_action(&module_id);
8346 if let Err(err) = module
8347 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
8348 .await
8349 {
8350 self.route_outages
8351 .operator_action_ended_unrefused(&module_id);
8352 use crate::supervise::SuperviseError;
8353 let message = err.to_string();
8354 let error = match err {
8355 SuperviseError::ConfigHeld { .. } => ErrorBody::new("config_gated", message),
8356 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
8357 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
8358 code: "swap_refused".to_string(),
8359 message,
8360 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
8361 },
8362 SuperviseError::SwapFailed {
8363 arm,
8364 candidate_exit,
8365 ..
8366 } => ErrorBody {
8367 code: "swap_failed".to_string(),
8368 message,
8369 detail: Some(serde_json::json!({
8370 "arm": arm.as_str(),
8371 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
8372 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
8373 })),
8374 },
8375 _ => ErrorBody::new(
8376 "target_unavailable",
8377 format!("failed to swap module_id '{module_id}': {message}"),
8378 ),
8379 };
8380 return Ok(vec![control_error_body_frame(&frame, error)?]);
8381 }
8382 self.route_outages
8386 .operator_action_ended_unrefused(&module_id);
8387
8388 let response = ClientControlResponse::SupervisorAck {
8389 module_id,
8390 applied: true,
8391 };
8392 Ok(vec![control_response_body_frame(
8393 &frame,
8394 &response,
8395 "ClientControlResponse::SupervisorAck",
8396 )?])
8397 }
8398
8399 async fn handle_supervisor_reload(
8400 &self,
8401 frame: Frame,
8402 module_id: String,
8403 requester: Principal,
8404 ) -> Result<Vec<Frame>, RouterError> {
8405 let operation_lock = self.supervisor.operation_lock();
8406 let _operation_guard = operation_lock.lock().await;
8407 let Some(module) = self.supervisor.get(&module_id) else {
8408 return Ok(vec![control_error_frame(
8409 &frame,
8410 "unknown_module",
8411 format!("module_id '{module_id}' is not supervised"),
8412 )?]);
8413 };
8414
8415 self.route_outages.mark_operator_action(&module_id);
8416 if let Err(err) = module.reload_requested(Some(requester)).await {
8417 self.route_outages
8418 .operator_action_ended_unrefused(&module_id);
8419 let (code, message) = match err {
8420 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8421 crate::supervise::SuperviseError::Disabled { .. } => {
8422 ("module_disabled", err.to_string())
8423 }
8424 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8425 ("swap_in_progress", err.to_string())
8426 }
8427 _ => (
8428 "reload_failed",
8429 format!("failed to reload module_id '{module_id}': {err}"),
8430 ),
8431 };
8432 return Ok(vec![control_error_frame(&frame, code, message)?]);
8433 }
8434
8435 let response = ClientControlResponse::SupervisorAck {
8436 module_id,
8437 applied: true,
8438 };
8439 Ok(vec![control_response_body_frame(
8440 &frame,
8441 &response,
8442 "ClientControlResponse::SupervisorAck",
8443 )?])
8444 }
8445
8446 async fn handle_supervisor_rescan(
8450 &self,
8451 frame: Frame,
8452 mode: RescanMode,
8453 requester: Principal,
8454 ) -> Result<Vec<Frame>, RouterError> {
8455 let preview = mode.previews();
8456 let mut approved_preview = None;
8457 if matches!(mode, RescanMode::Preview) {
8458 if let Some(owner) = &self.config_runtime {
8459 match owner
8460 .on_blocking(|owner| {
8461 owner.observe_file_proposal()?;
8462 owner.approved_daemon_configuration()
8463 })
8464 .await
8465 {
8466 Ok(config) => approved_preview = config,
8467 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8468 }
8469 }
8470 }
8471 let Some(context) = self.rescan.clone() else {
8472 return Ok(vec![control_error_frame(
8473 &frame,
8474 "rescan_unavailable",
8475 "the daemon was not started with a reloadable config path".to_string(),
8476 )?]);
8477 };
8478
8479 let operation_lock = self.supervisor.operation_lock();
8480 let _operation_guard = operation_lock.lock().await;
8481 let loaded = match self.load_rescan_config(&mode, &context) {
8485 Ok(config) => config,
8486 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
8487 };
8488 let Some((config, activity_config)) = loaded else {
8496 return Ok(vec![control_error_frame(
8497 &frame,
8498 "invalid_daemon_config",
8499 format!(
8500 "daemon config not found at {}; refusing to rescan (an absent config would \
8501 retire every supervised module)",
8502 context.config_path.display()
8503 ),
8504 )?]);
8505 };
8506 let (
8507 configured_port,
8508 storage_config,
8509 admission_facts_carrier_module_id,
8510 admission_facts_targets,
8511 scope_authority_owners,
8512 federated_scope_owner,
8513 privileged_operator_prompt,
8514 modules,
8515 reserved_capabilities,
8516 ) = (
8517 config.port,
8518 config.storage,
8519 config.admission_facts_carrier_module_id,
8520 config.admission_facts_targets,
8521 config.scope_authority_owners,
8522 config.federated_scope_owner,
8523 config.privileged_operator_prompt,
8524 config.modules,
8525 config.reserved_capabilities,
8526 );
8527
8528 let mut restart_required = Vec::new();
8537 for section in RestartRequiredSection::ALL {
8538 let changed = match section {
8539 RestartRequiredSection::Port => configured_port != context.configured_port,
8540 RestartRequiredSection::Storage => storage_config != context.storage_config,
8541 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
8542 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
8543 }
8544 RestartRequiredSection::AdmissionFactsTargets => {
8545 admission_facts_targets != context.admission_facts_targets
8546 }
8547 RestartRequiredSection::ScopeAuthorityOwners => {
8548 scope_authority_owners != context.scope_authority_owners
8549 }
8550 RestartRequiredSection::FederatedScopeOwner => {
8551 federated_scope_owner != context.federated_scope_owner
8552 }
8553 RestartRequiredSection::PrivilegedOperatorPrompt => {
8554 privileged_operator_prompt != context.privileged_operator_prompt
8555 }
8556 };
8557 if changed {
8558 restart_required.push(section.label().to_string());
8559 }
8560 }
8561 if !restart_required.is_empty() {
8562 warn!(
8563 config_path = %context.config_path.display(),
8564 sections = %restart_required.join(", "),
8565 "daemon config changed outside the modules section; restart the daemon to apply those changes"
8566 );
8567 }
8568
8569 for configured in &modules {
8570 if let Err(err) = validate_spec(&configured.module_spec()) {
8571 return Ok(vec![control_error_frame(
8572 &frame,
8573 "invalid_daemon_config",
8574 format!("supervisor rescan rejected daemon config: {err}"),
8575 )?]);
8576 }
8577 }
8578
8579 let configured_capabilities = modules
8580 .iter()
8581 .map(|module| (module.module_id.clone(), module.enabled))
8582 .collect::<Vec<_>>();
8583 let preview_capability_warnings = if preview {
8584 let (_, registrations) = self.runtime_capability_snapshot()?;
8585 let current_modules = self
8586 .supervisor
8587 .list()
8588 .into_iter()
8589 .map(|module| module.module_id().to_string())
8590 .collect::<BTreeSet<_>>();
8591 let resulting_modules = configured_capabilities.clone();
8592 let removed = current_modules
8593 .into_iter()
8594 .filter(|module_id| {
8595 !resulting_modules
8596 .iter()
8597 .any(|(configured_id, _)| configured_id == module_id)
8598 })
8599 .collect::<Vec<_>>();
8600 let disabled = match &mode {
8603 RescanMode::Candidate(_) => resulting_modules
8604 .iter()
8605 .filter(|(_, enabled)| !enabled)
8606 .map(|(module_id, _)| module_id.clone())
8607 .collect::<Vec<_>>(),
8608 _ => Vec::new(),
8609 };
8610 self.capability_evaluator.preview_warnings(
8611 resulting_modules,
8612 &removed,
8613 &disabled,
8614 ®istrations,
8615 )
8616 } else {
8617 Vec::new()
8618 };
8619 let reconciled = match (&mode, approved_preview.as_ref()) {
8620 (RescanMode::Preview, Some(approved)) => Ok((
8621 config_gate::preview_approved_daemon(approved, &modules),
8622 Vec::new(),
8623 )),
8624 (RescanMode::Candidate(candidate), _) => self
8625 .preview_candidate(&context, modules, candidate)
8626 .map(|result| (result, Vec::new())),
8627 _ => {
8628 self.reconcile_supervised_modules(
8629 &context.supervisor,
8630 modules,
8631 preview,
8632 requester.clone(),
8633 )
8634 .await
8635 }
8636 };
8637 let (result, changed) = match reconciled {
8638 Ok(result) => result,
8639 Err(message) => {
8640 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
8641 }
8642 };
8643 if !preview {
8644 if let Some(ledger) = &self.activity {
8645 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
8646 ledger.config_changed(&result.added, &result.removed, &changed, requester);
8647 }
8648 if let Err(code) = self.configure_activity(activity_config).await {
8649 warn!(
8650 code,
8651 "activity retention could not be applied during rescan"
8652 );
8653 }
8654 self.capability_evaluator
8655 .configure(configured_capabilities, reserved_capabilities);
8656 self.capability_evaluator.wake_deadline_loop();
8657 self.refresh_capability_requirements();
8658 }
8659 let mut result = result;
8660 result.restart_required = restart_required;
8661 result.capability_warnings = preview_capability_warnings;
8662 let response = ClientControlResponse::SupervisorRescan { result };
8663 Ok(vec![control_response_body_frame(
8664 &frame,
8665 &response,
8666 "ClientControlResponse::SupervisorRescan",
8667 )?])
8668 }
8669
8670 async fn handle_supervisor_release_reserved(
8671 &self,
8672 frame: Frame,
8673 module_id: String,
8674 ) -> Result<Vec<Frame>, RouterError> {
8675 let Some(context) = self.rescan.clone() else {
8676 return Ok(vec![control_error_frame(
8677 &frame,
8678 "release_unavailable",
8679 "reserved-id release requires a daemon started with a reloadable config path",
8680 )?]);
8681 };
8682 let operation_lock = self.supervisor.operation_lock();
8683 let _operation_guard = operation_lock.lock().await;
8684 let approved = if let Some(owner) = &self.config_runtime {
8685 match owner
8688 .on_blocking(|owner| {
8689 owner.observe_file_proposal()?;
8690 owner.approved_daemon_configuration()
8691 })
8692 .await
8693 {
8694 Ok(config) => config,
8695 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8696 }
8697 } else {
8698 None
8699 };
8700 let loaded = if let Some(approved) = approved {
8701 Ok(Some(approved))
8702 } else {
8703 crate::daemon_config::load(&context.config_path)
8704 };
8705 let loaded = match loaded {
8706 Ok(Some(config)) => config,
8707 Ok(None) => {
8708 return Ok(vec![control_error_frame(
8709 &frame,
8710 "invalid_daemon_config",
8711 format!(
8712 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
8713 context.config_path.display()
8714 ),
8715 )?])
8716 }
8717 Err(err) => {
8718 return Ok(vec![control_error_frame(
8719 &frame,
8720 "invalid_daemon_config",
8721 format!("unable to verify reserved-id release against daemon config: {err}"),
8722 )?])
8723 }
8724 };
8725 if loaded
8726 .modules
8727 .iter()
8728 .any(|configured| configured.module_id == module_id)
8729 {
8730 return Ok(vec![control_error_frame(
8731 &frame,
8732 "reserved_module_configured",
8733 format!(
8734 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
8735 ),
8736 )?]);
8737 }
8738 if !self.supervisor.release_retained_reserved_gate(&module_id) {
8739 return Ok(vec![control_error_frame(
8740 &frame,
8741 "reserved_gate_not_retained",
8742 format!(
8743 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
8744 ),
8745 )?]);
8746 }
8747
8748 let response = ClientControlResponse::SupervisorAck {
8749 module_id,
8750 applied: true,
8751 };
8752 Ok(vec![control_response_body_frame(
8753 &frame,
8754 &response,
8755 "ClientControlResponse::SupervisorAck",
8756 )?])
8757 }
8758
8759 fn plan_rescan(
8763 &self,
8764 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8765 ) -> Result<RescanPlan, String> {
8766 let mut current = BTreeMap::new();
8767 for module in self.supervisor.list() {
8768 let (spec, health) = module.configuration().map_err(|err| {
8769 format!(
8770 "failed to read configuration for module_id '{}': {err}",
8771 module.module_id()
8772 )
8773 })?;
8774 let enabled = module
8775 .status()
8776 .map_err(|err| {
8777 format!(
8778 "failed to read status for module_id '{}': {err}",
8779 module.module_id()
8780 )
8781 })?
8782 .enabled;
8783 current.insert(
8784 module.module_id().to_string(),
8785 (module, spec, health, enabled),
8786 );
8787 }
8788 let configured = configured_modules
8789 .into_iter()
8790 .map(|module| (module.module_id.clone(), module))
8791 .collect::<BTreeMap<_, _>>();
8792
8793 let added = configured
8794 .keys()
8795 .filter(|module_id| !current.contains_key(*module_id))
8796 .cloned()
8797 .collect::<Vec<_>>();
8798 let removed = current
8799 .keys()
8800 .filter(|module_id| !configured.contains_key(*module_id))
8801 .cloned()
8802 .collect::<Vec<_>>();
8803 let mut changed_pending_reload = Vec::new();
8804 let mut changed_pending = Vec::new();
8805 let mut configuration_changes = BTreeSet::new();
8806 let mut enabled_changes = BTreeSet::new();
8807 let mut unchanged = 0_u32;
8808 let publication = self
8809 .config_runtime
8810 .as_ref()
8811 .and_then(|owner| owner.polled_snapshot().ok());
8812
8813 for (module_id, configured_module) in &configured {
8814 let Some((module, current_spec, current_health, current_enabled)) =
8815 current.get(module_id)
8816 else {
8817 continue;
8818 };
8819 let launch_changed = *current_spec != configured_module.module_spec();
8823 let configuration_changed =
8824 launch_changed || *current_health != configured_module.health;
8825 let pending = self.pending_module_fields(
8826 module,
8827 Some(configured_module),
8828 configuration_changed,
8829 Self::registration_channel_in_force(
8830 publication
8831 .as_ref()
8832 .and_then(|view| view.registration_declaration(module_id))
8833 .as_deref(),
8834 publication
8835 .as_ref()
8836 .and_then(|view| match &view.approved_daemon {
8837 crate::config_gate::runtime::ApprovedDaemonView::Approved {
8838 config,
8839 ..
8840 } => config
8841 .modules
8842 .iter()
8843 .find(|entry| entry.module_id == *module_id)
8844 .map(|entry| entry.registration_channel),
8845 _ => None,
8846 }),
8847 ),
8848 )?;
8849 let has_pending = !pending.is_empty();
8850 if has_pending {
8851 changed_pending.push(subc_control::SupervisorPendingModule {
8852 module_id: module_id.clone(),
8853 fields: pending
8854 .into_iter()
8855 .map(|mut field| {
8856 field.value = None;
8857 field
8858 })
8859 .collect(),
8860 });
8861 }
8862 let enabled_changed = *current_enabled != configured_module.enabled
8865 || self
8866 .capability_evaluator
8867 .configured_enabled(module_id)
8868 .is_some_and(|enabled| enabled != configured_module.enabled);
8869 if configuration_changed {
8870 configuration_changes.insert(module_id.clone());
8871 }
8872 if launch_changed {
8873 changed_pending_reload.push(module_id.clone());
8874 }
8875 if enabled_changed {
8876 enabled_changes.insert(module_id.clone());
8877 }
8878 if !configuration_changed && !enabled_changed && !has_pending {
8879 unchanged = unchanged.saturating_add(1);
8880 }
8881 }
8882 Ok(RescanPlan {
8883 current,
8884 configured,
8885 added,
8886 removed,
8887 changed_pending_reload,
8888 changed_pending,
8889 configuration_changes,
8890 enabled_changes,
8891 unchanged,
8892 })
8893 }
8894
8895 async fn reconcile_supervised_modules(
8904 &self,
8905 supervisor: &Supervisor,
8906 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8907 preview: bool,
8908 requester: Principal,
8909 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
8910 let plan = self.plan_rescan(configured_modules)?;
8911
8912 if preview {
8918 return Ok((plan.preview_result(), Vec::new()));
8919 }
8920 let changed = plan
8921 .configuration_changes
8922 .union(&plan.enabled_changes)
8923 .cloned()
8924 .collect::<Vec<_>>();
8925 let RescanPlan {
8926 current,
8927 configured,
8928 added,
8929 removed,
8930 changed_pending_reload,
8931 changed_pending,
8932 configuration_changes,
8933 enabled_changes,
8934 unchanged,
8935 } = plan;
8936 if !(added.is_empty()
8941 && removed.is_empty()
8942 && configuration_changes.is_empty()
8943 && changed_pending.is_empty()
8944 && enabled_changes.is_empty())
8945 {
8946 if let Some(context) = &self.rescan {
8947 context
8948 .generation
8949 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8950 }
8951 }
8952
8953 for module_id in &removed {
8954 let module = ¤t
8955 .get(module_id)
8956 .expect("removed module came from current supervisor state")
8957 .0;
8958 module
8959 .retire_requested(Some(requester.clone()))
8960 .await
8961 .map_err(|err| {
8962 format!("failed to retire module_id '{module_id}' during rescan: {err}")
8963 })?;
8964 self.supervisor.record_rescan_removal(module_id);
8998 self.supervisor.retire(module_id);
8999 self.route_outages.forget(module_id);
9000 }
9001
9002 for module_id in configured.keys() {
9003 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
9004 continue;
9005 };
9006 let configured_module = configured
9007 .get(module_id)
9008 .expect("configured module id came from configured map");
9009 if configuration_changes.contains(module_id) {
9010 module
9011 .update_configuration(
9012 configured_module.module_spec(),
9013 configured_module.health.clone(),
9014 configured_module.drain_timeout_ms,
9015 )
9016 .await
9017 .map_err(|err| {
9018 format!(
9019 "failed to update module_id '{module_id}' configuration during rescan: {err}"
9020 )
9021 })?;
9022 }
9023 if *current_enabled != configured_module.enabled {
9026 self.route_outages.mark_operator_action(module_id);
9029 module
9030 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
9031 .await
9032 .map_err(|err| {
9033 self.route_outages.operator_action_ended_unrefused(module_id);
9034 format!(
9035 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
9036 configured_module.enabled
9037 )
9038 })?;
9039 }
9040 }
9041
9042 for module_id in &added {
9043 let configured_module = configured
9044 .get(module_id)
9045 .expect("added module id came from configured map");
9046 supervisor
9047 .supervise_configured_requested_async(
9048 configured_module.module_spec(),
9049 configured_module.enabled,
9050 configured_module.health.clone(),
9051 configured_module.drain_timeout_ms,
9052 configured_module.restart,
9053 configured_module.register_timeout,
9054 Some(requester.clone()),
9055 )
9056 .await
9057 .map_err(|err| {
9058 format!("failed to add module_id '{module_id}' during rescan: {err}")
9059 })?;
9060 }
9061
9062 if let Some(context) = &self.rescan {
9063 let mut entries = context
9064 .module_entries
9065 .lock()
9066 .unwrap_or_else(|e| e.into_inner());
9067 entries.record(&configured, &added, &removed);
9068 }
9069
9070 Ok((
9071 SupervisorRescanResult {
9072 added,
9073 removed,
9074 changed_pending_reload,
9075 changed_pending,
9076 enabled_changes: enabled_changes.iter().cloned().collect(),
9077 unchanged,
9078 preview: false,
9079 restart_required: Vec::new(),
9082 capability_warnings: Vec::new(),
9083 module_actions: Vec::new(),
9084 earlier_pending: Vec::new(),
9085 expect: None,
9086 },
9087 changed,
9088 ))
9089 }
9090
9091 async fn handle_supervisor_set_enabled(
9092 &self,
9093 frame: Frame,
9094 module_id: String,
9095 enabled: bool,
9096 requester: Principal,
9097 ) -> Result<Vec<Frame>, RouterError> {
9098 let operation_lock = self.supervisor.operation_lock();
9099 let _operation_guard = operation_lock.lock().await;
9100 let Some(module) = self.supervisor.get(&module_id) else {
9101 return Ok(vec![control_error_frame(
9102 &frame,
9103 "unknown_module",
9104 format!("module_id '{module_id}' is not supervised"),
9105 )?]);
9106 };
9107
9108 self.route_outages.mark_operator_action(&module_id);
9111 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
9112 Ok(applied) => applied,
9113 Err(err) => {
9114 self.route_outages
9115 .operator_action_ended_unrefused(&module_id);
9116 return Ok(vec![control_error_frame(
9117 &frame,
9118 if err.is_config_gate_error() {
9119 "config_gated"
9120 } else {
9121 "target_unavailable"
9122 },
9123 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
9124 )?]);
9125 }
9126 };
9127 if !applied {
9128 self.route_outages
9131 .operator_action_ended_unrefused(&module_id);
9132 }
9133
9134 self.capability_evaluator.wake_deadline_loop();
9135 self.refresh_capability_requirements();
9136 let response = ClientControlResponse::SupervisorAck { module_id, applied };
9137 Ok(vec![control_response_body_frame(
9138 &frame,
9139 &response,
9140 "ClientControlResponse::SupervisorAck",
9141 )?])
9142 }
9143
9144 async fn handle_supervisor_health_probe(
9145 &self,
9146 ctx: &RouteCtx,
9147 frame: Frame,
9148 module_id: String,
9149 ) -> Result<Vec<Frame>, RouterError> {
9150 self.refresh_capability_requirements();
9151 let Some(registration) = self
9152 .registry
9153 .get_module(&module_id)
9154 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
9155 else {
9156 return Ok(vec![control_error_frame(
9157 &frame,
9158 "unknown_module",
9159 format!("module_id '{module_id}' is not registered"),
9160 )?]);
9161 };
9162
9163 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
9176 {
9177 return Ok(vec![control_error_frame(
9178 &frame,
9179 "health_not_advertised",
9180 format!("module_id '{module_id}' did not advertise health.check"),
9181 )?]);
9182 }
9183
9184 self.observe_control_wait(
9185 ctx,
9186 Some(®istration.manifest.module_id),
9187 ops::SUPERVISOR_HEALTH_PROBE,
9188 async {
9189 let deadline = Instant::now() + self.health_probe_timeout;
9190 let pending = match self.forwarding.begin_module_control_rpc_for(
9191 &module_id,
9192 MODULE_CONTROL_OP_HEALTH_CHECK,
9193 deadline,
9194 ) {
9195 Ok(pending) => pending,
9196 Err(err) => {
9197 return Ok(vec![control_error_frame(
9198 &frame,
9199 forwarding_error_code(&err),
9200 err.to_string(),
9201 )?])
9202 }
9203 };
9204
9205 let PendingModuleControlRpc {
9206 endpoint,
9207 module_sink,
9208 negotiated_ver,
9209 corr: probe_corr,
9210 receiver,
9211 } = pending;
9212 let mut guard =
9213 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
9214 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
9215 .map_err(|err| {
9216 RouterError::backend(
9217 0,
9218 frame.header.corr,
9219 format!("failed to encode health.check request: {err}"),
9220 )
9221 })?;
9222 let probe_frame = Frame::build_with_version(
9223 negotiated_ver,
9224 FrameType::Request,
9225 control_flags(),
9226 0,
9227 0,
9228 probe_corr,
9229 probe_body,
9230 )
9231 .map_err(RouterError::FrameBuild)?;
9232
9233 match timeout_at(deadline, module_sink.send(probe_frame)).await {
9236 Ok(Ok(())) => {}
9237 Ok(Err(err)) => {
9238 return Ok(vec![control_error_frame(
9239 &frame,
9240 "target_unavailable",
9241 err.to_string(),
9242 )?]);
9243 }
9244 Err(_) => {
9245 return Ok(vec![control_error_frame(
9246 &frame,
9247 "module_timeout",
9248 format!(
9249 "module_id '{module_id}' did not answer health.check within {:?}",
9250 self.health_probe_timeout
9251 ),
9252 )?]);
9253 }
9254 }
9255
9256 match timeout_at(deadline, receiver).await {
9257 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => {
9258 Ok(vec![control_error_frame(
9259 &frame,
9260 "invalid_control_body",
9261 "health.check returned an unexpected config response",
9262 )?])
9263 }
9264 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
9265 guard.disarm();
9266 let Some(report) = response.health_report() else {
9267 return Ok(vec![control_error_frame(
9268 &frame,
9269 "invalid_control_body",
9270 "health.check RPC returned a non-health response",
9271 )?]);
9272 };
9273 let HealthReport {
9278 status,
9279 detail,
9280 metrics,
9281 } = report;
9282 let capability_detail = self
9283 .capability_evaluator
9284 .required_problem_detail(&module_id);
9285 let response = ClientControlResponse::SupervisorHealthProbe {
9286 module_id,
9287 status,
9288 detail: append_capability_problem_detail(detail, capability_detail),
9289 metrics,
9290 };
9291 Ok(vec![control_response_body_frame(
9292 &frame,
9293 &response,
9294 "ClientControlResponse::SupervisorHealthProbe",
9295 )?])
9296 }
9297 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
9298 guard.disarm();
9299 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
9304 }
9305 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
9306 guard.disarm();
9307 Ok(vec![control_error_frame(
9308 &frame,
9309 "target_unavailable",
9310 message,
9311 )?])
9312 }
9313 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
9314 guard.disarm();
9315 Ok(vec![control_error_frame(
9316 &frame,
9317 "invalid_control_body",
9318 message,
9319 )?])
9320 }
9321 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
9322 guard.disarm();
9323 Ok(vec![control_error_frame(
9324 &frame,
9325 "invalid_control_body",
9326 format!("expected module-control op '{expected}', got '{actual}'"),
9327 )?])
9328 }
9329 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
9330 guard.disarm();
9331 Ok(vec![control_error_frame(
9332 &frame,
9333 "module_timeout",
9334 format!(
9335 "module_id '{module_id}' answered health.check after {:?}",
9336 self.health_probe_timeout
9337 ),
9338 )?])
9339 }
9340 Ok(Err(_)) => Ok(vec![control_error_frame(
9341 &frame,
9342 "target_unavailable",
9343 "health.check waiter was canceled before the module responded",
9344 )?]),
9345 Err(_) => Ok(vec![control_error_frame(
9346 &frame,
9347 "module_timeout",
9348 format!(
9349 "module_id '{module_id}' did not answer health.check within {:?}",
9350 self.health_probe_timeout
9351 ),
9352 )?]),
9353 }
9354 },
9355 )
9356 .await
9357 }
9358
9359 fn supervisor_status(
9360 &self,
9361 module_id: &str,
9362 corr: u64,
9363 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
9364 self.supervisor
9365 .get(module_id)
9366 .map(|module| {
9367 let warming = module.is_warming_for_control("status").map_err(|err| {
9368 RouterError::backend(
9369 0,
9370 corr,
9371 format!(
9372 "failed to read supervisor warming state for module_id '{module_id}': {err}"
9373 ),
9374 )
9375 })?;
9376 module.status_for_control("status").map_err(|err| {
9377 RouterError::backend(
9378 0,
9379 corr,
9380 format!(
9381 "failed to read supervisor status for module_id '{module_id}': {err}"
9382 ),
9383 )
9384 }).map(|status| (status, warming))
9385 })
9386 .transpose()
9387 }
9388
9389 fn guard_module_control_op(
9390 &self,
9391 frame: &Frame,
9392 module_id: &str,
9393 op: &str,
9394 ) -> Result<Option<Frame>, RouterError> {
9395 if self.module_grants_op(module_id, op, frame.header.corr)? {
9396 return Ok(None);
9397 }
9398
9399 Ok(Some(control_error_frame(
9400 frame,
9401 "op_not_allowed",
9402 format!("module_id '{module_id}' did not grant control op '{op}'"),
9403 )?))
9404 }
9405
9406 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
9407 let Some(registration) = self
9408 .registry
9409 .get_module(module_id)
9410 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
9411 else {
9412 return Ok(false);
9413 };
9414 Ok(module_registration_grants_op(®istration.control_ops, op))
9415 }
9416
9417 fn handle_status_update(
9418 &self,
9419 endpoint: ModuleEndpointId,
9420 frame: Frame,
9421 ) -> Result<Vec<Frame>, RouterError> {
9422 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
9423 Ok(update) => update,
9424 Err(err) => {
9425 if is_known_module_push_op(&frame.body) {
9430 return Ok(vec![control_error_frame(
9431 &frame,
9432 "invalid_control_body",
9433 format!("malformed module control push body: {err}"),
9434 )?]);
9435 }
9436 return Ok(Vec::new());
9437 }
9438 };
9439
9440 match update {
9441 ModuleControlPush::RouteStatus {
9442 route_channel,
9443 route_epoch,
9444 status,
9445 } => {
9446 self.forwarding
9447 .cache_status(endpoint, route_channel, route_epoch, status)
9448 .map_err(RouterError::Forwarding)?;
9449 }
9450 }
9451 Ok(Vec::new())
9452 }
9453
9454 fn handle_route_poll(
9455 &self,
9456 ctx: &RouteCtx,
9457 frame: Frame,
9458 route_channel: u16,
9459 route_epoch: u32,
9460 kind: PollKind,
9461 ) -> Result<Vec<Frame>, RouterError> {
9462 let snapshot = self
9463 .forwarding
9464 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
9465 .map_err(RouterError::Forwarding)?;
9466 let response = match (kind, snapshot) {
9467 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
9468 ClientControlResponse::RoutePoll {
9469 route_channel,
9470 route_epoch,
9471 status,
9472 live: None,
9473 }
9474 }
9475 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9476 route_channel,
9477 route_epoch,
9478 status: None,
9479 live: None,
9480 },
9481 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
9482 let live = self
9508 .process_liveness
9509 .as_ref()
9510 .and_then(|source| source.process_live(&module_id))
9511 .unwrap_or(true);
9512 ClientControlResponse::RoutePoll {
9513 route_channel,
9514 route_epoch,
9515 status: None,
9516 live: Some(live),
9517 }
9518 }
9519 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9520 route_channel,
9521 route_epoch,
9522 status: None,
9523 live: Some(false),
9524 },
9525 };
9526
9527 Ok(vec![control_response_body_frame(
9528 &frame,
9529 &response,
9530 "ClientControlResponse::RoutePoll",
9531 )?])
9532 }
9533
9534 pub(crate) fn observe_module_control_completion(
9535 &self,
9536 completion: ModuleControlRpcCompletion,
9537 ) -> bool {
9538 match completion {
9539 ModuleControlRpcCompletion::Unknown => false,
9540 ModuleControlRpcCompletion::Settled => true,
9541 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
9542 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
9543 info!(
9544 module_id = %module_id,
9545 latency_ms,
9546 "late health.check answer proves the module is alive"
9547 );
9548 match self
9549 .supervisor
9550 .record_late_health_answer(&module_id, latency_ms)
9551 {
9552 Ok(true) => {}
9553 Ok(false) => debug!(
9554 module_id = %module_id,
9555 latency_ms,
9556 "late health.check answer has no active supervisor snapshot"
9557 ),
9558 Err(err) => warn!(
9559 module_id = %module_id,
9560 latency_ms,
9561 error = %err,
9562 "failed to record late health.check answer"
9563 ),
9564 }
9565 true
9566 }
9567 }
9568 }
9569
9570 fn refuse_to_end_module_connection_for_a_client(
9591 &self,
9592 module_connection_id: ConnectionId,
9593 corr: u64,
9594 err: ForwardingError,
9595 ) -> Result<(), RouterError> {
9596 if let ForwardingError::ConnectionClosing { connection_id } = err {
9597 if connection_id != module_connection_id {
9598 warn!(
9599 module_connection_id = module_connection_id.get(),
9600 client_connection_id = connection_id.get(),
9601 corr,
9602 "dropping a route.bind response for a closing client; the module connection keeps serving"
9603 );
9604 return Ok(());
9605 }
9606 }
9607 Err(RouterError::Forwarding(err))
9608 }
9609
9610 fn handle_module_relay_response(
9611 &self,
9612 connection_id: ConnectionId,
9613 frame: Frame,
9614 ) -> Result<Vec<Frame>, RouterError> {
9615 let mut secondary_error = None;
9616 let outcome = match frame.header.ty {
9617 FrameType::Response => match module_control_response_probe(&frame.body) {
9618 Ok(probe) if probe.op == "route.bind" => {
9619 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9620 Ok(ModuleControlResponse::RouteBindAck {}) => {
9621 RouteBindRelayOutcome::Accepted
9622 }
9623 Ok(other) => {
9624 let message =
9625 format!("route.bind response carried unexpected body: {other:?}");
9626 secondary_error = Some(control_error_frame(
9627 &frame,
9628 "invalid_control_body",
9629 message.clone(),
9630 )?);
9631 RouteBindRelayOutcome::ModuleGone(message)
9632 }
9633 Err(err) => {
9634 let message = format!("malformed route.bind response body: {err}");
9635 secondary_error = Some(control_error_frame(
9636 &frame,
9637 "invalid_control_body",
9638 message.clone(),
9639 )?);
9640 RouteBindRelayOutcome::ModuleGone(message)
9641 }
9642 }
9643 }
9644 Ok(probe) => {
9645 let outcome = if matches!(
9646 probe.op.as_str(),
9647 "config.review" | "consent/v1" | "projects.enumerate"
9648 ) {
9649 ModuleControlRpcOutcome::RawResponse(frame.body.clone())
9650 } else {
9651 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9652 Ok(response) => ModuleControlRpcOutcome::Response(response),
9653 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9654 "malformed {} response body: {err}",
9655 probe.op
9656 )),
9657 }
9658 };
9659 let completion = self
9660 .forwarding
9661 .complete_module_control_rpc(
9662 connection_id,
9663 frame.header.corr,
9664 Some(&probe.op),
9665 outcome,
9666 )
9667 .map_err(RouterError::Forwarding)?;
9668 if !self.observe_module_control_completion(completion) {
9669 debug!(
9670 connection_id = connection_id.get(),
9671 corr = frame.header.corr,
9672 op = %probe.op,
9673 "dropping late or unknown module-control RPC response"
9674 );
9675 }
9676 return Ok(Vec::new());
9677 }
9678 Err(err) => {
9679 if let Some(expected_op) = self
9680 .forwarding
9681 .pending_module_control_op(connection_id, frame.header.corr)
9682 .map_err(RouterError::Forwarding)?
9683 {
9684 let completion = self
9685 .forwarding
9686 .complete_module_control_rpc(
9687 connection_id,
9688 frame.header.corr,
9689 None,
9690 ModuleControlRpcOutcome::MalformedResponse(format!(
9691 "malformed {expected_op} response body: {err}"
9692 )),
9693 )
9694 .map_err(RouterError::Forwarding)?;
9695 if !self.observe_module_control_completion(completion) {
9696 debug!(
9697 connection_id = connection_id.get(),
9698 corr = frame.header.corr,
9699 "dropping late malformed module-control RPC response"
9700 );
9701 }
9702 return Ok(Vec::new());
9703 }
9704 let message = format!("malformed route.bind response body: {err}");
9705 secondary_error = Some(control_error_frame(
9706 &frame,
9707 "invalid_control_body",
9708 message.clone(),
9709 )?);
9710 RouteBindRelayOutcome::ModuleGone(message)
9711 }
9712 },
9713 FrameType::Error => {
9714 if self
9715 .forwarding
9716 .pending_module_control_op(connection_id, frame.header.corr)
9717 .map_err(RouterError::Forwarding)?
9718 .is_some()
9719 {
9720 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
9721 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
9722 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9723 "malformed module-control ERROR body: {err}"
9724 )),
9725 };
9726 let completion = self
9727 .forwarding
9728 .complete_module_control_rpc(
9729 connection_id,
9730 frame.header.corr,
9731 None,
9732 outcome,
9733 )
9734 .map_err(RouterError::Forwarding)?;
9735 if !self.observe_module_control_completion(completion) {
9736 debug!(
9737 connection_id = connection_id.get(),
9738 corr = frame.header.corr,
9739 "dropping late or unknown module-control RPC error"
9740 );
9741 }
9742 return Ok(Vec::new());
9743 }
9744 match serde_json::from_slice::<ErrorBody>(&frame.body) {
9745 Ok(body) => RouteBindRelayOutcome::Rejected(body),
9746 Err(err) => {
9747 let message = format!("malformed route.bind ERROR body: {err}");
9748 secondary_error = Some(control_error_frame(
9749 &frame,
9750 "invalid_control_body",
9751 message.clone(),
9752 )?);
9753 RouteBindRelayOutcome::ModuleGone(message)
9754 }
9755 }
9756 }
9757 ty => {
9758 return Ok(vec![control_error_frame(
9759 &frame,
9760 "unsupported_control_frame",
9761 format!("unsupported module channel-0 frame {ty:?}"),
9762 )?])
9763 }
9764 };
9765
9766 let settled =
9767 self.forwarding
9768 .complete_pending_relay(connection_id, frame.header.corr, outcome);
9769 let completion = match settled {
9770 Ok(completion) => completion,
9771 Err(err) => {
9772 self.refuse_to_end_module_connection_for_a_client(
9773 connection_id,
9774 frame.header.corr,
9775 err,
9776 )?;
9777 return Ok(secondary_error.into_iter().collect());
9778 }
9779 };
9780 if let Some(target) = completion.abandoned.as_ref() {
9781 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
9782 }
9783 if !completion.settled {
9784 debug!(
9785 connection_id = connection_id.get(),
9786 corr = frame.header.corr,
9787 frame_type = ?frame.header.ty,
9788 "dropping late or unknown route.bind relay response"
9789 );
9790 }
9791 Ok(secondary_error.into_iter().collect())
9792 }
9793
9794 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
9795 debug!(connection_id = connection_id.get(), "handling GOODBYE");
9796 self.cleanup_connection_with_end_reason(
9800 connection_id,
9801 RegistrationEndReason::ExplicitGoodbye,
9802 )
9803 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
9804 Ok(Vec::new())
9805 }
9806}
9807
9808impl Default for ControlHandler {
9809 fn default() -> Self {
9810 Self::new(Arc::new(Registry::default()))
9811 }
9812}
9813
9814impl crate::supervise::SwapPromotionObserver for ControlHandler {
9815 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
9816 self.apply_registration_capabilities(registration);
9817 }
9818}
9819
9820fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
9821 CapabilityRequirementStatus {
9822 consumer: status.consumer,
9823 capability: status.capability,
9824 need: match status.need {
9825 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
9826 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
9827 },
9828 verdict: status.verdict.as_str().to_string(),
9829 episode_seq: status.episode_seq,
9830 config_satisfiable: status.config_satisfiable,
9831 runtime_available: status.runtime_available,
9832 detail: status.detail,
9833 }
9834}
9835
9836fn append_capability_problem_detail(
9837 detail: Option<String>,
9838 capability_detail: Option<String>,
9839) -> Option<String> {
9840 match (detail, capability_detail) {
9841 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
9842 (Some(detail), None) => Some(detail),
9843 (None, Some(capability_detail)) => Some(capability_detail),
9844 (None, None) => None,
9845 }
9846}
9847
9848fn subc_ops() -> Vec<String> {
9849 SUBC_CONTROL_OPS
9850 .iter()
9851 .map(|op| (*op).to_string())
9852 .collect()
9853}
9854
9855fn module_subc_ops() -> Vec<String> {
9856 SUBC_CONTROL_OPS
9857 .iter()
9858 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
9859 .map(|op| (*op).to_string())
9860 .collect()
9861}
9862
9863#[cfg(test)]
9864fn module_baseline_control_ops() -> Vec<String> {
9865 MODULE_BASELINE_CONTROL_OPS
9866 .iter()
9867 .map(|op| (*op).to_string())
9868 .collect()
9869}
9870
9871fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
9872 let mut seen = HashSet::new();
9873 let mut effective = Vec::new();
9874 for op in MODULE_BASELINE_CONTROL_OPS {
9875 if seen.insert((*op).to_string()) {
9876 effective.push((*op).to_string());
9877 }
9878 }
9879 for op in declared.unwrap_or_default() {
9880 if seen.insert(op.clone()) {
9881 effective.push(op);
9882 }
9883 }
9884 effective
9885}
9886
9887fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
9888 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
9889}
9890
9891fn target_module_id(target: &RouteTarget) -> &str {
9892 match target {
9893 RouteTarget::ToolProvider { module_id }
9894 | RouteTarget::ManagementSurface { module_id }
9895 | RouteTarget::InternalService { module_id, .. } => module_id,
9896 }
9897}
9898
9899fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
9900 roles.iter().any(|role| match (target, role) {
9901 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
9902 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
9903 (
9904 RouteTarget::InternalService { service_id, .. },
9905 ProviderRole::InternalService {
9906 service_id: provided,
9907 ..
9908 },
9909 ) => service_id == provided,
9910 _ => false,
9911 })
9912}
9913
9914fn is_routable_role(role: &ProviderRole) -> bool {
9915 matches!(
9916 role,
9917 ProviderRole::ToolProvider { .. }
9918 | ProviderRole::ManagementSurface { .. }
9919 | ProviderRole::InternalService { .. }
9920 )
9921}
9922
9923#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9924enum ControlRequestBodyError {
9925 UnknownOp,
9926 InvalidBody,
9927}
9928
9929#[derive(Debug, Deserialize)]
9930struct ControlOpProbe {
9931 op: String,
9932}
9933
9934fn module_control_response_probe(body: &[u8]) -> Result<ControlOpProbe, serde_json::Error> {
9935 serde_json::from_slice(body).or_else(|original| {
9936 #[derive(Deserialize)]
9937 struct ConsentProbe {
9938 protocol: String,
9939 }
9940 match serde_json::from_slice::<ConsentProbe>(body) {
9941 Ok(probe) if probe.protocol == subc_protocol::consent::CONSENT_V1 => {
9942 Ok(ControlOpProbe { op: probe.protocol })
9943 }
9944 _ => Err(original),
9945 }
9946 })
9947}
9948
9949const MODULE_PUSH_OPS: &[&str] = &["route.status"];
9952
9953fn is_known_module_push_op(body: &[u8]) -> bool {
9954 serde_json::from_slice::<ControlOpProbe>(body)
9955 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
9956 .unwrap_or(false)
9957}
9958
9959fn is_known_module_request_op(body: &[u8]) -> bool {
9960 serde_json::from_slice::<ControlOpProbe>(body)
9961 .map(|probe| is_module_to_subc_op(&probe.op))
9962 .unwrap_or(false)
9963}
9964
9965fn is_module_to_subc_op(op: &str) -> bool {
9966 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
9967}
9968
9969fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
9970 debug!(
9971 op = %op,
9972 connection_id = connection_id.get(),
9973 corr,
9974 "control dispatch"
9975 );
9976}
9977
9978fn log_slow_control_dispatch(
9979 dispatch_started_at: Option<StdInstant>,
9980 op: &'static str,
9981 connection_id: ConnectionId,
9982 corr: u64,
9983 route_open_timings: Option<&RouteOpenTimings>,
9984) {
9985 let Some(dispatch_started_at) = dispatch_started_at else {
9986 return;
9987 };
9988 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
9990 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
9991 if let Some(timings) = route_open_timings {
9992 warn!(
9993 op = %op,
9994 connection_id = connection_id.get(),
9995 corr,
9996 elapsed_ms = elapsed.as_millis() as u64,
9997 module_id = ?timings.module_id,
9998 principal = %timings.principal,
9999 admission_ms = timings.admission.as_millis() as u64,
10000 project_root_ms = timings.project_root.as_millis() as u64,
10001 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
10002 bind_send_ms = timings.bind_send.as_millis() as u64,
10003 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
10004 "slow control dispatch"
10005 );
10006 return;
10007 }
10008 warn!(
10009 op = %op,
10010 connection_id = connection_id.get(),
10011 corr,
10012 elapsed_ms = elapsed.as_millis() as u64,
10013 "slow control dispatch"
10014 );
10015 }
10016}
10017
10018#[cfg(test)]
10020type TestCallerLookup =
10021 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
10022
10023fn privileged_operation(request: &impl PrivilegedRequest) -> Option<PrivilegedOperation<'_>> {
10025 request.privileged_operation()
10026}
10027
10028trait PrivilegedRequest {
10029 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>>;
10030}
10031
10032impl PrivilegedRequest for ClientControlRequest {
10033 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>> {
10034 let request = self;
10035 match request {
10036 ClientControlRequest::SupervisorRestart { module_id, .. } => {
10037 Some(PrivilegedOperation::Restart { module_id })
10038 }
10039 ClientControlRequest::SupervisorSwap { module_id, .. } => {
10040 Some(PrivilegedOperation::Swap { module_id })
10041 }
10042 ClientControlRequest::SupervisorReload { module_id } => {
10043 Some(PrivilegedOperation::Reload { module_id })
10044 }
10045 ClientControlRequest::SupervisorRescan { preview } => {
10046 if *preview {
10047 None
10048 } else {
10049 Some(PrivilegedOperation::Rescan)
10050 }
10051 }
10052 ClientControlRequest::SupervisorRescanChecked {
10057 preview,
10058 candidate,
10059 expect,
10060 } => (!*preview && candidate.is_none() && expect.is_some())
10061 .then_some(PrivilegedOperation::Rescan),
10062 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
10063 Some(PrivilegedOperation::SetEnabled {
10064 module_id,
10065 enabled: *enabled,
10066 })
10067 }
10068 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
10069 Some(PrivilegedOperation::ReleaseReserved { module_id })
10070 }
10071 ClientControlRequest::ServerDescribe {}
10072 | ClientControlRequest::DaemonShutdownIntent { .. }
10075 | ClientControlRequest::CatalogList { .. }
10076 | ClientControlRequest::RouteOpen { .. }
10077 | ClientControlRequest::RoutePoll { .. }
10078 | ClientControlRequest::SupervisorList {}
10079 | ClientControlRequest::SupervisorHealth {}
10080 | ClientControlRequest::SupervisorHealthProbe { .. }
10081 | ClientControlRequest::SupervisorRoutes { .. }
10082 | ClientControlRequest::SupervisorProvenance { .. }
10083 | ClientControlRequest::SupervisorStderrTail { .. }
10084 | ClientControlRequest::SupervisorTerminals { .. }
10085 | ClientControlRequest::SupervisorSpawnSnapshot {}
10086 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
10087 }
10088 }
10089}
10090
10091fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
10092 match request {
10093 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
10094 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
10095 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
10096 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
10097 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
10098 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
10099 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
10100 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
10101 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
10102 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
10103 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
10104 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
10105 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
10106 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
10107 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
10108 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
10109 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
10110 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
10111 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
10112 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
10113 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
10114 }
10115}
10116
10117struct RouteOpenCaller {
10125 principal: String,
10129 scope: String,
10132 harness: String,
10135}
10136
10137impl RouteOpenCaller {
10138 fn new(
10139 principal: Option<&Principal>,
10140 scope: Option<&ScopeSelector>,
10141 identity: &BindIdentity,
10142 ) -> Self {
10143 Self {
10144 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
10145 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
10146 harness: bounded_log_text(&identity.harness),
10147 }
10148 }
10149
10150 fn unknown() -> Self {
10153 Self {
10154 principal: "unknown".to_string(),
10155 scope: "unknown".to_string(),
10156 harness: "unknown".to_string(),
10157 }
10158 }
10159}
10160
10161const MAX_LOGGED_TEXT_CHARS: usize = 64;
10164
10165fn bounded_log_text(value: &str) -> String {
10169 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
10170 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
10171 bounded.push('…');
10172 }
10173 format!("{bounded:?}")
10174}
10175
10176fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
10182 use subc_protocol::scope::ScopeKind;
10183 let kind = match stamp.kind {
10184 ScopeKind::Head => "head",
10185 ScopeKind::Worker => "worker",
10186 ScopeKind::Ephemeral => "ephemeral",
10187 ScopeKind::Federated => "federated",
10188 _ => "unknown",
10189 };
10190 format!(
10191 "{kind}/{}/{}@{}",
10192 principal_label(&stamp.owner),
10193 bounded_log_text(&stamp.scope_ref),
10194 stamp.scope_epoch
10195 )
10196}
10197
10198fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
10205 let epoch = selector
10206 .scope_epoch
10207 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
10208 format!(
10209 "requested/{}/{}@{epoch}",
10210 bounded_log_text(&principal_label(&selector.owner)),
10211 bounded_log_text(&selector.scope_ref),
10212 )
10213}
10214
10215fn principal_label(principal: &Principal) -> String {
10216 match principal {
10217 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
10218 Principal::Direct => "direct".to_string(),
10219 other => format!("{other:?}"),
10220 }
10221}
10222
10223fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
10224 match request {
10225 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
10226 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
10227 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
10228 }
10229 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
10230 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
10231 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
10232 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
10233 ModuleControlRequestFromModule::RouteAuthorize { .. } => {
10234 subc_protocol::scope::ROUTE_AUTHORIZE_OP
10235 }
10236 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
10237 _ => "unsupported",
10238 }
10239}
10240
10241fn parse_client_control_request(
10242 body: &[u8],
10243) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
10244 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
10245 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10246 Ok(probe)
10247 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
10248 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
10249 {
10250 ControlRequestBodyError::InvalidBody
10251 }
10252 Ok(_) => ControlRequestBodyError::UnknownOp,
10253 Err(_) => ControlRequestBodyError::InvalidBody,
10254 };
10255 (err, classification)
10256 })
10257}
10258
10259fn parse_module_control_request_from_module(
10260 body: &[u8],
10261) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
10262 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
10263 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10264 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
10265 Ok(_) => ControlRequestBodyError::UnknownOp,
10266 Err(_) => ControlRequestBodyError::InvalidBody,
10267 };
10268 (err, classification)
10269 })
10270}
10271
10272#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
10273enum ProviderRoleKind {
10274 ToolProvider,
10275 PipelineStage,
10276 ManagementSurface,
10277 InternalService,
10278}
10279
10280fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
10281 match role {
10282 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
10283 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
10284 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
10285 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
10286 }
10287}
10288
10289fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
10290 roles.iter().map(provider_role_kind).collect()
10291}
10292
10293const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
10296
10297#[derive(Debug, Default, PartialEq, Eq)]
10299struct ScopeOutcomeCounts {
10300 created: usize,
10301 replaced: usize,
10302 updated: usize,
10303 unchanged: usize,
10304 refused: usize,
10305}
10306
10307impl ScopeOutcomeCounts {
10308 fn of(results: &[ScopeRecordResult]) -> Self {
10309 let mut counts = Self::default();
10310 for result in results {
10311 let slot = match result.outcome {
10312 ScopeRecordOutcome::Created => &mut counts.created,
10313 ScopeRecordOutcome::Replaced => &mut counts.replaced,
10314 ScopeRecordOutcome::Updated => &mut counts.updated,
10315 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
10316 ScopeRecordOutcome::Refused => &mut counts.refused,
10317 };
10318 *slot += 1;
10319 }
10320 counts
10321 }
10322}
10323
10324#[cfg(test)]
10325mod scope_outcome_count_tests {
10326 use super::*;
10327
10328 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
10329 ScopeRecordResult {
10330 scope_ref: "r".to_string(),
10331 scope_epoch: 1,
10332 outcome,
10333 code: None,
10334 message: None,
10335 version: None,
10336 parent_state: None,
10337 }
10338 }
10339
10340 #[test]
10343 fn every_outcome_is_counted_in_its_own_field() {
10344 let results = [
10345 result(ScopeRecordOutcome::Created),
10346 result(ScopeRecordOutcome::Created),
10347 result(ScopeRecordOutcome::Replaced),
10348 result(ScopeRecordOutcome::Updated),
10349 result(ScopeRecordOutcome::Unchanged),
10350 result(ScopeRecordOutcome::Refused),
10351 result(ScopeRecordOutcome::Refused),
10352 result(ScopeRecordOutcome::Refused),
10353 ];
10354 assert_eq!(
10355 ScopeOutcomeCounts::of(&results),
10356 ScopeOutcomeCounts {
10357 created: 2,
10358 replaced: 1,
10359 updated: 1,
10360 unchanged: 1,
10361 refused: 3,
10362 }
10363 );
10364 }
10365}
10366
10367fn capability_census_trigger(
10371 old: Option<&CapabilityDeclarations>,
10372 new: Option<&CapabilityDeclarations>,
10373) -> bool {
10374 let old_provides = old
10375 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
10376 .unwrap_or_default();
10377 let old_denies = old
10378 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
10379 .unwrap_or_default();
10380 let new = new.cloned().unwrap_or(CapabilityDeclarations {
10381 provides: Vec::new(),
10382 requires: Vec::new(),
10383 must_never_reach: Vec::new(),
10384 });
10385
10386 new.provides
10387 .iter()
10388 .any(|capability| !old_provides.contains(capability))
10389 || new
10390 .must_never_reach
10391 .iter()
10392 .any(|capability| !old_denies.contains(capability))
10393}
10394
10395fn catalog_update_frozen_field_message(
10396 registered: &ModuleManifest,
10397 provides: &[ProviderRole],
10398) -> Option<String> {
10399 let old_has_provides = !registered.provides.is_empty();
10400 let new_has_provides = !provides.is_empty();
10401 if old_has_provides != new_has_provides {
10402 return Some(format!(
10403 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
10404 registered.module_id
10405 ));
10406 }
10407
10408 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
10409 return Some(format!(
10410 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
10411 registered.module_id
10412 ));
10413 }
10414
10415 let registered_concurrency = manifest_concurrency(registered);
10416 let mut candidate = registered.clone();
10417 candidate.provides = provides.to_vec();
10418 let candidate_concurrency = manifest_concurrency(&candidate);
10419 if candidate_concurrency != registered_concurrency {
10420 return Some(format!(
10421 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
10422 registered.module_id, registered_concurrency, candidate_concurrency
10423 ));
10424 }
10425
10426 None
10429}
10430
10431fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
10432 manifest.provides.iter().any(is_routable_role)
10433}
10434
10435fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
10441 manifest
10442 .provides
10443 .iter()
10444 .find_map(|provider| match provider {
10445 ProviderRole::ToolProvider { concurrency, .. }
10446 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
10447 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
10448 })
10449 .unwrap_or(Concurrency::ModuleManaged)
10450}
10451
10452fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
10459 let has_management_surface = manifest
10460 .provides
10461 .iter()
10462 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
10463 if !has_management_surface {
10464 return false;
10465 }
10466 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
10467 return false;
10468 };
10469 let Some(provides) = raw
10470 .get("manifest")
10471 .and_then(|manifest| manifest.get("provides"))
10472 .and_then(serde_json::Value::as_array)
10473 else {
10474 return false;
10475 };
10476 provides.iter().any(|role| {
10481 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
10482 && role.get("concurrency").is_none()
10483 })
10484}
10485
10486fn negotiate_version(peer_version: u8) -> Result<u8, String> {
10487 if peer_version != PROTOCOL_VERSION {
10488 return Err(format!(
10489 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
10490 ));
10491 }
10492 Ok(PROTOCOL_VERSION)
10493}
10494
10495fn pong(frame: &Frame) -> Result<Frame, RouterError> {
10496 Frame::build_with_version(
10497 response_version(frame),
10498 FrameType::Pong,
10499 frame.header.flags,
10500 0,
10501 0,
10502 frame.header.corr,
10503 Vec::new(),
10504 )
10505 .map_err(RouterError::FrameBuild)
10506}
10507
10508pub(crate) fn control_error_frame(
10509 frame: &Frame,
10510 code: &'static str,
10511 message: impl Into<String>,
10512) -> Result<Frame, RouterError> {
10513 control_error_body_frame(
10514 frame,
10515 ErrorBody {
10516 code: code.to_string(),
10517 message: message.into(),
10518 detail: None,
10519 },
10520 )
10521}
10522
10523pub(crate) fn control_error_body_frame(
10531 frame: &Frame,
10532 error: ErrorBody,
10533) -> Result<Frame, RouterError> {
10534 control_error_frame_with_flags(frame, error, daemon_error_flags())
10535}
10536
10537fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
10543 control_error_frame_with_flags(frame, error, control_flags())
10544}
10545
10546fn control_error_frame_with_flags(
10547 frame: &Frame,
10548 error: ErrorBody,
10549 flags: Flags,
10550) -> Result<Frame, RouterError> {
10551 let body = serde_json::to_vec(&error).map_err(|err| {
10552 RouterError::backend(
10553 0,
10554 frame.header.corr,
10555 format!("failed to encode control ERROR: {err}"),
10556 )
10557 })?;
10558
10559 Frame::build_with_version(
10560 response_version(frame),
10561 FrameType::Error,
10562 flags,
10563 0,
10564 0,
10565 frame.header.corr,
10566 body,
10567 )
10568 .map_err(RouterError::FrameBuild)
10569}
10570
10571pub(crate) fn control_response_body_frame<T: Serialize>(
10572 frame: &Frame,
10573 reply: &T,
10574 label: &'static str,
10575) -> Result<Frame, RouterError> {
10576 let body = serde_json::to_vec(reply).map_err(|err| {
10577 RouterError::backend(
10578 0,
10579 frame.header.corr,
10580 format!("failed to encode {label}: {err}"),
10581 )
10582 })?;
10583
10584 Frame::build_with_version(
10585 response_version(frame),
10586 FrameType::Response,
10587 control_flags(),
10588 0,
10589 0,
10590 frame.header.corr,
10591 body,
10592 )
10593 .map_err(RouterError::FrameBuild)
10594}
10595
10596struct HelloAuthority {
10629 record: Arc<crate::spawn_authority::SpawnAuthorityRecord>,
10630 requested: ProviderDeclarations,
10631 froze: std::sync::atomic::AtomicBool,
10634}
10635
10636fn forwarding_error_code(err: &ForwardingError) -> &'static str {
10637 match err {
10638 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
10639 ForwardingError::NoModuleConnection => "target_unavailable",
10640 ForwardingError::ModuleReloading { .. } => "module_reloading",
10641 ForwardingError::ClientRouteChannelExhausted { .. }
10642 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
10643 ForwardingError::StaleModuleEndpoint
10644 | ForwardingError::UnknownReservation { .. }
10645 | ForwardingError::ConnectionClosing { .. }
10646 | ForwardingError::ClientEgressClosed { .. }
10647 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
10648 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
10651 ForwardingError::RelayCorrelationExhausted
10652 | ForwardingError::RouteOpenBuild(_)
10653 | ForwardingError::Poisoned => "forwarding_error",
10654 }
10655}
10656
10657fn response_version(frame: &Frame) -> u8 {
10658 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
10659 frame.header.ver
10660 } else {
10661 PROTOCOL_VERSION
10662 }
10663}
10664
10665pub(crate) fn control_flags() -> Flags {
10666 Flags::new(false, Priority::Passive, false)
10667}
10668
10669fn send_goodbye_target_best_effort(
10674 counters: &DaemonCounters,
10675 target: &GoodbyeTarget,
10676 context: &'static str,
10677) {
10678 let Ok(frame) = Frame::build_with_version(
10679 target.negotiated_ver,
10680 FrameType::Goodbye,
10681 control_flags(),
10682 target.channel,
10683 target.epoch,
10684 0,
10685 Vec::new(),
10686 ) else {
10687 return;
10688 };
10689 crate::forwarding::send_module_route_goodbye(
10690 counters,
10691 &target.sink,
10692 frame,
10693 target.module_id.as_deref(),
10694 context,
10695 );
10696}
10697
10698fn route_close_reason_for_opener(
10699 reason: RouteCloseReason,
10700 accepts_close_reasons: Option<&[String]>,
10701) -> RouteCloseReason {
10702 if reason == RouteCloseReason::AuthorityChanged
10703 && !accepts_close_reasons
10704 .is_some_and(|reasons| reasons.iter().any(|reason| reason == "authority_changed"))
10705 {
10706 RouteCloseReason::Reload
10709 } else {
10710 reason
10711 }
10712}
10713
10714pub(crate) fn send_route_control_pushes(
10715 forwarding: &ForwardingTable,
10716 routes: Vec<EndpointRoute>,
10717 push: ClientControlPush,
10718) {
10719 let reason = match &push {
10720 ClientControlPush::RouteClosing { reason, .. }
10721 | ClientControlPush::RouteClosed { reason, .. } => *reason,
10722 };
10723 let mut targets: Vec<(GoodbyeTarget, RouteCloseReason, Vec<u16>)> = Vec::new();
10724 for route in routes {
10725 let recipient_reason =
10726 route_close_reason_for_opener(reason, route.accepts_close_reasons.as_deref());
10727 let target = route.goodbye_target;
10728 if let Some((existing, _, channels)) = targets.iter_mut().find(|(existing, reason, _)| {
10729 existing.connection_id == target.connection_id && *reason == recipient_reason
10730 }) {
10731 debug_assert_eq!(
10732 existing.negotiated_ver, target.negotiated_ver,
10733 "one connection cannot negotiate multiple frame versions"
10734 );
10735 if !channels.contains(&target.channel) {
10736 channels.push(target.channel);
10737 }
10738 continue;
10739 }
10740 let channel = target.channel;
10741 targets.push((target, recipient_reason, vec![channel]));
10742 }
10743 for (target, recipient_reason, mut channels) in targets {
10744 channels.sort_unstable();
10745 let mut push = push.clone();
10746 match &mut push {
10747 ClientControlPush::RouteClosing {
10748 channels: covered,
10749 reason,
10750 ..
10751 }
10752 | ClientControlPush::RouteClosed {
10753 channels: covered,
10754 reason,
10755 ..
10756 } => {
10757 *covered = channels;
10758 *reason = recipient_reason;
10759 }
10760 }
10761 let body = match serde_json::to_vec(&push) {
10762 Ok(body) => body,
10763 Err(err) => {
10764 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
10765 continue;
10766 }
10767 };
10768 let frame = match Frame::build_with_version(
10769 target.negotiated_ver,
10770 FrameType::Push,
10771 control_flags(),
10772 0,
10773 0,
10774 0,
10775 body.clone(),
10776 ) {
10777 Ok(frame) => frame,
10778 Err(err) => {
10779 warn!(
10780 route_channel = target.channel,
10781 error = %err,
10782 "failed to build route lifecycle control PUSH frame"
10783 );
10784 continue;
10785 }
10786 };
10787 if let Err(err) = target.sink.try_send(frame) {
10788 if target.close_on_delivery_failure() {
10789 warn!(
10790 target_connection_id = target.connection_id.get(),
10791 route_channel = target.channel,
10792 error = %err,
10793 "route lifecycle control PUSH was not delivered to client; closing target connection"
10794 );
10795 let _ = forwarding.escalate_client_delivery_failure(
10796 target.connection_id,
10797 target.channel,
10798 target.epoch,
10799 CloseReason::new(
10800 "route_lifecycle_push_delivery_failed",
10801 format!(
10802 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
10803 target.channel
10804 ),
10805 ),
10806 crate::forwarding::UndeliveredFrame {
10807 module_id: target.module_id.as_deref(),
10808 sink: &target.sink,
10809 },
10810 );
10811 }
10812 }
10813 }
10814}
10815
10816#[cfg(test)]
10817mod tests {
10818 include!("control/agent_commands_tests.rs");
10819 include!("control/resource_revision_tests.rs");
10820 include!("control/route_open_recovery_tests.rs");
10821 #[tokio::test]
10822 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
10823 let registry = Arc::new(Registry::default());
10824 let handler = ControlHandler::new(registry);
10825 assert_eq!(
10826 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10827 None
10828 );
10829 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
10830 hello_via_sink(
10831 &handler,
10832 &ctx,
10833 &mut rx,
10834 hello_frame("daemon", PROTOCOL_VERSION, 1),
10835 )
10836 .await;
10837 assert_eq!(
10838 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10839 Some("daemon".into())
10840 );
10841 assert_eq!(
10842 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
10843 None
10844 );
10845 assert_eq!(
10846 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
10847 None,
10848 "a registered module is not necessarily supervised"
10849 );
10850 assert_eq!(
10851 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
10852 None,
10853 "aggregate provenance has no single target"
10854 );
10855 assert_eq!(
10856 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
10857 None
10858 );
10859 }
10860
10861 mod peer_manifest {
10862 include!("peer_manifest_tests.rs");
10863 }
10864 mod catalog_declarations {
10865 include!("control/catalog_tests.rs");
10866 }
10867 mod activity_read {
10868 include!("activity_read_tests.rs");
10869 }
10870
10871 mod lifecycle_activity {
10872 include!("control_lifecycle_tests.rs");
10873 }
10874 #[cfg(unix)]
10875 #[tokio::test]
10876 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
10877 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
10878 let path = dir.join("subc.jsonc");
10879 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
10880 "program":"/bin/sleep","args":["60"],"protocol":"none",
10881 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
10882 }}}).to_string()).unwrap();
10883 let mut configured = crate::daemon_config::load(&path)
10884 .unwrap()
10885 .unwrap()
10886 .modules
10887 .pop()
10888 .unwrap();
10889 let registry = std::sync::Arc::new(crate::Registry::default());
10890 let handle = crate::SupervisorHandle::new();
10891 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
10892 .with_handle(handle.clone());
10893 let module = supervisor
10894 .supervise_configured_with_health(
10895 configured.module_spec(),
10896 true,
10897 configured.health.clone(),
10898 None,
10899 configured.restart,
10900 )
10901 .unwrap();
10902 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
10903 let before = module.status().unwrap().pid;
10904 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
10905 configured.health.cadence = std::time::Duration::from_secs(3600);
10906 let health_only = handler
10907 .reconcile_supervised_modules(
10908 &supervisor,
10909 vec![configured.clone()],
10910 false,
10911 Principal::Direct,
10912 )
10913 .await
10914 .unwrap();
10915 assert!(
10916 health_only.0.changed_pending_reload.is_empty(),
10917 "health policy is already applied live"
10918 );
10919 assert_eq!(module.status().unwrap().pid, before);
10920 assert_eq!(
10921 module.configuration().unwrap().1.http,
10922 configured.health.http
10923 );
10924 configured.args = vec!["61".into()];
10925 let launch = handler
10926 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
10927 .await
10928 .unwrap();
10929 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
10930 assert_eq!(
10931 module.status().unwrap().pid,
10932 before,
10933 "a launch edit is stored until reload"
10934 );
10935 module.drain().await.unwrap();
10936 }
10937 use cortexkit_test_support::ScratchDir;
10938 use std::{
10939 collections::BTreeMap,
10940 fmt,
10941 path::PathBuf,
10942 sync::{Arc, Mutex},
10943 time::Duration,
10944 };
10945
10946 use serde_json::{json, Value};
10947 use subc_protocol::{
10948 manifest::{
10949 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
10950 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
10951 },
10952 session::HealthStatus,
10953 FrameType,
10954 };
10955
10956 use super::*;
10957 use crate::{
10958 forwarding::{DataRoute, DataRouteState},
10959 registry::ChannelState,
10960 router::FrameSink,
10961 stderr_tail::DEFAULT_MAX_LINE_BYTES,
10962 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
10963 RouteCtx, Router,
10964 };
10965 use tokio::{
10966 sync::mpsc,
10967 time::{sleep, Instant},
10968 };
10969 use tracing::{
10970 field::{Field, Visit},
10971 Event, Subscriber,
10972 };
10973 use tracing_subscriber::{layer::Context, prelude::*, Layer};
10974
10975 fn fake_aft_stub_path() -> PathBuf {
10994 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
10995 assert!(
10996 path.exists(),
10997 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
10998 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
10999 path.display()
11000 );
11001 path
11002 }
11003
11004 fn client_retries(code: &str) -> bool {
11009 subc_protocol::error_codes::is_retryable_route_open(code)
11010 }
11011
11012 #[test]
11023 fn retryability_of_forwarding_codes_matches_the_failure() {
11024 let transient = [
11027 ForwardingError::NoModuleConnection,
11028 ForwardingError::ModuleReloading {
11029 module_id: "m".into(),
11030 },
11031 ForwardingError::StaleModuleEndpoint,
11032 ForwardingError::UnknownReservation {
11033 client_channel: 1,
11034 module_channel: 1,
11035 },
11036 ForwardingError::ConnectionClosing {
11037 connection_id: ConnectionId::new(1),
11038 },
11039 ForwardingError::ClientEgressClosed {
11040 connection_id: ConnectionId::new(1),
11041 },
11042 ForwardingError::ModuleEgressUnavailable {
11043 connection_id: ConnectionId::new(1),
11044 },
11045 ];
11046 for err in transient {
11047 let code = forwarding_error_code(&err);
11048 assert!(
11049 client_retries(code),
11050 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
11051 );
11052 }
11053
11054 let permanent = [
11059 ForwardingError::ConnectionRoleConflict {
11060 connection_id: ConnectionId::new(1),
11061 },
11062 ForwardingError::ClientRouteChannelExhausted {
11063 connection_id: ConnectionId::new(1),
11064 },
11065 ForwardingError::ModuleRouteChannelExhausted {
11066 endpoint: ModuleEndpointId {
11067 connection_id: ConnectionId::new(1),
11068 generation: 1,
11069 },
11070 },
11071 ForwardingError::RelayCorrelationExhausted,
11072 ForwardingError::RouteOpenBuild("x".into()),
11073 ForwardingError::Poisoned,
11074 ];
11075 for err in permanent {
11076 let code = forwarding_error_code(&err);
11077 assert!(
11078 !client_retries(code),
11079 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
11080 );
11081 }
11082 }
11083
11084 #[tokio::test]
11112 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
11113 let handler = ControlHandler::default();
11114 let frame =
11115 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
11116
11117 let stamped = handler
11121 .route_open_principal(ConnectionId::new(1), &frame, None)
11122 .unwrap()
11123 .unwrap();
11124 assert_eq!(
11125 stamped,
11126 Principal::Direct,
11127 "a caller that proved nothing must not be stamped as a supervised module"
11128 );
11129
11130 let forged = handler
11135 .route_open_principal(
11136 ConnectionId::new(1),
11137 &frame,
11138 Some(ConsumerIdentity {
11139 module_id: "aft".to_string(),
11140 launch_nonce: "not-a-real-nonce".to_string(),
11141 }),
11142 )
11143 .unwrap();
11144 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
11145 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
11146 }
11147
11148 #[test]
11165 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
11166 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
11167 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
11168 let ClientControlRequest::RouteOpen {
11169 consumer_identity, ..
11170 } = parsed
11171 else {
11172 panic!("route.open body must parse as RouteOpen");
11173 };
11174 assert_eq!(
11175 consumer_identity,
11176 Some(ConsumerIdentity {
11177 module_id: "aft".to_string(),
11178 launch_nonce: "n".to_string(),
11179 }),
11180 "the wire field name must reach the value route_open_principal reads"
11181 );
11182 }
11183
11184 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
11185 ModuleManifest::builder(module_id, "0.1.0")
11186 .protocol_ver(protocol_ver)
11187 .provides(vec![ProviderRole::ToolProvider {
11188 tools: vec![Tool {
11189 name: "read".to_string(),
11190 description: None,
11191 execution_mode: ExecutionMode::Pure,
11192 schema: json!({"type": "object"}),
11193 }],
11194 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
11195 concurrency: Concurrency::ModuleManaged,
11196 emits_push: true,
11197 sub_supervises: true,
11198 }])
11199 .build()
11200 }
11201
11202 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
11203 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
11204 }
11205
11206 fn hello_frame_with_control_ops(
11207 module_id: &str,
11208 protocol_ver: u8,
11209 corr: u64,
11210 control_ops: Option<Vec<String>>,
11211 ) -> Frame {
11212 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
11213 }
11214
11215 fn hello_frame_with_nonce(
11216 module_id: &str,
11217 protocol_ver: u8,
11218 corr: u64,
11219 launch_nonce: Option<&str>,
11220 ) -> Frame {
11221 hello_frame_full(
11222 module_id,
11223 protocol_ver,
11224 corr,
11225 None,
11226 launch_nonce.map(ToOwned::to_owned),
11227 )
11228 }
11229
11230 fn hello_frame_full(
11231 module_id: &str,
11232 protocol_ver: u8,
11233 corr: u64,
11234 control_ops: Option<Vec<String>>,
11235 launch_nonce: Option<String>,
11236 ) -> Frame {
11237 let body = serde_json::to_vec(&ModuleHelloBody {
11238 manifest: manifest(module_id, protocol_ver),
11239 protocol_ver,
11240 control_ops,
11241 launch_nonce,
11242 })
11243 .unwrap();
11244 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11245 }
11246
11247 fn hello_frame_with_rich_ui(
11248 module_id: &str,
11249 protocol_ver: u8,
11250 corr: u64,
11251 launch_nonce: Option<&str>,
11252 rich_ui: Value,
11253 ) -> Frame {
11254 let mut body = serde_json::to_value(ModuleHelloBody {
11255 manifest: manifest(module_id, protocol_ver),
11256 protocol_ver,
11257 control_ops: None,
11258 launch_nonce: launch_nonce.map(str::to_string),
11259 })
11260 .unwrap();
11261 body["manifest"]["rich_ui"] = rich_ui;
11262 Frame::build(
11263 FrameType::Hello,
11264 control_flags(),
11265 0,
11266 0,
11267 corr,
11268 serde_json::to_vec(&body).unwrap(),
11269 )
11270 .unwrap()
11271 }
11272
11273 fn non_routable_hello_frame_with_control_ops(
11274 module_id: &str,
11275 corr: u64,
11276 control_ops: Option<Vec<String>>,
11277 ) -> Frame {
11278 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11279 manifest.provides.clear();
11280 let body = serde_json::to_vec(&ModuleHelloBody {
11281 manifest,
11282 protocol_ver: PROTOCOL_VERSION,
11283 control_ops,
11284 launch_nonce: None,
11285 })
11286 .unwrap();
11287 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11288 }
11289
11290 fn capability_grammar_hello_frame(
11291 capabilities: Value,
11292 runtime_computed: Option<Value>,
11293 corr: u64,
11294 ) -> Frame {
11295 let mut body = serde_json::to_value(ModuleHelloBody {
11296 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
11297 protocol_ver: PROTOCOL_VERSION,
11298 control_ops: None,
11299 launch_nonce: None,
11300 })
11301 .expect("HELLO body serializes");
11302 body["manifest"]["capabilities"] = capabilities;
11303 if let Some(runtime_computed) = runtime_computed {
11304 body["runtime_computed"] = runtime_computed;
11305 }
11306 Frame::build(
11307 FrameType::Hello,
11308 control_flags(),
11309 0,
11310 0,
11311 corr,
11312 serde_json::to_vec(&body).expect("HELLO body reserializes"),
11313 )
11314 .expect("HELLO frame builds")
11315 }
11316
11317 fn channel_request(channel: u16, corr: u64) -> Frame {
11318 Frame::build(
11319 FrameType::Request,
11320 Flags::new(true, Priority::Interactive, false),
11321 channel,
11322 0,
11323 corr,
11324 b"opaque".to_vec(),
11325 )
11326 .unwrap()
11327 }
11328
11329 fn route_ctx(
11330 connection_id: ConnectionId,
11331 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11332 let (tx, rx) = mpsc::channel(8);
11333 (
11334 RouteCtx {
11335 connection_id,
11336 egress: FrameSink::new(tx),
11337 },
11338 rx,
11339 )
11340 }
11341
11342 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
11343 serde_json::from_slice(&frame.body).unwrap()
11344 }
11345
11346 async fn hello_via_sink(
11351 handler: &ControlHandler,
11352 ctx: &RouteCtx,
11353 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11354 hello: Frame,
11355 ) -> Frame {
11356 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
11357 assert!(
11358 replies.is_empty(),
11359 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
11360 );
11361 let ack = rx
11362 .try_recv()
11363 .expect("HELLO_ACK is queued on the module sink")
11364 .frame;
11365 assert_eq!(ack.header.ty, FrameType::HelloAck);
11366 ack
11367 }
11368
11369 fn parse_error(frame: &Frame) -> Value {
11370 serde_json::from_slice(&frame.body).unwrap()
11371 }
11372
11373 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
11374 serde_json::from_slice(&frame.body).unwrap()
11375 }
11376
11377 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
11378 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
11379 route_channel,
11380 route_epoch: 0,
11381 kind,
11382 })
11383 .unwrap();
11384 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11385 }
11386
11387 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
11388 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
11389 module_id: module_id.to_string(),
11390 })
11391 .unwrap();
11392 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11393 }
11394
11395 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
11396 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
11397 }
11398
11399 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
11401 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
11402 let mut frame = route_open_frame(corr, module_id, root);
11403 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
11404 body["identity"]["project_root"] = serde_json::json!(path.as_path());
11405 frame.body = serde_json::to_vec(&body).unwrap();
11406 frame
11407 }
11408
11409 fn route_open_frame_with_consumer_capabilities(
11410 corr: u64,
11411 module_id: &str,
11412 _project_root: ScratchDir,
11413 consumer_capabilities: Option<Vec<String>>,
11414 ) -> Frame {
11415 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11416 busy_refusals: None,
11417 target: RouteTarget::ToolProvider {
11418 module_id: module_id.to_string(),
11419 },
11420 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
11421 consumer_identity: None,
11422 consumer_capabilities,
11423 role_versions: None,
11424 admission_facts: None,
11425 scope: None,
11426 })
11427 .unwrap();
11428 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11429 }
11430
11431 fn route_open_frame_with_role_versions(
11432 corr: u64,
11433 module_id: &str,
11434 _project_root: ScratchDir,
11435 role_versions: Option<BTreeMap<String, String>>,
11436 ) -> Frame {
11437 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11438 busy_refusals: None,
11439 target: RouteTarget::ToolProvider {
11440 module_id: module_id.to_string(),
11441 },
11442 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11443 consumer_identity: None,
11444 consumer_capabilities: None,
11445 role_versions,
11446 admission_facts: None,
11447 scope: None,
11448 })
11449 .unwrap();
11450 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11451 }
11452
11453 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
11454 entries
11455 .iter()
11456 .map(|(role, version)| (role.to_string(), version.to_string()))
11457 .collect()
11458 }
11459
11460 fn route_open_frame_with_admission_facts(
11461 corr: u64,
11462 module_id: &str,
11463 _project_root: ScratchDir,
11464 consumer_identity: Option<subc_control::ConsumerIdentity>,
11465 facts: Option<Value>,
11466 ) -> Frame {
11467 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11468 busy_refusals: None,
11469 target: RouteTarget::ToolProvider {
11470 module_id: module_id.to_string(),
11471 },
11472 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11473 consumer_identity,
11474 consumer_capabilities: None,
11475 role_versions: None,
11476 admission_facts: facts,
11477 scope: None,
11478 })
11479 .unwrap();
11480 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11481 }
11482
11483 #[derive(Clone, Default)]
11484 struct EventCapture {
11485 events: Arc<Mutex<Vec<CapturedEvent>>>,
11486 }
11487
11488 #[derive(Clone, Debug)]
11489 struct CapturedEvent {
11490 target: String,
11491 level: tracing::Level,
11492 fields: BTreeMap<String, String>,
11493 }
11494
11495 impl EventCapture {
11496 fn events(&self) -> Vec<CapturedEvent> {
11497 self.events.lock().unwrap().clone()
11498 }
11499 }
11500
11501 impl<S> Layer<S> for EventCapture
11502 where
11503 S: Subscriber,
11504 {
11505 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
11506 let mut visitor = EventFieldVisitor::default();
11507 event.record(&mut visitor);
11508 self.events.lock().unwrap().push(CapturedEvent {
11509 target: event.metadata().target().to_string(),
11510 level: *event.metadata().level(),
11511 fields: visitor.fields,
11512 });
11513 }
11514 }
11515
11516 #[derive(Default)]
11517 struct EventFieldVisitor {
11518 fields: BTreeMap<String, String>,
11519 }
11520
11521 impl Visit for EventFieldVisitor {
11522 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
11523 self.fields
11524 .insert(field.name().to_string(), format!("{value:?}"));
11525 }
11526 }
11527
11528 fn health_response(corr: u64, status: HealthStatus) -> Frame {
11529 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
11530 status,
11531 detail: Some("warming".to_string()),
11532 metrics: Some(json!({"queue_depth": 3})),
11533 })
11534 .unwrap();
11535 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11536 }
11537
11538 fn route_bind_ack(corr: u64) -> Frame {
11539 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
11540 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11541 }
11542
11543 fn unique_project_root(label: &str) -> ScratchDir {
11544 ScratchDir::new(label)
11545 }
11546
11547 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
11548 match parse_route_poll(frame) {
11549 ClientControlResponse::RoutePoll {
11550 status: None,
11551 live: Some(live),
11552 ..
11553 } => assert_eq!(live, expected_live),
11554 other => panic!("unexpected route.poll response: {other:?}"),
11555 }
11556 }
11557
11558 fn bind_liveness_route(
11559 registry: &Registry,
11560 forwarding: &ForwardingTable,
11561 module_id: &str,
11562 ) -> (RouteCtx, u16, u32) {
11563 let module_connection = ConnectionId::new(101);
11564 let client_connection = ConnectionId::new(202);
11565 let registration = registry
11566 .register_with_control_ops(
11567 manifest(module_id, PROTOCOL_VERSION),
11568 PROTOCOL_VERSION,
11569 module_connection,
11570 module_baseline_control_ops(),
11571 )
11572 .unwrap();
11573 let (module_tx, _module_rx) = mpsc::channel(8);
11574 let endpoint = forwarding
11575 .register_module_connection(
11576 module_connection,
11577 module_id.to_string(),
11578 PROTOCOL_VERSION,
11579 manifest_concurrency(®istration.manifest),
11580 FrameSink::new(module_tx),
11581 )
11582 .unwrap();
11583 let (client_ctx, _client_rx) = route_ctx(client_connection);
11584 let pending = forwarding
11585 .begin_route_bind_relay_for_test(
11586 client_connection,
11587 client_ctx.egress.clone(),
11588 1,
11589 module_id,
11590 )
11591 .unwrap();
11592 assert_eq!(pending.endpoint, endpoint);
11593 let route_channel = pending.client_channel;
11594 let route_epoch = pending.client_epoch;
11595 forwarding
11596 .complete_pending_relay(
11597 module_connection,
11598 pending.corr,
11599 RouteBindRelayOutcome::Accepted,
11600 )
11601 .unwrap();
11602 (client_ctx, route_channel, route_epoch)
11603 }
11604
11605 struct FakeProcessLiveness {
11606 live: Option<bool>,
11607 }
11608
11609 impl ModuleProcessLiveness for FakeProcessLiveness {
11610 fn process_live(&self, _module_id: &str) -> Option<bool> {
11611 self.live
11612 }
11613 }
11614
11615 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11616 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
11617 {
11618 let registry = Arc::new(Registry::default());
11619 let supervisor_handle = SupervisorHandle::new();
11620 let supervisor = Supervisor::new_for_test(
11621 Arc::clone(®istry),
11622 RestartPolicy::new(1, Duration::from_millis(10)),
11623 )
11624 .with_handle(supervisor_handle.clone());
11625 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
11626 let module = supervisor
11627 .spawn(ModuleSpec {
11628 script: None,
11629 carry_containment: false,
11630 module_id: "stderr-tail-wire".to_string(),
11631 program: fake_aft_stub_path(),
11632 args: Vec::new(),
11633 env: vec![
11634 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
11635 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
11636 ],
11637 reserved: false,
11638 reserved_prefixes: Vec::new(),
11639 protocol: ModuleProtocol::Subc,
11640 overlap: Default::default(),
11641 })
11642 .unwrap();
11643
11644 let deadline = Instant::now() + Duration::from_secs(5);
11645 loop {
11646 let tail = module.stderr_tail(None, None);
11647 if tail
11648 .entries
11649 .iter()
11650 .any(|entry| matches!(entry, TailEntry::ProcessStart))
11651 && tail.entries.iter().any(|entry| {
11652 matches!(
11653 entry,
11654 TailEntry::Line {
11655 truncated: true,
11656 ..
11657 }
11658 )
11659 })
11660 {
11661 break;
11662 }
11663 assert!(
11664 Instant::now() < deadline,
11665 "module did not produce a truncated line and restart boundary: {tail:?}"
11666 );
11667 sleep(Duration::from_millis(10)).await;
11668 }
11669
11670 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11671 let request = ClientControlRequest::SupervisorStderrTail {
11672 module_id: "stderr-tail-wire".to_string(),
11673 max_lines: None,
11674 max_bytes: None,
11675 };
11676 let frame = Frame::build(
11677 FrameType::Request,
11678 control_flags(),
11679 0,
11680 0,
11681 1,
11682 serde_json::to_vec(&request).unwrap(),
11683 )
11684 .unwrap();
11685 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11686 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11687 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
11688 serde_json::from_slice(&responses[0].body).unwrap()
11689 else {
11690 panic!("expected supervisor.stderr_tail response");
11691 };
11692
11693 assert!(
11694 tail.entries
11695 .iter()
11696 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
11697 "the control response lost the restart boundary"
11698 );
11699 let Some(StderrTailEntry::Line {
11700 text,
11701 truncated,
11702 at_ms,
11703 }) = tail.entries.iter().find(|entry| {
11704 matches!(
11705 entry,
11706 StderrTailEntry::Line {
11707 truncated: true,
11708 ..
11709 }
11710 )
11711 })
11712 else {
11713 panic!("the control response lost the truncated line");
11714 };
11715 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
11716 assert!(*truncated);
11717 assert!(
11718 at_ms.is_some(),
11719 "the control response lost the line's capture time"
11720 );
11721 }
11722
11723 #[tokio::test(flavor = "current_thread")]
11728 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
11729 let dir = ScratchDir::new("terminals-off-worker");
11730 let journal_path = dir.join("terminals.jsonl");
11731 let registry = Arc::new(Registry::default());
11732 let supervisor_handle = SupervisorHandle::new();
11733 let supervisor =
11734 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11735 .with_handle(supervisor_handle.clone())
11736 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
11737 let module = supervisor
11738 .spawn(ModuleSpec {
11739 script: None,
11740 carry_containment: false,
11741 module_id: "terminal-off-worker".to_string(),
11742 program: fake_aft_stub_path(),
11743 args: Vec::new(),
11744 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11745 reserved: false,
11746 reserved_prefixes: Vec::new(),
11747 protocol: ModuleProtocol::Subc,
11748 overlap: Default::default(),
11749 })
11750 .unwrap();
11751 let deadline = Instant::now() + Duration::from_secs(5);
11752 while module.terminal_history().entries.len() != 2 {
11753 assert!(Instant::now() < deadline, "module did not record two exits");
11754 sleep(Duration::from_millis(10)).await;
11755 }
11756
11757 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
11758 let handler =
11759 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
11760 let frame = Frame::build(
11761 FrameType::Request,
11762 control_flags(),
11763 0,
11764 0,
11765 1,
11766 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
11767 module_id: "terminal-off-worker".to_string(),
11768 })
11769 .unwrap(),
11770 )
11771 .unwrap();
11772 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11773 let spawned_at = std::time::Instant::now();
11774 let read = tokio::spawn({
11775 let handler = Arc::clone(&handler);
11776 async move { handler.handle_control_frame(&ctx, frame).await }
11777 });
11778 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
11781 .await
11782 .unwrap()
11783 .expect("the history read reached its pause");
11784 let elapsed = spawned_at.elapsed();
11785 assert!(
11786 elapsed < Duration::from_secs(2) && !read.is_finished(),
11787 "this task could not run while the history read was paused \
11788 (resumed after {elapsed:?}, read finished: {})",
11789 read.is_finished()
11790 );
11791
11792 drop(release);
11793 let responses = read.await.unwrap().unwrap();
11794 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11795 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
11796 panic!("expected supervisor.terminals response");
11797 };
11798 assert_eq!(terminals.entries.len(), 2);
11799 assert_eq!(terminals.journal_skipped_lines, 0);
11800 assert_eq!(terminals.journal_read_errors, 0);
11801 }
11802
11803 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11804 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
11805 let registry = Arc::new(Registry::default());
11806 let supervisor_handle = SupervisorHandle::new();
11807 let supervisor =
11808 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11809 .with_handle(supervisor_handle.clone());
11810 let module = supervisor
11811 .spawn(ModuleSpec {
11812 script: None,
11813 carry_containment: false,
11814 module_id: "terminal-golden".to_string(),
11815 program: fake_aft_stub_path(),
11816 args: Vec::new(),
11817 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11818 reserved: false,
11819 reserved_prefixes: Vec::new(),
11820 protocol: ModuleProtocol::Subc,
11821 overlap: Default::default(),
11822 })
11823 .unwrap();
11824
11825 let deadline = Instant::now() + Duration::from_secs(5);
11826 while module.terminal_history().entries.len() != 2 {
11827 assert!(
11828 Instant::now() < deadline,
11829 "module did not retain two terminal exits: {:?}",
11830 module.terminal_history()
11831 );
11832 sleep(Duration::from_millis(10)).await;
11833 }
11834
11835 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11836 let request = ClientControlRequest::SupervisorTerminals {
11837 module_id: "terminal-golden".to_string(),
11838 };
11839 let frame = Frame::build(
11840 FrameType::Request,
11841 control_flags(),
11842 0,
11843 0,
11844 1,
11845 serde_json::to_vec(&request).unwrap(),
11846 )
11847 .unwrap();
11848 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11849 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11850 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11851 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
11852 panic!("expected supervisor.terminals response");
11853 };
11854 assert_eq!(terminals.entries.len(), 2);
11855 assert_eq!(terminals.dropped, 0);
11856
11857 let mut rendered = serde_json::to_value(response).unwrap();
11858 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
11861 for (index, entry) in rendered["entries"]
11862 .as_array_mut()
11863 .expect("terminal response entries array")
11864 .iter_mut()
11865 .enumerate()
11866 {
11867 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
11868 }
11869
11870 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11871 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
11872 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
11873 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11874 std::fs::write(&golden_path, &serialized).unwrap();
11875 }
11876 let expected: Value =
11877 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
11878 assert_eq!(rendered, expected);
11879 }
11880
11881 #[test]
11882 fn hello_registers_manifest_and_returns_ack() {
11883 let registry = Arc::new(Registry::default());
11884 let handler = ControlHandler::new(Arc::clone(®istry));
11885 let conn = ConnectionId::new(1);
11886
11887 let responses = handler
11888 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
11889 .unwrap();
11890
11891 assert_eq!(responses.len(), 1);
11892 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11893 assert_eq!(responses[0].header.channel, 0);
11894 assert_eq!(responses[0].header.corr, 7);
11895 let ack = parse_ack(&responses[0]);
11896 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11897 assert!(ack
11898 .subc_capabilities
11899 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
11900 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
11901 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
11902 assert!(ack
11903 .subc_ops
11904 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
11905 assert!(ack
11906 .subc_ops
11907 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
11908 assert!(ack
11909 .subc_ops
11910 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
11911
11912 let registration = registry.get_module("aft").unwrap().unwrap();
11913 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
11914 assert_eq!(registration.state, ChannelState::Active);
11915 assert_eq!(registration.connection_id, conn);
11916 assert_eq!(registration.control_ops, module_baseline_control_ops());
11917 assert!(registration.manifest.display_name.is_none());
11918 }
11919
11920 #[test]
11921 fn hello_display_name_refuses_invalid_manifest_by_field() {
11922 let registry = Arc::new(Registry::default());
11923 let handler = ControlHandler::new(Arc::clone(®istry));
11924 for (index, invalid) in [
11925 json!(""),
11926 json!("a".repeat(41)),
11927 json!(" Routing"),
11928 json!("a\nb"),
11929 json!("a\u{202e}b"),
11930 json!(7),
11931 ]
11932 .into_iter()
11933 .enumerate()
11934 {
11935 let module_id = format!("bad-display-name-{index}");
11936 let mut body = serde_json::to_value(ModuleHelloBody {
11937 manifest: manifest(&module_id, PROTOCOL_VERSION),
11938 protocol_ver: PROTOCOL_VERSION,
11939 control_ops: None,
11940 launch_nonce: None,
11941 })
11942 .unwrap();
11943 body["manifest"]["display_name"] = invalid;
11944 let hello = Frame::build(
11945 FrameType::Hello,
11946 control_flags(),
11947 0,
11948 0,
11949 7,
11950 serde_json::to_vec(&body).unwrap(),
11951 )
11952 .unwrap();
11953 let response = handler
11954 .handle_control(ConnectionId::new(100 + index as u64), hello)
11955 .unwrap();
11956 assert_eq!(response[0].header.ty, FrameType::Error);
11957 let error = parse_error(&response[0]);
11958 assert_eq!(error["code"], "invalid_manifest");
11959 assert!(error["message"].as_str().unwrap().contains("display_name"));
11960 assert!(registry.get_module(&module_id).unwrap().is_none());
11961 }
11962 }
11963
11964 #[test]
11965 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
11966 let invalid_identifiers = [
11967 ("case_change", "credentials-Provider/v1"),
11968 ("leading_zero", "credentials-provider/v01"),
11969 ("trailing_hyphen", "credentials-provider-/v1"),
11970 ("consecutive_hyphens", "credentials--provider/v1"),
11971 ("uppercase", "Credentials-provider/v1"),
11972 ("missing_v", "credentials-provider/1"),
11973 ("whitespace", "credentials provider/v1"),
11974 ("zero_version", "credentials-provider/v0"),
11975 ("out_of_range_version", "credentials-provider/v4294967296"),
11976 (
11977 "overlength_name",
11978 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
11979 ),
11980 ];
11981 let mut cases = invalid_identifiers
11982 .into_iter()
11983 .map(|(name, identifier)| {
11984 (
11985 format!("identifier_{name}"),
11986 "capabilities.provides[0]".to_string(),
11987 identifier.to_string(),
11988 json!({ "provides": [identifier] }),
11989 None,
11990 )
11991 })
11992 .collect::<Vec<_>>();
11993 cases.extend([
11994 (
11995 "unknown_need".to_string(),
11996 "capabilities.requires[0].need".to_string(),
11997 "deferred".to_string(),
11998 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
11999 None,
12000 ),
12001 (
12002 "duplicate_provides".to_string(),
12003 "capabilities.provides[1]".to_string(),
12004 "credentials-provider/v1".to_string(),
12005 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
12006 None,
12007 ),
12008 (
12009 "duplicate_must_never_reach".to_string(),
12010 "capabilities.must_never_reach[1]".to_string(),
12011 "credentials-provider/v1".to_string(),
12012 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
12013 None,
12014 ),
12015 (
12016 "duplicate_requires_same_need".to_string(),
12017 "capabilities.requires[1]".to_string(),
12018 "credentials-provider/v1".to_string(),
12019 json!({ "requires": [
12020 { "capability": "credentials-provider/v1", "need": "required" },
12021 { "capability": "credentials-provider/v1", "need": "required" }
12022 ] }),
12023 None,
12024 ),
12025 (
12026 "duplicate_requires_conflicting_need".to_string(),
12027 "capabilities.requires[1]".to_string(),
12028 "credentials-provider/v1".to_string(),
12029 json!({ "requires": [
12030 { "capability": "credentials-provider/v1", "need": "required" },
12031 { "capability": "credentials-provider/v1", "need": "optional" }
12032 ] }),
12033 None,
12034 ),
12035 (
12036 "capabilities_root_pointer".to_string(),
12037 "runtime_computed[0]".to_string(),
12038 "/capabilities".to_string(),
12039 json!({}),
12040 Some(json!(["/capabilities"])),
12041 ),
12042 (
12043 "capabilities_descendant_pointer".to_string(),
12044 "runtime_computed[0]".to_string(),
12045 "/capabilities/provides".to_string(),
12046 json!({}),
12047 Some(json!(["/capabilities/provides"])),
12048 ),
12049 (
12050 "malformed_pointer_without_leading_slash".to_string(),
12051 "runtime_computed[0]".to_string(),
12052 "capabilities".to_string(),
12053 json!({}),
12054 Some(json!(["capabilities"])),
12055 ),
12056 (
12057 "malformed_pointer_escape".to_string(),
12058 "runtime_computed[0]".to_string(),
12059 "/roles/~2/tools".to_string(),
12060 json!({}),
12061 Some(json!(["/roles/~2/tools"])),
12062 ),
12063 (
12064 "unknown_capabilities_field".to_string(),
12065 "capabilities.future".to_string(),
12066 "<array>".to_string(),
12067 json!({ "future": [] }),
12068 None,
12069 ),
12070 ]);
12071
12072 for (index, (name, field, value, capabilities, runtime_computed)) in
12073 cases.into_iter().enumerate()
12074 {
12075 let registry = Arc::new(Registry::default());
12076 let handler = ControlHandler::new(Arc::clone(®istry));
12077 let response = handler
12078 .handle_control(
12079 ConnectionId::new((index + 1) as u64),
12080 capability_grammar_hello_frame(
12081 capabilities,
12082 runtime_computed,
12083 index as u64 + 1,
12084 ),
12085 )
12086 .expect("invalid HELLO returns a refusal");
12087
12088 assert_eq!(response.len(), 1, "{name} must emit one refusal");
12089 let error = parse_error(&response[0]);
12090 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
12091 let message = error["message"]
12092 .as_str()
12093 .expect("error message is a string");
12094 assert!(
12095 message.contains(&field),
12096 "{name}: field missing from {message}"
12097 );
12098 assert!(
12099 message.contains(&value),
12100 "{name}: value missing from {message}"
12101 );
12102 assert_eq!(
12103 registry
12104 .active_registration_count()
12105 .expect("registry reads"),
12106 0,
12107 "{name}: refused HELLO must not create a catalog entry"
12108 );
12109 }
12110 }
12111
12112 #[test]
12113 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
12114 let registry = Arc::new(Registry::default());
12115 let handler = ControlHandler::new(Arc::clone(®istry));
12116 let capabilities = json!({
12117 "provides": ["credentials-provider/v1"],
12118 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
12119 "must_never_reach": ["federation-transport/v1"]
12120 });
12121 let response = handler
12122 .handle_control(
12123 ConnectionId::new(99),
12124 capability_grammar_hello_frame(
12125 capabilities.clone(),
12126 Some(json!(["/roles/0/tools"])),
12127 99,
12128 ),
12129 )
12130 .expect("valid HELLO registers");
12131 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12132
12133 let request = Frame::build(
12134 FrameType::Request,
12135 control_flags(),
12136 0,
12137 0,
12138 100,
12139 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12140 .expect("catalog request serializes"),
12141 )
12142 .expect("catalog request frame builds");
12143 let response = handler
12144 .handle_catalog_list(request, None)
12145 .expect("catalog list succeeds");
12146 let ClientControlResponse::CatalogList { modules, .. } =
12147 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
12148 else {
12149 panic!("catalog request must return catalog.list");
12150 };
12151 assert_eq!(modules.len(), 1);
12152 assert_eq!(
12153 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
12154 capabilities
12155 );
12156 }
12157
12158 #[test]
12159 fn catalog_list_mirrors_management_operation_description() {
12160 let registry = Arc::new(Registry::default());
12161 let handler = ControlHandler::new(Arc::clone(®istry));
12162 let description = "List managed records and return their identifiers and metadata.";
12163 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
12164 manifest.provides = vec![ProviderRole::ManagementSurface {
12165 operations: vec![ManagementOperation {
12166 name: "records.list".to_string(),
12167 kind: ManagementOperationKind::Query,
12168 description: Some(description.to_string()),
12169 }],
12170 config_schema: json!({"type": "object"}),
12171 observability: vec![ObservabilitySurface {
12172 name: "records.stats".to_string(),
12173 kind: ObservabilityKind::Snapshot,
12174 }],
12175 identity_scope: vec![IdentityScope::Project],
12176 concurrency: Concurrency::ModuleManaged,
12177 }];
12178 registry
12179 .register_with_control_ops(
12180 manifest,
12181 PROTOCOL_VERSION,
12182 ConnectionId::new(99),
12183 Vec::new(),
12184 )
12185 .expect("described management manifest registers");
12186
12187 let request = Frame::build(
12188 FrameType::Request,
12189 control_flags(),
12190 0,
12191 0,
12192 100,
12193 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12194 .expect("catalog request serializes"),
12195 )
12196 .expect("catalog request frame builds");
12197 let response = handler
12198 .handle_catalog_list(request, None)
12199 .expect("catalog list succeeds");
12200 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
12201 assert_eq!(
12202 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
12203 "catalog.list must preserve the declared operation description verbatim"
12204 );
12205 }
12206
12207 #[test]
12208 fn catalog_list_mirrors_registered_operation_delivery() {
12209 let registry = Arc::new(Registry::default());
12210 let handler = ControlHandler::new(Arc::clone(®istry));
12211 let delivery =
12212 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
12213 let mut body = serde_json::to_value(ModuleHelloBody {
12214 manifest: manifest("watch-management", PROTOCOL_VERSION),
12215 protocol_ver: PROTOCOL_VERSION,
12216 control_ops: None,
12217 launch_nonce: None,
12218 })
12219 .unwrap();
12220 body["manifest"]["provides"] = json!([{
12221 "role": "management_surface",
12222 "operations": [
12223 {"name": "status", "kind": "query"},
12224 {"name": "future", "kind": "query"},
12225 {"name": "reset", "kind": "mutate"}
12226 ],
12227 "config_schema": {}, "observability": [], "identity_scope": []
12228 }]);
12229 body["manifest"]["operation_delivery"] = delivery.clone();
12230 let hello = Frame::build(
12231 FrameType::Hello,
12232 control_flags(),
12233 0,
12234 0,
12235 99,
12236 serde_json::to_vec(&body).unwrap(),
12237 )
12238 .unwrap();
12239 let response = handler
12240 .handle_control(ConnectionId::new(99), hello)
12241 .unwrap();
12242 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12243
12244 let (_, registrations) = registry.list_modules().unwrap();
12245 let registered = ®istrations[0].manifest;
12246 assert!(registered.is_watch_operation("status"));
12247 assert!(!registered.is_watch_operation("future"));
12248 assert!(!registered.is_watch_operation("reset"));
12249 assert!(!registered.is_watch_operation("missing"));
12250
12251 let request = Frame::build(
12252 FrameType::Request,
12253 control_flags(),
12254 0,
12255 0,
12256 100,
12257 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12258 )
12259 .unwrap();
12260 let response = handler.handle_catalog_list(request, None).unwrap();
12261 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
12262 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
12263 let ClientControlResponse::CatalogList { modules, .. } =
12264 serde_json::from_slice(&response[0].body).unwrap()
12265 else {
12266 panic!("expected catalog.list");
12267 };
12268 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
12269 }
12270
12271 #[test]
12272 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
12273 let registry = Arc::new(Registry::default());
12274 let supervisor = SupervisorHandle::new();
12275 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12276 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12277 let descriptor = json!({
12278 "schema": "ckios.richui.manifest/1",
12279 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12280 "bytes": 4096,
12281 "chunks": 4
12282 });
12283 let connection = ConnectionId::new(201);
12284 let hello = handler
12285 .handle_control(
12286 connection,
12287 hello_frame_with_rich_ui(
12288 "vault",
12289 PROTOCOL_VERSION,
12290 201,
12291 Some("the-real-nonce"),
12292 descriptor.clone(),
12293 ),
12294 )
12295 .unwrap();
12296 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12297
12298 let request =
12299 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
12300 let provides = registry
12301 .get_module_by_connection(connection)
12302 .unwrap()
12303 .unwrap()
12304 .manifest
12305 .provides;
12306 handler
12307 .handle_catalog_update(connection, request, provides, None, Some(false))
12308 .unwrap();
12309
12310 let request = Frame::build(
12311 FrameType::Request,
12312 control_flags(),
12313 0,
12314 0,
12315 203,
12316 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12317 )
12318 .unwrap();
12319 let response = handler.handle_catalog_list(request, None).unwrap();
12320 let ClientControlResponse::CatalogList { modules, .. } =
12321 serde_json::from_slice(&response[0].body).unwrap()
12322 else {
12323 panic!("expected catalog.list");
12324 };
12325 assert_eq!(
12326 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
12327 descriptor
12328 );
12329 assert!(modules[0].nonce_verified);
12330 }
12331
12332 fn hello_with_role_declarations(declarations: Value) -> Frame {
12333 let mut body = serde_json::to_value(ModuleHelloBody {
12334 manifest: manifest("role-test", PROTOCOL_VERSION),
12335 protocol_ver: PROTOCOL_VERSION,
12336 control_ops: None,
12337 launch_nonce: None,
12338 })
12339 .unwrap();
12340 body["manifest"]["role_declarations"] = declarations;
12341 Frame::build(
12342 FrameType::Hello,
12343 control_flags(),
12344 0,
12345 0,
12346 301,
12347 serde_json::to_vec(&body).unwrap(),
12348 )
12349 .unwrap()
12350 }
12351
12352 fn update_role_declarations_frame(declarations: Value) -> Frame {
12353 Frame::build(
12354 FrameType::Request,
12355 control_flags(),
12356 0,
12357 0,
12358 303,
12359 serde_json::to_vec(&json!({
12360 "op": "catalog.update_role_declarations",
12361 "role_declarations": declarations,
12362 }))
12363 .unwrap(),
12364 )
12365 .unwrap()
12366 }
12367
12368 #[tokio::test]
12369 async fn role_declarations_update_replaces_then_clears_catalog() {
12370 let registry = Arc::new(Registry::default());
12371 let handler = ControlHandler::new(Arc::clone(®istry));
12372 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12373 let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
12374 hello_via_sink(
12375 &handler,
12376 &ctx,
12377 &mut rx,
12378 hello_with_role_declarations(initial.clone()),
12379 )
12380 .await;
12381 assert_eq!(
12382 role_declarations_catalog(&handler).0["role_declarations"],
12383 initial
12384 );
12385 let before = registry.get_module("role-test").unwrap().unwrap();
12386 for declarations in [
12387 json!({"media-provider/v1": {"engines": ["new"]}}),
12388 json!({}),
12389 ] {
12390 let response = handler
12391 .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
12392 .await
12393 .unwrap();
12394 assert_eq!(response[0].header.ty, FrameType::Response);
12395 assert_eq!(
12396 serde_json::from_slice::<Value>(&response[0].body).unwrap(),
12397 json!({"op": "catalog.update_role_declarations"})
12398 );
12399 let (raw, entry) = role_declarations_catalog(&handler);
12400 assert_eq!(
12401 serde_json::to_value(entry.role_declarations).unwrap(),
12402 declarations
12403 );
12404 if declarations.as_object().unwrap().is_empty() {
12405 assert!(raw.get("role_declarations").is_none());
12406 } else {
12407 assert_eq!(raw["role_declarations"], declarations);
12408 }
12409 let after = registry.get_module("role-test").unwrap().unwrap();
12410 let mut expected = before.manifest.clone();
12411 expected.role_declarations = serde_json::from_value(declarations).unwrap();
12412 assert_eq!(after.manifest, expected);
12413 assert_eq!(after.ready, before.ready);
12414 }
12415 }
12416
12417 #[tokio::test]
12418 async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
12419 let registry = Arc::new(Registry::default());
12420 let handler = ControlHandler::new(registry);
12421 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12422 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12423 hello_via_sink(
12424 &handler,
12425 &ctx,
12426 &mut rx,
12427 hello_with_role_declarations(initial.clone()),
12428 )
12429 .await;
12430 for (invalid, reason) in [
12431 (json!(null), "role_declarations must be an object"),
12432 (json!([]), "role_declarations must be an object"),
12433 (
12434 json!({"bad key": {}}),
12435 "role_declarations key must use capability-name grammar",
12436 ),
12437 (
12438 json!({"media-provider/v1": []}),
12439 "role_declarations values must be objects",
12440 ),
12441 (
12442 json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
12443 "role_declarations exceeds 16384 compact JSON bytes",
12444 ),
12445 ] {
12446 let response = handler
12447 .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
12448 .await
12449 .unwrap();
12450 let error = parse_error(&response[0]);
12451 assert_eq!(error["code"], "invalid_role_declarations");
12452 assert_eq!(error["message"], reason);
12453 assert_eq!(
12454 role_declarations_catalog(&handler).0["role_declarations"],
12455 initial
12456 );
12457 }
12458 }
12459
12460 #[tokio::test]
12461 async fn role_declarations_update_is_owned_by_the_registered_connection() {
12462 let registry = Arc::new(Registry::default());
12463 let handler = ControlHandler::new(Arc::clone(®istry));
12464 let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
12465 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12466 hello_via_sink(
12467 &handler,
12468 &owner,
12469 &mut owner_rx,
12470 hello_with_role_declarations(initial.clone()),
12471 )
12472 .await;
12473 let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
12474 let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
12475 let response = handler
12476 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12477 .await
12478 .unwrap();
12479 assert_eq!(parse_error(&response[0])["code"], "not_registered");
12480 hello_via_sink(
12481 &handler,
12482 &other,
12483 &mut other_rx,
12484 hello_frame("other", PROTOCOL_VERSION, 304),
12485 )
12486 .await;
12487 let response = handler
12488 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12489 .await
12490 .unwrap();
12491 assert_eq!(response[0].header.ty, FrameType::Response);
12492 assert_eq!(
12493 serde_json::to_value(
12494 registry
12495 .get_module("role-test")
12496 .unwrap()
12497 .unwrap()
12498 .manifest
12499 .role_declarations
12500 )
12501 .unwrap(),
12502 initial
12503 );
12504 assert_eq!(
12505 serde_json::to_value(
12506 registry
12507 .get_module("other")
12508 .unwrap()
12509 .unwrap()
12510 .manifest
12511 .role_declarations
12512 )
12513 .unwrap(),
12514 replacement
12515 );
12516 }
12517
12518 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
12519 let request = Frame::build(
12520 FrameType::Request,
12521 control_flags(),
12522 0,
12523 0,
12524 302,
12525 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12526 )
12527 .unwrap();
12528 let response = handler.handle_catalog_list(request, None).unwrap();
12529 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
12530 let ClientControlResponse::CatalogList { mut modules, .. } =
12531 serde_json::from_slice(&response[0].body).unwrap()
12532 else {
12533 panic!("expected catalog.list");
12534 };
12535 (raw["modules"][0].clone(), modules.remove(0))
12536 }
12537
12538 #[test]
12539 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
12540 let registry = Arc::new(Registry::default());
12541 let handler = ControlHandler::new(Arc::clone(®istry));
12542 let declarations = json!({
12543 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
12544 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
12545 });
12546 let connection = ConnectionId::new(301);
12547 let response = handler
12548 .handle_control(
12549 connection,
12550 hello_with_role_declarations(declarations.clone()),
12551 )
12552 .unwrap();
12553 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12554 let registered = registry
12555 .get_module_by_connection(connection)
12556 .unwrap()
12557 .unwrap()
12558 .manifest;
12559 assert_eq!(
12563 serde_json::to_value(®istered.role_declarations).unwrap(),
12564 declarations
12565 );
12566 let (raw, entry) = role_declarations_catalog(&handler);
12567 assert_eq!(raw["role_declarations"], declarations);
12568 assert_eq!(entry.role_declarations, registered.role_declarations);
12569
12570 let request =
12571 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
12572 handler
12573 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
12574 .unwrap();
12575 let (raw, entry) = role_declarations_catalog(&handler);
12576 assert_eq!(raw["role_declarations"], declarations);
12577 assert_eq!(entry.role_declarations, registered.role_declarations);
12578 }
12579
12580 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
12581 let registry = Arc::new(Registry::default());
12582 let handler = ControlHandler::new(Arc::clone(®istry));
12583 let capture = EventCapture::default();
12584 let _subscriber =
12585 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12586 let response = handler
12587 .handle_control(
12588 ConnectionId::new(301),
12589 hello_with_role_declarations(declarations),
12590 )
12591 .unwrap();
12592 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12593 assert!(registry
12594 .get_module_by_connection(ConnectionId::new(301))
12595 .unwrap()
12596 .unwrap()
12597 .manifest
12598 .role_declarations
12599 .is_empty());
12600 let (raw, entry) = role_declarations_catalog(&handler);
12601 assert!(raw.get("role_declarations").is_none());
12602 assert!(entry.role_declarations.is_empty());
12603 let warnings: Vec<_> = capture
12604 .events()
12605 .into_iter()
12606 .filter(|event| {
12607 event.fields.get("message").map(String::as_str)
12608 == Some("dropping whole role_declarations map from HELLO")
12609 })
12610 .collect();
12611 assert_eq!(warnings.len(), 1);
12612 assert_eq!(warnings[0].level, tracing::Level::WARN);
12613 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
12614 assert_eq!(warnings[0].fields["reason"], reason);
12615 }
12616
12617 #[test]
12618 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
12619 for value in [
12620 json!(null),
12621 json!(false),
12622 json!(0),
12623 json!("opaque"),
12624 json!([]),
12625 ] {
12626 assert_hello_drops_role_declarations_with_warning(
12627 json!({"routing/v1": {}, "broken/v1": value}),
12628 "role_declarations values must be objects",
12629 );
12630 }
12631 assert_hello_drops_role_declarations_with_warning(
12632 json!([]),
12633 "role_declarations must be an object",
12634 );
12635 }
12636
12637 #[test]
12638 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
12639 assert_hello_drops_role_declarations_with_warning(
12640 json!({"routing/v1": {}, "bad key": {}}),
12641 "role_declarations key must use capability-name grammar",
12642 );
12643 }
12644
12645 #[test]
12646 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
12647 assert_hello_drops_role_declarations_with_warning(
12648 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
12649 "role_declarations exceeds 16384 compact JSON bytes",
12650 );
12651 }
12652
12653 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
12654 let mut body = serde_json::to_value(ModuleHelloBody {
12655 manifest: manifest("flow-ops", PROTOCOL_VERSION),
12656 protocol_ver: PROTOCOL_VERSION,
12657 control_ops: None,
12658 launch_nonce: None,
12659 })
12660 .unwrap();
12661 body["manifest"]["provides"] = json!([{
12662 "role": "management_surface",
12663 "operations": [
12664 {"name": "diagnostics.findings", "kind": "query"},
12665 {"name": "reset", "kind": "mutate"}
12666 ],
12667 "config_schema": {}, "observability": [], "identity_scope": []
12668 }]);
12669 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
12670 if let Some(flow_operations) = flow_operations {
12671 body["manifest"]["flow_operations"] = flow_operations;
12672 }
12673 Frame::build(
12674 FrameType::Hello,
12675 control_flags(),
12676 0,
12677 0,
12678 401,
12679 serde_json::to_vec(&body).unwrap(),
12680 )
12681 .unwrap()
12682 }
12683
12684 #[test]
12685 fn catalog_list_mirrors_flow_operations_unchanged() {
12686 for (capability, flow_operations) in [
12689 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
12690 ("call-modes/v1", json!(["diagnostics.findings"])),
12691 ("flow-scopes/v1", json!([])),
12692 ] {
12693 let registry = Arc::new(Registry::default());
12694 let handler = ControlHandler::new(Arc::clone(®istry));
12695 let connection = ConnectionId::new(401);
12696 let response = handler
12697 .handle_control(
12698 connection,
12699 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
12700 )
12701 .unwrap();
12702 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12703 let registered = registry
12704 .get_module_by_connection(connection)
12705 .unwrap()
12706 .unwrap()
12707 .manifest;
12708 assert_eq!(
12709 serde_json::to_value(®istered.flow_operations).unwrap(),
12710 flow_operations
12711 );
12712 let (raw, entry) = role_declarations_catalog(&handler);
12713 assert_eq!(raw["flow_operations"], flow_operations);
12714 assert_eq!(entry.flow_operations, registered.flow_operations);
12715 }
12716
12717 let registry = Arc::new(Registry::default());
12718 let handler = ControlHandler::new(Arc::clone(®istry));
12719 let response = handler
12720 .handle_control(
12721 ConnectionId::new(401),
12722 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
12723 )
12724 .unwrap();
12725 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12726 let (raw, entry) = role_declarations_catalog(&handler);
12727 assert!(raw.get("flow_operations").is_none());
12728 assert_eq!(entry.flow_operations, None);
12729 }
12730
12731 #[test]
12732 fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
12733 let oversized: serde_json::Map<String, Value> = (0..257)
12734 .map(|n| (format!("tool:t{n}"), json!(true)))
12735 .collect();
12736 for (capabilities, map, valid) in [
12737 (
12738 json!(["scope-federated/v1"]),
12739 json!({"tool:read":true,"management:reset":true}),
12740 true,
12741 ),
12742 (json!(["scope-federated/v1"]), json!({}), true),
12743 (json!([]), json!({"tool:read":true}), false),
12744 (
12745 json!(["scope-federated/v1"]),
12746 json!({"tool:valid":true,"tool:bad":false}),
12747 false,
12748 ),
12749 (
12750 json!(["scope-federated/v1"]),
12751 json!({"tool:valid":true,"management:missing":true}),
12752 false,
12753 ),
12754 (
12755 json!(["scope-federated/v1"]),
12756 Value::Object(oversized),
12757 false,
12758 ),
12759 ] {
12760 let capture = EventCapture::default();
12761 let _subscriber = tracing::subscriber::set_default(
12762 tracing_subscriber::registry().with(capture.clone()),
12763 );
12764 let registry = Arc::new(Registry::default());
12765 let handler = ControlHandler::new(Arc::clone(®istry));
12766 let hello = hello_with_flow_operations(capabilities, None);
12767 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12768 body["manifest"]["operation_deferral"] = map.clone();
12769 let hello = Frame::build(
12770 FrameType::Hello,
12771 control_flags(),
12772 0,
12773 0,
12774 401,
12775 serde_json::to_vec(&body).unwrap(),
12776 )
12777 .unwrap();
12778 let response = handler
12779 .handle_control(ConnectionId::new(401), hello)
12780 .unwrap();
12781 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12782 let (raw, entry) = role_declarations_catalog(&handler);
12783 if valid {
12784 assert_eq!(raw["operation_deferral"], map);
12785 assert_eq!(
12786 serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
12787 map
12788 );
12789 } else {
12790 assert!(entry.operation_deferral.is_none());
12791 assert!(raw.get("operation_deferral").is_none());
12792 assert_eq!(
12793 capture
12794 .events()
12795 .iter()
12796 .filter(|event| event.fields.get("message").map(String::as_str)
12797 == Some("dropping operation_deferral from HELLO"))
12798 .count(),
12799 1
12800 );
12801 }
12802 }
12803 }
12804
12805 fn assert_hello_drops_flow_operations_with_warning(
12806 capabilities: Value,
12807 flow_operations: Value,
12808 reason: &str,
12809 ) {
12810 let registry = Arc::new(Registry::default());
12811 let handler = ControlHandler::new(Arc::clone(®istry));
12812 let capture = EventCapture::default();
12813 let _subscriber =
12814 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12815 let response = handler
12816 .handle_control(
12817 ConnectionId::new(401),
12818 hello_with_flow_operations(capabilities, Some(flow_operations)),
12819 )
12820 .unwrap();
12821 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12822 let registered = registry
12823 .get_module_by_connection(ConnectionId::new(401))
12824 .unwrap()
12825 .expect("the module still registers")
12826 .manifest;
12827 assert_eq!(registered.module_id, "flow-ops");
12828 assert_eq!(registered.flow_operations, None);
12829 let (raw, entry) = role_declarations_catalog(&handler);
12830 assert_eq!(raw["module_id"], "flow-ops");
12831 assert!(raw.get("flow_operations").is_none());
12832 assert_eq!(entry.flow_operations, None);
12833 let warnings: Vec<_> = capture
12834 .events()
12835 .into_iter()
12836 .filter(|event| {
12837 event.fields.get("message").map(String::as_str)
12838 == Some("dropping flow_operations from HELLO")
12839 })
12840 .collect();
12841 assert_eq!(warnings.len(), 1);
12842 assert_eq!(warnings[0].level, tracing::Level::WARN);
12843 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
12844 assert_eq!(warnings[0].fields["reason"], reason);
12845 }
12846
12847 #[test]
12848 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
12849 assert_hello_drops_flow_operations_with_warning(
12850 json!(["flow-scopes/v1"]),
12851 json!(["diagnostics.findings", "missing"]),
12852 "flow_operations names 'missing', which is not a declared management operation",
12853 );
12854 }
12855
12856 #[test]
12857 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
12858 assert_hello_drops_flow_operations_with_warning(
12859 json!([]),
12860 json!(["diagnostics.findings"]),
12861 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12862 );
12863 assert_hello_drops_flow_operations_with_warning(
12865 json!(["agent-run-scopes/v1"]),
12866 json!([]),
12867 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12868 );
12869 }
12870
12871 #[test]
12872 fn hello_malformed_flow_operations_is_dropped_with_warning() {
12873 assert_hello_drops_flow_operations_with_warning(
12874 json!(["flow-scopes/v1"]),
12875 json!(["reset", "reset"]),
12876 "flow_operations names 'reset' more than once",
12877 );
12878 assert_hello_drops_flow_operations_with_warning(
12879 json!(["flow-scopes/v1"]),
12880 json!(vec!["reset"; 257]),
12881 "flow_operations lists 257 names, more than 256",
12882 );
12883 assert_hello_drops_flow_operations_with_warning(
12884 json!(["flow-scopes/v1"]),
12885 json!("reset"),
12886 "flow_operations must be an array of operation names",
12887 );
12888 }
12889
12890 #[test]
12891 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
12892 let registry = Arc::new(Registry::default());
12893 let handler = ControlHandler::new(registry.clone());
12894 let descriptor = json!({
12895 "schema": "ckios.richui.manifest/1",
12896 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12897 "bytes": 128,
12898 "chunks": 2
12899 });
12900 let malformed = [
12901 ("schema", json!("")),
12902 ("schema", json!("a".repeat(65))),
12903 ("schema", json!("Bad schema")),
12904 (
12905 "manifest_digest",
12906 json!(format!("sha256:{}", "A".repeat(64))),
12907 ),
12908 ("manifest_digest", json!("sha256:abc")),
12909 ("bytes", json!(0)),
12910 ("bytes", json!(16_777_217)),
12911 ("bytes", json!("128")),
12912 ("chunks", json!(0)),
12913 ("chunks", json!(1025)),
12914 ("chunks", json!("2")),
12915 ];
12916 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
12917 let mut invalid = descriptor.clone();
12918 invalid[field] = invalid_value;
12919 let response = handler
12920 .handle_control(
12921 ConnectionId::new(210 + index as u64),
12922 hello_frame_with_rich_ui(
12923 &format!("bad-rich-ui-{index}"),
12924 PROTOCOL_VERSION,
12925 210 + index as u64,
12926 None,
12927 invalid,
12928 ),
12929 )
12930 .unwrap();
12931 assert_eq!(response[0].header.ty, FrameType::Error);
12932 let error = parse_error(&response[0]);
12933 assert_eq!(error["code"], "invalid_manifest");
12934 assert!(error["message"]
12935 .as_str()
12936 .unwrap()
12937 .contains(&format!("rich_ui.{field}")));
12938 assert!(registry
12939 .get_module(&format!("bad-rich-ui-{index}"))
12940 .unwrap()
12941 .is_none());
12942 }
12943 }
12944
12945 #[test]
12946 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
12947 let registry = Arc::new(Registry::default());
12948 let handler = ControlHandler::new(Arc::clone(®istry));
12949 let connection = ConnectionId::new(219);
12950 registry
12951 .register_with_control_ops(
12952 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
12953 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
12954 "bad schema",
12955 format!("sha256:{}", "a".repeat(64)),
12956 128,
12957 2,
12958 )))
12959 .build(),
12960 PROTOCOL_VERSION,
12961 connection,
12962 Vec::new(),
12963 )
12964 .unwrap();
12965 let request =
12966 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
12967 let response = handler
12968 .handle_catalog_update(connection, request, Vec::new(), None, None)
12969 .unwrap();
12970 assert_eq!(response[0].header.ty, FrameType::Error);
12971 let error = parse_error(&response[0]);
12972 assert_eq!(error["code"], "invalid_manifest");
12973 assert!(error["message"]
12974 .as_str()
12975 .unwrap()
12976 .contains("rich_ui.schema"));
12977 }
12978
12979 #[test]
12980 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
12981 let registry = Arc::new(Registry::default());
12982 let supervisor = SupervisorHandle::new();
12983 supervisor.open_swap("provider", "candidate-nonce".to_string());
12984 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12985 let connection = ConnectionId::new(218);
12986 let hello = handler
12987 .handle_control(
12988 connection,
12989 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
12990 )
12991 .unwrap();
12992 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12993 assert!(registry.list_modules().unwrap().1.is_empty());
12994
12995 registry.promote_candidate("provider").unwrap().unwrap();
12996 let request = Frame::build(
12997 FrameType::Request,
12998 control_flags(),
12999 0,
13000 0,
13001 219,
13002 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13003 )
13004 .unwrap();
13005 let response = handler.handle_catalog_list(request, None).unwrap();
13006 let ClientControlResponse::CatalogList { modules, .. } =
13007 serde_json::from_slice(&response[0].body).unwrap()
13008 else {
13009 panic!("expected catalog.list");
13010 };
13011 assert_eq!(modules.len(), 1);
13012 assert!(modules[0].nonce_verified);
13013 }
13014
13015 #[test]
13016 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
13017 for provenance_json in [
13018 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
13019 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
13020 ] {
13021 let registry = Arc::new(Registry::default());
13022 let handler = ControlHandler::new(registry);
13023 let hello = Frame::build(
13024 FrameType::Hello,
13025 control_flags(),
13026 0,
13027 0,
13028 220,
13029 serde_json::to_vec(&ModuleHelloBody {
13030 manifest: ModuleManifest::builder("built", "0.1.0")
13031 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
13032 .build(),
13033 protocol_ver: PROTOCOL_VERSION,
13034 control_ops: None,
13035 launch_nonce: None,
13036 })
13037 .unwrap(),
13038 )
13039 .unwrap();
13040 assert_eq!(
13041 handler
13042 .handle_control(ConnectionId::new(220), hello)
13043 .unwrap()[0]
13044 .header
13045 .ty,
13046 FrameType::HelloAck
13047 );
13048 let request = Frame::build(
13049 FrameType::Request,
13050 control_flags(),
13051 0,
13052 0,
13053 221,
13054 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13055 )
13056 .unwrap();
13057 let response = handler.handle_catalog_list(request, None).unwrap();
13058 let ClientControlResponse::CatalogList { modules, .. } =
13059 serde_json::from_slice(&response[0].body).unwrap()
13060 else {
13061 panic!("expected catalog.list");
13062 };
13063 assert_eq!(modules.len(), 1);
13064 assert_eq!(
13065 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13066 format!(
13067 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
13068 )
13069 .as_bytes()
13070 );
13071 }
13072 }
13073
13074 #[test]
13075 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
13076 let registry = Arc::new(Registry::default());
13077 let handler = ControlHandler::new(Arc::clone(®istry));
13078 let hello = Frame::build(
13079 FrameType::Hello,
13080 control_flags(),
13081 0,
13082 0,
13083 220,
13084 serde_json::to_vec(&ModuleHelloBody {
13085 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
13086 protocol_ver: PROTOCOL_VERSION,
13087 control_ops: None,
13088 launch_nonce: Some("unverified".to_string()),
13089 })
13090 .unwrap(),
13091 )
13092 .unwrap();
13093 assert_eq!(
13094 handler
13095 .handle_control(ConnectionId::new(220), hello)
13096 .unwrap()[0]
13097 .header
13098 .ty,
13099 FrameType::HelloAck
13100 );
13101 let request = Frame::build(
13102 FrameType::Request,
13103 control_flags(),
13104 0,
13105 0,
13106 221,
13107 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13108 )
13109 .unwrap();
13110 let response = handler.handle_catalog_list(request, None).unwrap();
13111 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
13112 assert!(wire["modules"][0].get("provenance").is_none());
13113 let ClientControlResponse::CatalogList { modules, .. } =
13114 serde_json::from_slice(&response[0].body).unwrap()
13115 else {
13116 panic!("expected catalog.list");
13117 };
13118 assert_eq!(
13119 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13120 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
13121 );
13122 assert!(!modules[0].nonce_verified);
13123 assert!(modules[0].provenance.is_none());
13124 }
13125
13126 fn catalog_legacy_projection(mut entry: CatalogEntry) -> CatalogEntry {
13128 assert_eq!(entry.first_party, Some(false));
13129 entry.incarnation = None;
13130 entry.spawn = None;
13131 entry.first_party = None;
13132 entry.origin = None;
13133 entry.requested_declarations_digest = None;
13134 entry.approved_declarations_digest = None;
13135 entry
13136 }
13137
13138 #[test]
13139 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
13140 let registry = Arc::new(Registry::default());
13141 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13142 [("vault".to_string(), true), ("squatter".to_string(), true)],
13143 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13144 );
13145 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
13146 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13147 provides: vec!["credentials-provider/v1".to_string()],
13148 requires: Vec::new(),
13149 must_never_reach: Vec::new(),
13150 });
13151 let frame = Frame::build(
13152 FrameType::Hello,
13153 control_flags(),
13154 0,
13155 0,
13156 77,
13157 serde_json::to_vec(&ModuleHelloBody {
13158 manifest: squatter,
13159 protocol_ver: PROTOCOL_VERSION,
13160 control_ops: None,
13161 launch_nonce: None,
13162 })
13163 .expect("HELLO serializes"),
13164 )
13165 .expect("HELLO frame builds");
13166 let response = handler
13167 .handle_control(ConnectionId::new(77), frame)
13168 .expect("reserved claim receives a typed refusal");
13169 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13170 assert_eq!(
13171 registry
13172 .active_registration_count()
13173 .expect("registry reads"),
13174 0,
13175 "a reserved capability refusal must not leave a catalog entry"
13176 );
13177 }
13178
13179 #[test]
13180 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
13181 for settlement in ["timeout", "inconclusive", "drop"] {
13182 let breakers = RouteBindBreakers::default();
13183 let RouteBindAdmission::Admitted {
13184 guard: mut old,
13185 probe: false,
13186 } = breakers.admit("prov")
13187 else {
13188 panic!("ordinary relay admitted")
13189 };
13190 let RouteBindAdmission::Admitted {
13191 guard: mut opener, ..
13192 } = breakers.admit("prov")
13193 else {
13194 panic!("second relay admitted")
13195 };
13196 assert!(
13197 !opener
13198 .record_timeout(1, Duration::ZERO)
13199 .unwrap()
13200 .reopened_after_probe
13201 );
13202 let RouteBindAdmission::Admitted {
13203 guard: mut probe,
13204 probe: true,
13205 } = breakers.admit("prov")
13206 else {
13207 panic!("one cooldown probe admitted")
13208 };
13209 match settlement {
13210 "timeout" => assert!(
13211 !old.record_timeout(1, Duration::ZERO)
13212 .unwrap()
13213 .reopened_after_probe
13214 ),
13215 "inconclusive" => old.record_inconclusive(),
13216 "drop" => drop(old),
13217 _ => unreachable!(),
13218 }
13219 assert!(
13220 matches!(
13221 breakers.admit("prov"),
13222 RouteBindAdmission::Refused {
13223 probe_in_flight: true,
13224 ..
13225 }
13226 ),
13227 "{settlement} of a pre-open relay cannot release the real probe"
13228 );
13229 assert!(
13230 probe
13231 .record_timeout(1, Duration::ZERO)
13232 .unwrap()
13233 .reopened_after_probe
13234 );
13235 assert!(matches!(
13236 breakers.admit("prov"),
13237 RouteBindAdmission::Admitted { probe: true, .. }
13238 ));
13239 }
13240 let breakers = RouteBindBreakers::default();
13241 let admit = || match breakers.admit("prov") {
13242 RouteBindAdmission::Admitted { guard, .. } => guard,
13243 _ => panic!("relay admitted"),
13244 };
13245 admit().record_timeout(1, Duration::ZERO);
13246 let mut old_probe = admit();
13247 breakers.reset_for_new_module_connection("prov");
13248 admit().record_timeout(1, Duration::ZERO);
13249 let _new_probe = admit();
13250 old_probe.record_inconclusive();
13251 assert!(matches!(
13252 breakers.admit("prov"),
13253 RouteBindAdmission::Refused {
13254 probe_in_flight: true,
13255 ..
13256 }
13257 ));
13258 }
13259
13260 #[tokio::test]
13261 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
13262 for candidate in [false, true] {
13263 let registry = Arc::new(Registry::default());
13264 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13265 [("vault".to_string(), true), ("squatter".to_string(), true)],
13266 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13267 );
13268 let conn = ConnectionId::new(77);
13269 let (ctx, mut rx) = route_ctx(conn);
13270 let initial = capability_manifest("squatter", &[], &[]);
13271 if candidate {
13272 registry
13273 .register_candidate_with_control_ops(
13274 initial.clone(),
13275 PROTOCOL_VERSION,
13276 conn,
13277 module_baseline_control_ops(),
13278 )
13279 .unwrap();
13280 handler
13281 .forwarding
13282 .register_candidate_module_connection(
13283 conn,
13284 "squatter".to_string(),
13285 PROTOCOL_VERSION,
13286 manifest_concurrency(&initial),
13287 ctx.egress.clone(),
13288 )
13289 .unwrap();
13290 } else {
13291 hello_via_sink(
13292 &handler,
13293 &ctx,
13294 &mut rx,
13295 hello_frame_with_manifest(initial.clone(), 1),
13296 )
13297 .await;
13298 }
13299 let response = handler
13300 .handle_control_frame(
13301 &ctx,
13302 catalog_update_with_capabilities_frame(
13303 2,
13304 capability_manifest("squatter", &["credentials-provider/v1"], &[])
13305 .capabilities
13306 .unwrap(),
13307 ),
13308 )
13309 .await
13310 .unwrap();
13311 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13312 assert_eq!(
13313 registry
13314 .get_module_by_connection(conn)
13315 .unwrap()
13316 .unwrap()
13317 .manifest,
13318 initial
13319 );
13320 }
13321 }
13322
13323 #[test]
13324 #[cfg(not(feature = "dev-desktop-approval"))]
13325 fn server_describe_default_build_advertises_no_config_capability() {
13326 let assert_default = |handler: &ControlHandler| {
13327 let request = Frame::build(
13328 FrameType::Request,
13329 control_flags(),
13330 0,
13331 0,
13332 79,
13333 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13334 )
13335 .unwrap();
13336 let responses = handler.handle_server_describe(request).unwrap();
13337 assert_eq!(responses.len(), 1);
13338 let response: Value = serde_json::from_slice(&responses[0].body).unwrap();
13339 assert_eq!(response["op"], "server.describe");
13340 let capabilities = response["capabilities"]
13341 .as_array()
13342 .expect("capability array");
13343 assert!(
13344 !capabilities.is_empty(),
13345 "exercise an actual describe response, not an empty fixture"
13346 );
13347 assert!(
13348 capabilities
13349 .iter()
13350 .all(|capability| !capability.as_str().unwrap().starts_with("config.")),
13351 "a default build must not announce configuration authority: {capabilities:?}"
13352 );
13353 };
13354 assert_default(&ControlHandler::new(Arc::new(Registry::default())));
13355 #[cfg(unix)]
13356 {
13357 let (_rig, _owner, _supervisor, handler, _spec) =
13361 crate::config_gate::runtime::RuntimeOwner::unactivated_supervisor_fixture_for_test(
13362 );
13363 assert_default(&handler);
13364 }
13365 }
13366
13367 #[test]
13368 fn server_describe_surfaces_required_capability_verdict_fields() {
13369 let registry = Arc::new(Registry::default());
13370 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13371 [
13372 ("consumer".to_string(), true),
13373 ("provider".to_string(), false),
13374 ],
13375 BTreeMap::new(),
13376 );
13377 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13378 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13379 provides: Vec::new(),
13380 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13381 capability: "credentials-provider/v1".to_string(),
13382 need: subc_protocol::manifest::CapabilityNeed::Required,
13383 }],
13384 must_never_reach: Vec::new(),
13385 });
13386 let hello = Frame::build(
13387 FrameType::Hello,
13388 control_flags(),
13389 0,
13390 0,
13391 78,
13392 serde_json::to_vec(&ModuleHelloBody {
13393 manifest: consumer,
13394 protocol_ver: PROTOCOL_VERSION,
13395 control_ops: None,
13396 launch_nonce: None,
13397 })
13398 .expect("HELLO serializes"),
13399 )
13400 .expect("HELLO frame builds");
13401 handler
13402 .handle_control(ConnectionId::new(78), hello)
13403 .expect("consumer registers");
13404 let describe = Frame::build(
13405 FrameType::Request,
13406 control_flags(),
13407 0,
13408 0,
13409 79,
13410 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
13411 .expect("request serializes"),
13412 )
13413 .expect("describe frame builds");
13414 let response = handler
13415 .handle_server_describe(describe)
13416 .expect("server.describe succeeds");
13417 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
13418 let requirement = &rendered["capability_requirements"][0];
13419 assert_eq!(requirement["consumer"], "consumer");
13420 assert_eq!(requirement["verdict"], "never_provided");
13421 assert_eq!(requirement["episode_seq"], 1);
13422 assert_eq!(requirement["config_satisfiable"], false);
13423 assert_eq!(requirement["runtime_available"], false);
13424 assert!(requirement["detail"]
13425 .as_str()
13426 .expect("detail string")
13427 .contains("credentials-provider/v1"));
13428 }
13429
13430 fn deregister_required_provider(
13436 daemon_shutdown_begun: bool,
13437 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
13438 const CAPABILITY: &str = "project-identity/v1";
13439 let registry = Arc::new(Registry::default());
13440 let daemon_shutdown = DaemonShutdownFlag::default();
13441 let handler = ControlHandler::new(Arc::clone(®istry))
13442 .with_capability_config(
13443 [
13444 ("consumer".to_string(), true),
13445 ("provider".to_string(), true),
13446 ],
13447 BTreeMap::new(),
13448 )
13449 .with_daemon_shutdown_flag(daemon_shutdown.clone());
13450 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13451 consumer.capabilities = Some(CapabilityDeclarations {
13452 provides: Vec::new(),
13453 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13454 capability: CAPABILITY.to_string(),
13455 need: subc_protocol::manifest::CapabilityNeed::Required,
13456 }],
13457 must_never_reach: Vec::new(),
13458 });
13459 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
13460 let provider_conn = ConnectionId::new(182);
13461 for (conn, manifest, corr) in [
13462 (ConnectionId::new(181), consumer, 181),
13463 (provider_conn, provider, 182),
13464 ] {
13465 let hello = handler
13466 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
13467 .expect("module registers");
13468 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13469 }
13470 assert_eq!(
13471 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13472 Some(CapabilityVerdict::Provided),
13473 "the registered provider must satisfy the requirement first"
13474 );
13475 if daemon_shutdown_begun {
13476 daemon_shutdown.set_for_test();
13477 }
13478
13479 let capture = EventCapture::default();
13480 let _subscriber =
13481 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13482 handler
13483 .cleanup_connection(provider_conn)
13484 .expect("provider deregisters");
13485 let events = capture
13486 .events()
13487 .into_iter()
13488 .filter(|event| {
13489 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
13490 })
13491 .collect();
13492 (
13493 events,
13494 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13495 )
13496 }
13497
13498 #[test]
13499 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
13500 let (events, verdict) = deregister_required_provider(true);
13501 assert_eq!(
13502 verdict,
13503 Some(CapabilityVerdict::NeverProvided),
13504 "shutdown must not change the verdict route.open refusals read"
13505 );
13506 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13507 let event = &events[0];
13508 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
13509 assert_eq!(
13510 event.fields.get("cause").map(String::as_str),
13511 Some("\"daemon_shutdown\""),
13512 "{event:?}"
13513 );
13514 assert_eq!(
13515 event.fields.get("verdict").map(String::as_str),
13516 Some("\"never_provided\""),
13517 "{event:?}"
13518 );
13519 }
13520
13521 #[test]
13522 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
13523 let (events, verdict) = deregister_required_provider(false);
13524 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
13525 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13526 let event = &events[0];
13527 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
13528 assert_eq!(event.fields.get("cause"), None, "{event:?}");
13529 assert_eq!(
13530 event.fields.get("verdict").map(String::as_str),
13531 Some("\"never_provided\""),
13532 "{event:?}"
13533 );
13534 }
13535
13536 #[test]
13537 fn catalog_list_omits_capabilities_for_legacy_manifest() {
13538 let registry = Arc::new(Registry::default());
13539 let handler = ControlHandler::new(Arc::clone(®istry));
13540 let hello = handler
13541 .handle_control(
13542 ConnectionId::new(101),
13543 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
13544 )
13545 .expect("legacy HELLO registers");
13546 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13547
13548 let request = Frame::build(
13549 FrameType::Request,
13550 control_flags(),
13551 0,
13552 0,
13553 102,
13554 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
13555 .expect("catalog request serializes"),
13556 )
13557 .expect("catalog request frame builds");
13558 let response = handler
13559 .handle_catalog_list(request, None)
13560 .expect("catalog list succeeds");
13561 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
13562 assert!(
13563 body["modules"][0].get("capabilities").is_none(),
13564 "legacy manifest must retain an absent capabilities field on catalog.list"
13565 );
13566 }
13567
13568 #[test]
13569 fn hello_ack_omits_storage_when_no_storage_config() {
13570 let registry = Arc::new(Registry::default());
13571 let handler = ControlHandler::new(Arc::clone(®istry));
13572 let responses = handler
13573 .handle_control(
13574 ConnectionId::new(1),
13575 hello_frame("aft", PROTOCOL_VERSION, 7),
13576 )
13577 .unwrap();
13578 let ack = parse_ack(&responses[0]);
13579 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
13580 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
13581 }
13582
13583 #[tokio::test]
13584 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
13585 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
13586 let registry = Arc::new(Registry::default());
13587 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
13588 let responses = handler
13589 .handle_control(
13590 ConnectionId::new(1),
13591 hello_frame("aft", PROTOCOL_VERSION, 7),
13592 )
13593 .unwrap();
13594 let ack = parse_ack(&responses[0]);
13595 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
13596
13597 let described = handler
13598 .handle_control_frame(
13599 &route_ctx(ConnectionId::new(2)).0,
13600 Frame::build(
13601 FrameType::Request,
13602 control_flags(),
13603 0,
13604 0,
13605 9,
13606 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13607 )
13608 .unwrap(),
13609 )
13610 .await
13611 .unwrap();
13612 let ClientControlResponse::ServerDescribe { machine_id, .. } =
13613 serde_json::from_slice(&described[0].body).unwrap()
13614 else {
13615 panic!("server.describe answered with another shape");
13616 };
13617 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
13618 }
13619
13620 #[test]
13621 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
13622 let registry = Arc::new(Registry::default());
13625 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
13626 crate::daemon_config::StorageConfig::Sqlite {
13627 data_home: std::path::PathBuf::from("/data"),
13628 },
13629 ));
13630
13631 let responses = handler
13632 .handle_control(
13633 ConnectionId::new(1),
13634 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
13635 )
13636 .unwrap();
13637 let ack = parse_ack(&responses[0]);
13638 assert_eq!(
13639 ack.storage,
13640 Some(serde_json::json!({
13641 "module_id": "alfonso-routing",
13642 "storage_namespace": "default",
13643 "isolation": { "kind": "module" },
13644 "backend": {
13645 "backend": "sqlite",
13646 "path": "/data/cortexkit/alfonso-routing/store.db"
13647 }
13648 })),
13649 "the delivered descriptor is the module's own sqlite store path"
13650 );
13651 }
13652
13653 #[test]
13654 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
13655 let registry = Arc::new(Registry::default());
13656 let handler = ControlHandler::new(Arc::clone(®istry));
13657 let conn = ConnectionId::new(1);
13658 let responses = handler
13659 .handle_control(
13660 conn,
13661 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13662 )
13663 .unwrap();
13664 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
13665 let registration = registry.get_module("aft").unwrap().unwrap();
13666 assert_eq!(registration.control_ops, module_baseline_control_ops());
13667
13668 let frame =
13669 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
13670 assert!(handler
13671 .guard_module_control_op(&frame, "aft", "route.bind")
13672 .unwrap()
13673 .is_none());
13674 let error = handler
13675 .guard_module_control_op(&frame, "aft", "test.synthetic")
13676 .unwrap()
13677 .expect("synthetic ungranted op should be rejected");
13678 assert_eq!(error.header.ty, FrameType::Error);
13679 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
13680 }
13681
13682 #[test]
13683 fn hello_control_ops_some_adds_optional_grants() {
13684 let registry = Arc::new(Registry::default());
13685 let handler = ControlHandler::new(Arc::clone(®istry));
13686 handler
13687 .handle_control(
13688 ConnectionId::new(1),
13689 hello_frame_with_control_ops(
13690 "aft",
13691 PROTOCOL_VERSION,
13692 7,
13693 Some(vec![
13694 "future.synthetic".to_string(),
13695 "route.bind".to_string(),
13696 ]),
13697 ),
13698 )
13699 .unwrap();
13700 let registration = registry.get_module("aft").unwrap().unwrap();
13701 assert_eq!(
13702 registration.control_ops,
13703 vec![
13704 "route.bind".to_string(),
13705 "route.status".to_string(),
13706 "future.synthetic".to_string(),
13707 ]
13708 );
13709 let frame =
13710 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
13711 assert!(handler
13712 .guard_module_control_op(&frame, "aft", "future.synthetic")
13713 .unwrap()
13714 .is_none());
13715 }
13716
13717 #[tokio::test]
13718 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
13719 let registry = Arc::new(Registry::default());
13720 let forwarding = Arc::new(ForwardingTable::default());
13721 let handler =
13722 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13723 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
13724 hello_via_sink(
13725 &handler,
13726 &module_ctx,
13727 &mut module_rx,
13728 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13729 )
13730 .await;
13731
13732 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
13733 let responses = handler
13734 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
13735 .await
13736 .unwrap();
13737 assert_eq!(responses.len(), 1);
13738 assert_eq!(responses[0].header.ty, FrameType::Error);
13739 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
13740 assert!(module_rx.try_recv().is_err());
13741 }
13742
13743 #[tokio::test]
13744 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
13745 let registry = Arc::new(Registry::default());
13746 let forwarding = Arc::new(ForwardingTable::default());
13747 let handler =
13748 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13749 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
13750 hello_via_sink(
13751 &handler,
13752 &module_ctx,
13753 &mut module_rx,
13754 hello_frame_with_control_ops(
13755 "aft",
13756 PROTOCOL_VERSION,
13757 7,
13758 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13759 ),
13760 )
13761 .await;
13762
13763 let project_root = unique_project_root("demux");
13764 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
13765 let route_handler = handler.clone();
13766 let route_task = tokio::spawn(async move {
13767 route_handler
13768 .handle_control_frame(
13769 &route_client_ctx,
13770 route_open_frame(100, "aft", project_root),
13771 )
13772 .await
13773 .unwrap()
13774 });
13775 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13776 .await
13777 .unwrap()
13778 .unwrap();
13779 assert!(matches!(
13780 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
13781 ModuleControlRequest::RouteBind { .. }
13782 ));
13783
13784 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
13785 let health_handler = handler.clone();
13786 let health_task = tokio::spawn(async move {
13787 health_handler
13788 .handle_control_frame(
13789 &health_client_ctx,
13790 supervisor_health_probe_frame(101, "aft"),
13791 )
13792 .await
13793 .unwrap()
13794 });
13795 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13796 .await
13797 .unwrap()
13798 .unwrap();
13799 assert_eq!(
13800 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13801 ModuleControlRequest::HealthCheck {}
13802 );
13803
13804 handler
13805 .handle_control_frame(
13806 &module_ctx,
13807 health_response(health_frame.header.corr, HealthStatus::Degraded),
13808 )
13809 .await
13810 .unwrap();
13811 let health_response = health_task.await.unwrap();
13812 assert_eq!(health_response.len(), 1);
13813 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13814 ClientControlResponse::SupervisorHealthProbe {
13815 module_id,
13816 status,
13817 detail,
13818 metrics,
13819 } => {
13820 assert_eq!(module_id, "aft");
13821 assert_eq!(status, HealthStatus::Degraded);
13822 assert_eq!(detail.as_deref(), Some("warming"));
13823 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
13824 }
13825 other => panic!("unexpected health response: {other:?}"),
13826 }
13827
13828 handler
13829 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13830 .await
13831 .unwrap();
13832 let route_response = route_task.await.unwrap();
13833 assert!(route_response.is_empty());
13834 let published = route_client_rx.recv().await.unwrap();
13835 assert!(matches!(
13836 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13837 ClientControlResponse::RouteOpen { .. }
13838 ));
13839 }
13840
13841 async fn relay_route_open(
13846 handler: &ControlHandler,
13847 client_connection: ConnectionId,
13848 client_egress: &FrameSink,
13849 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13850 corr: u64,
13851 module_id: &str,
13852 project_root_label: &str,
13853 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
13854 let ctx = RouteCtx {
13855 connection_id: client_connection,
13856 egress: client_egress.clone(),
13857 };
13858 let handler = handler.clone();
13859 let project_root = unique_project_root(project_root_label);
13860 let module_id = module_id.to_string();
13861 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
13862 let task = tokio::spawn(async move {
13863 let _guard = tracing::dispatcher::set_default(&dispatch);
13864 handler
13865 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
13866 .await
13867 .unwrap()
13868 });
13869 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13870 .await
13871 .expect("module receives the relayed route.bind")
13872 .expect("module egress is open");
13873 (task, bind.frame)
13874 }
13875
13876 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
13877 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
13878 ModuleControlRequest::RouteBind {
13879 route_channel,
13880 epoch,
13881 ..
13882 } => (route_channel, epoch),
13883 other => panic!("expected a route.bind request, got {other:?}"),
13884 }
13885 }
13886
13887 fn published_route(frame: &Frame) -> (u16, u32) {
13888 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
13889 ClientControlResponse::RouteOpen {
13890 route_channel,
13891 route_epoch,
13892 ..
13893 } => (route_channel, route_epoch),
13894 other => panic!("expected a route.open response, got {other:?}"),
13895 }
13896 }
13897
13898 #[tokio::test]
13918 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
13919 let registry = Arc::new(Registry::default());
13920 let forwarding = Arc::new(ForwardingTable::default());
13921 let handler =
13922 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13923
13924 let module_connection = ConnectionId::new(30);
13925 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13926 hello_via_sink(
13927 &handler,
13928 &module_ctx,
13929 &mut module_rx,
13930 hello_frame("aft", PROTOCOL_VERSION, 7),
13931 )
13932 .await;
13933
13934 let dying_client = ConnectionId::new(31);
13935 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
13936
13937 let (first_task, first_bind) = relay_route_open(
13940 &handler,
13941 dying_client,
13942 &dying_ctx.egress,
13943 &mut module_rx,
13944 100,
13945 "aft",
13946 "closing-first",
13947 )
13948 .await;
13949 handler
13950 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
13951 .await
13952 .unwrap();
13953 assert!(first_task.await.unwrap().is_empty());
13954 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
13955
13956 let (second_task, second_bind) = relay_route_open(
13958 &handler,
13959 dying_client,
13960 &dying_ctx.egress,
13961 &mut module_rx,
13962 101,
13963 "aft",
13964 "closing-second",
13965 )
13966 .await;
13967 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
13968
13969 assert!(forwarding
13972 .escalate_client_delivery_failure(
13973 dying_client,
13974 first_channel,
13975 first_epoch,
13976 CloseReason::new(
13977 "module_to_client_delivery_failed",
13978 "client egress refused a module frame",
13979 ),
13980 crate::forwarding::UndeliveredFrame {
13981 module_id: None,
13982 sink: &dying_ctx.egress,
13983 },
13984 )
13985 .unwrap());
13986 assert!(!dying_ctx.egress.is_closed());
13987
13988 let ack = handler
13990 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
13991 .await;
13992 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13993 if module_loop_error.is_some() {
13994 handler.cleanup_connection(module_connection).unwrap();
13998 }
13999 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14003 .await
14004 .ok()
14005 .flatten();
14006
14007 assert!(
14009 registry
14010 .get_module_by_connection(module_connection)
14011 .unwrap()
14012 .is_some(),
14013 "one client's closing connection ended the shared module connection: \
14014 {module_loop_error:?}"
14015 );
14016 let cotenant = ConnectionId::new(32);
14018 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14019 let (cotenant_task, cotenant_bind) = relay_route_open(
14020 &handler,
14021 cotenant,
14022 &cotenant_ctx.egress,
14023 &mut module_rx,
14024 102,
14025 "aft",
14026 "closing-cotenant",
14027 )
14028 .await;
14029 handler
14030 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
14031 .await
14032 .unwrap();
14033 assert!(cotenant_task.await.unwrap().is_empty());
14034 let (cotenant_channel, cotenant_epoch) =
14035 published_route(&cotenant_rx.recv().await.unwrap());
14036 assert!(matches!(
14037 forwarding
14038 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14039 .unwrap(),
14040 DataRoute::Client(DataRouteState::Bound(_))
14041 ));
14042
14043 let goodbye = post_ack_module_frame
14046 .expect("module receives a GOODBYE for the abandoned route channel");
14047 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14048 assert_eq!(goodbye.header.channel, abandoned_channel);
14049 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14050
14051 assert!(dying_rx.try_recv().is_err());
14055 let second_response = second_task.await.unwrap();
14056 assert_eq!(second_response.len(), 1);
14057 assert_eq!(
14058 parse_error(&second_response[0])["code"],
14059 "target_unavailable"
14060 );
14061 }
14062
14063 async fn catalog_snapshot(
14066 handler: &ControlHandler,
14067 client_ctx: &RouteCtx,
14068 corr: u64,
14069 ) -> (u64, Vec<(String, bool)>) {
14070 let body =
14071 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
14072 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
14073 let reply = handler
14074 .handle_control_frame(client_ctx, request)
14075 .await
14076 .unwrap()
14077 .pop()
14078 .unwrap();
14079 let ClientControlResponse::CatalogList {
14080 generation,
14081 modules,
14082 ..
14083 } = serde_json::from_slice(&reply.body).unwrap()
14084 else {
14085 panic!("not a catalog.list reply");
14086 };
14087 (
14088 generation,
14089 modules
14090 .into_iter()
14091 .map(|entry| (entry.module_id, entry.ready))
14092 .collect(),
14093 )
14094 }
14095
14096 fn hello_held_before_forwarding() -> (
14102 ControlHandler,
14103 Arc<std::sync::Barrier>,
14104 std::thread::JoinHandle<Vec<Frame>>,
14105 RouteCtx,
14106 mpsc::Receiver<crate::router::OutboundFrame>,
14107 ) {
14108 let hold = Arc::new(std::sync::Barrier::new(2));
14109 let handler = ControlHandler::with_forwarding(
14110 Arc::new(Registry::default()),
14111 Arc::new(ForwardingTable::default()),
14112 )
14113 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14114 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14115 let hello = {
14116 let handler = handler.clone();
14117 let module_ctx = module_ctx.clone();
14118 std::thread::spawn(move || {
14119 handler
14120 .handle_hello(
14121 module_ctx.connection_id,
14122 Some(module_ctx.egress.clone()),
14123 hello_frame("aft", PROTOCOL_VERSION, 7),
14124 )
14125 .unwrap()
14126 })
14127 };
14128 (handler, hold, hello, module_ctx, module_rx)
14129 }
14130
14131 fn lookup_during_hello_publication<T: Send + 'static>(
14137 registry: &Registry,
14138 hold: &std::sync::Barrier,
14139 lookup: impl FnOnce() -> T + Send + 'static,
14140 ) -> T {
14141 let attempted = registry.observe_next_lock_attempt();
14142 let reader = std::thread::spawn(lookup);
14143 if attempted.recv().unwrap() {
14144 let result = reader.join().unwrap();
14145 hold.wait();
14146 result
14147 } else {
14148 hold.wait();
14149 reader.join().unwrap()
14150 }
14151 }
14152
14153 #[test]
14154 fn catalog_read_after_hello_ack_waits_for_publication() {
14155 let registry = Arc::new(Registry::default());
14156 let hold = Arc::new(std::sync::Barrier::new(2));
14157 let mut handler = ControlHandler::with_forwarding(
14158 Arc::clone(®istry),
14159 Arc::new(ForwardingTable::default()),
14160 );
14161 handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
14162 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14163 let hello = {
14164 let handler = handler.clone();
14165 std::thread::spawn(move || {
14166 handler
14167 .handle_hello(
14168 module_ctx.connection_id,
14169 Some(module_ctx.egress),
14170 hello_frame("aft", PROTOCOL_VERSION, 7),
14171 )
14172 .unwrap()
14173 })
14174 };
14175 hold.wait();
14176 assert_eq!(
14177 module_rx.try_recv().unwrap().frame.header.ty,
14178 FrameType::HelloAck
14179 );
14180 let replies = lookup_during_hello_publication(®istry, &hold, move || {
14181 let request = Frame::build(
14184 FrameType::Request,
14185 control_flags(),
14186 0,
14187 0,
14188 300,
14189 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
14190 )
14191 .unwrap();
14192 handler.handle_catalog_list(request, None).unwrap()
14193 });
14194 assert!(hello.join().unwrap().is_empty());
14195 let ClientControlResponse::CatalogList {
14196 generation,
14197 modules,
14198 ..
14199 } = serde_json::from_slice(&replies[0].body).unwrap()
14200 else {
14201 panic!("not a catalog.list reply");
14202 };
14203 assert_eq!(
14204 modules
14205 .iter()
14206 .map(|entry| entry.module_id.as_str())
14207 .collect::<Vec<_>>(),
14208 vec!["aft"]
14209 );
14210 assert_eq!(generation, 1);
14211 }
14212
14213 #[tokio::test]
14222 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
14223 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
14224 let client = ConnectionId::new(41);
14225 let (client_ctx, mut client_rx) = route_ctx(client);
14226
14227 hold.wait();
14228 assert!(handler.registry.get_module("aft").unwrap().is_some());
14230 assert!(!handler
14231 .forwarding
14232 .has_live_module_connection("aft")
14233 .unwrap());
14234 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
14235 assert!(
14236 listed.is_empty(),
14237 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
14238 );
14239 let refused = handler
14240 .handle_control_frame(
14241 &client_ctx,
14242 route_open_frame(301, "aft", unique_project_root("listed-window")),
14243 )
14244 .await
14245 .unwrap();
14246 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
14247 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
14248 let duplicate = handler
14249 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
14250 .await
14251 .unwrap();
14252 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14253
14254 hold.wait();
14255 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14256 let ack = module_rx.try_recv().unwrap().frame;
14257 assert_eq!(ack.header.ty, FrameType::HelloAck);
14258 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14259 assert_eq!(listed, vec![("aft".to_string(), true)]);
14260
14261 let (open_task, bind) = relay_route_open(
14262 &handler,
14263 client,
14264 &client_ctx.egress,
14265 &mut module_rx,
14266 303,
14267 "aft",
14268 "listed-after",
14269 )
14270 .await;
14271 assert!(handler
14272 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14273 .await
14274 .unwrap()
14275 .is_empty());
14276 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
14277 published_route(&client_rx.try_recv().unwrap().frame);
14278 }
14279
14280 #[tokio::test]
14287 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
14288 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
14289 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14290 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14291
14292 hold.wait();
14293 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
14294 assert!(listed.is_empty());
14295 assert_eq!(
14296 during, before,
14297 "the generation moved while the module was not listed"
14298 );
14299
14300 hold.wait();
14301 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14302 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14303 assert_eq!(listed, vec![("aft".to_string(), true)]);
14304 assert_eq!(
14305 after,
14306 before + 1,
14307 "the module appeared without exactly one generation bump"
14308 );
14309 }
14310
14311 #[tokio::test]
14316 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
14317 let registry = Arc::new(Registry::default());
14318 let handler = ControlHandler::with_forwarding(
14319 Arc::clone(®istry),
14320 Arc::new(ForwardingTable::default()),
14321 );
14322 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14323 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14324 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14325 drop(module_rx);
14326
14327 let replies = handler
14328 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
14329 .await
14330 .unwrap();
14331 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
14332 assert!(registry.get_module("aft").unwrap().is_none());
14333 assert!(registry
14334 .get_module_by_connection(module_ctx.connection_id)
14335 .unwrap()
14336 .is_none());
14337 assert_eq!(
14338 catalog_snapshot(&handler, &client_ctx, 301).await,
14339 (before, Vec::new())
14340 );
14341 }
14342
14343 async fn route_open_held_after_bind_send() -> (
14347 ControlHandler,
14348 Arc<tokio::sync::Barrier>,
14349 RouteCtx,
14350 mpsc::Receiver<crate::router::OutboundFrame>,
14351 ) {
14352 let hold = Arc::new(tokio::sync::Barrier::new(2));
14353 let handler = ControlHandler::with_forwarding(
14354 Arc::new(Registry::default()),
14355 Arc::new(ForwardingTable::default()),
14356 )
14357 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
14358 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14359 hello_via_sink(
14360 &handler,
14361 &module_ctx,
14362 &mut module_rx,
14363 hello_frame("aft", PROTOCOL_VERSION, 7),
14364 )
14365 .await;
14366 (handler, hold, module_ctx, module_rx)
14367 }
14368
14369 #[tokio::test]
14376 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
14377 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14378 let client = ConnectionId::new(41);
14379 let (client_ctx, mut client_rx) = route_ctx(client);
14380
14381 let (open_task, bind) = relay_route_open(
14382 &handler,
14383 client,
14384 &client_ctx.egress,
14385 &mut module_rx,
14386 200,
14387 "aft",
14388 "acked-before-mark",
14389 )
14390 .await;
14391 let (module_channel, module_epoch) = route_bind_channel(&bind);
14392 hold.wait().await;
14394
14395 assert!(handler
14397 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14398 .await
14399 .unwrap()
14400 .is_empty());
14401 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
14402
14403 hold.wait().await;
14404 assert!(
14405 open_task.await.unwrap().is_empty(),
14406 "route.open answers accepted"
14407 );
14408
14409 match module_rx.try_recv() {
14412 Err(mpsc::error::TryRecvError::Empty) => {}
14413 Ok(outbound) => panic!(
14414 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
14415 outbound.frame.header.ty,
14416 outbound.frame.header.channel,
14417 outbound.frame.header.epoch
14418 ),
14419 Err(err) => panic!("module egress closed: {err}"),
14420 }
14421
14422 let router = Router::with_control_handler(Arc::new(handler.clone()));
14425 let request = Frame::build(
14426 FrameType::Request,
14427 Flags::new(true, Priority::Interactive, false),
14428 client_channel,
14429 client_epoch,
14430 201,
14431 b"first request".to_vec(),
14432 )
14433 .unwrap();
14434 router
14435 .route_for_connection(&client_ctx, request)
14436 .await
14437 .unwrap();
14438 let delivered = module_rx.try_recv().unwrap().frame;
14439 assert_eq!(delivered.header.ty, FrameType::Request);
14440 assert_eq!(
14441 (delivered.header.channel, delivered.header.epoch),
14442 (module_channel, module_epoch)
14443 );
14444 assert_eq!(delivered.body, b"first request");
14445 }
14446
14447 #[tokio::test]
14453 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
14454 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14455 let client = ConnectionId::new(42);
14456 let (client_ctx, _client_rx) = route_ctx(client);
14457
14458 let (open_task, bind) = relay_route_open(
14459 &handler,
14460 client,
14461 &client_ctx.egress,
14462 &mut module_rx,
14463 300,
14464 "aft",
14465 "released-before-mark",
14466 )
14467 .await;
14468 let (module_channel, module_epoch) = route_bind_channel(&bind);
14469 hold.wait().await;
14470
14471 handler.cleanup_connection(client).unwrap();
14472 assert!(
14473 module_rx.try_recv().is_err(),
14474 "client teardown must not GOODBYE a bind route.open is still queueing"
14475 );
14476
14477 hold.wait().await;
14478 let response = open_task.await.unwrap();
14479 assert_eq!(response.len(), 1);
14480 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14481
14482 let goodbye = module_rx
14483 .try_recv()
14484 .expect("module is told to drop the binding the daemon abandoned")
14485 .frame;
14486 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14487 assert_eq!(
14488 (goodbye.header.channel, goodbye.header.epoch),
14489 (module_channel, module_epoch)
14490 );
14491 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14492 }
14493
14494 #[tokio::test]
14499 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
14500 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14501 let client = ConnectionId::new(43);
14502 let (client_ctx, client_rx) = route_ctx(client);
14503
14504 let (open_task, bind) = relay_route_open(
14505 &handler,
14506 client,
14507 &client_ctx.egress,
14508 &mut module_rx,
14509 400,
14510 "aft",
14511 "gone-client-before-mark",
14512 )
14513 .await;
14514 let (module_channel, module_epoch) = route_bind_channel(&bind);
14515 hold.wait().await;
14516
14517 drop(client_rx);
14518 assert!(client_ctx.egress.is_closed());
14519 handler
14520 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14521 .await
14522 .unwrap();
14523
14524 hold.wait().await;
14525 let response = open_task.await.unwrap();
14526 assert_eq!(response.len(), 1);
14527 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14528
14529 let goodbye = module_rx
14530 .try_recv()
14531 .expect("module is told to drop the binding it acked")
14532 .frame;
14533 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14534 assert_eq!(
14535 (goodbye.header.channel, goodbye.header.epoch),
14536 (module_channel, module_epoch)
14537 );
14538 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14539 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14540 }
14541
14542 #[test]
14549 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
14550 let handler = ControlHandler::default();
14551 let module_connection = ConnectionId::new(30);
14552 let client_connection = ConnectionId::new(31);
14553
14554 handler
14555 .refuse_to_end_module_connection_for_a_client(
14556 module_connection,
14557 77,
14558 ForwardingError::ConnectionClosing {
14559 connection_id: client_connection,
14560 },
14561 )
14562 .expect("a closing client must never end the module connection");
14563
14564 assert!(matches!(
14565 handler.refuse_to_end_module_connection_for_a_client(
14566 module_connection,
14567 78,
14568 ForwardingError::ConnectionClosing {
14569 connection_id: module_connection,
14570 },
14571 ),
14572 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
14573 connection_id
14574 })) if connection_id == module_connection
14575 ));
14576 assert!(matches!(
14577 handler.refuse_to_end_module_connection_for_a_client(
14578 module_connection,
14579 79,
14580 ForwardingError::Poisoned,
14581 ),
14582 Err(RouterError::Forwarding(ForwardingError::Poisoned))
14583 ));
14584 assert!(matches!(
14585 handler.refuse_to_end_module_connection_for_a_client(
14586 module_connection,
14587 80,
14588 ForwardingError::StaleModuleEndpoint,
14589 ),
14590 Err(RouterError::Forwarding(
14591 ForwardingError::StaleModuleEndpoint
14592 ))
14593 ));
14594 }
14595
14596 #[tokio::test]
14609 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
14610 let registry = Arc::new(Registry::default());
14611 let forwarding = Arc::new(ForwardingTable::default());
14612 let supervisor = SupervisorHandle::new();
14613 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14614 let handler =
14615 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14616 .with_supervisor(supervisor);
14617
14618 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14619 hello_via_sink(
14620 &handler,
14621 &target_ctx,
14622 &mut target_rx,
14623 hello_frame("target", PROTOCOL_VERSION, 1),
14624 )
14625 .await;
14626
14627 let wrong_nonce = handler
14631 .handle_control_frame(
14632 &route_ctx(ConnectionId::new(91)).0,
14633 route_open_frame_with_admission_facts(
14634 20,
14635 "target",
14636 unique_project_root("admission-facts"),
14637 Some(subc_control::ConsumerIdentity {
14638 module_id: "fed".to_string(),
14639 launch_nonce: "not-the-real-nonce".to_string(),
14640 }),
14641 None,
14642 ),
14643 )
14644 .await
14645 .unwrap();
14646 assert_eq!(
14647 parse_error(&wrong_nonce[0])["code"],
14648 "bad_consumer_identity",
14649 "a mismatched launch nonce must be refused, not stamped Reserved"
14650 );
14651
14652 let never_spawned = handler
14656 .handle_control_frame(
14657 &route_ctx(ConnectionId::new(92)).0,
14658 route_open_frame_with_admission_facts(
14659 21,
14660 "target",
14661 unique_project_root("admission-facts"),
14662 Some(subc_control::ConsumerIdentity {
14663 module_id: "never-spawned".to_string(),
14664 launch_nonce: "any-nonce".to_string(),
14665 }),
14666 None,
14667 ),
14668 )
14669 .await
14670 .unwrap();
14671 assert_eq!(
14672 parse_error(&never_spawned[0])["code"],
14673 "bad_consumer_identity",
14674 "an unspawned module_id must be refused rather than accepted for lack of a record"
14675 );
14676 }
14677
14678 #[tokio::test]
14701 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
14702 let registry = Arc::new(Registry::default());
14703 let forwarding = Arc::new(ForwardingTable::default());
14704 let supervisor = SupervisorHandle::new();
14705 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14706 let handler =
14707 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14708 .with_supervisor(supervisor);
14709
14710 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
14711 hello_via_sink(
14712 &handler,
14713 &target_ctx,
14714 &mut target_rx,
14715 hello_frame("target", PROTOCOL_VERSION, 1),
14716 )
14717 .await;
14718
14719 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
14720 let route_handler = handler.clone();
14721 let route_task = tokio::spawn(async move {
14722 route_handler
14723 .handle_control_frame(
14724 &client_ctx,
14725 route_open_frame_with_admission_facts(
14726 30,
14727 "target",
14728 unique_project_root("admission-facts"),
14729 Some(subc_control::ConsumerIdentity {
14730 module_id: "fed".to_string(),
14731 launch_nonce: "fed-nonce".to_string(),
14732 }),
14733 None,
14734 ),
14735 )
14736 .await
14737 .unwrap()
14738 });
14739
14740 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14747 .await
14748 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
14749 .expect("module control channel closed before route.bind");
14750 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14751 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14752 panic!("expected route.bind")
14753 };
14754 assert_eq!(
14755 principal,
14756 Some(Principal::Reserved {
14757 module_id: "fed".to_string()
14758 }),
14759 "a correctly attested consumer must be stamped Reserved for its own id"
14760 );
14761
14762 handler
14763 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14764 .await
14765 .unwrap();
14766 assert!(route_task.await.unwrap().is_empty());
14767 assert!(
14768 matches!(
14769 serde_json::from_slice::<ClientControlResponse>(
14770 &client_rx.recv().await.unwrap().body
14771 )
14772 .unwrap(),
14773 ClientControlResponse::RouteOpen { .. }
14774 ),
14775 "the route must actually open, not merely avoid an error"
14776 );
14777 }
14778
14779 #[tokio::test(start_paused = true)]
14780 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
14781 let registry = Arc::new(Registry::default());
14782 let forwarding = Arc::new(ForwardingTable::default());
14783 let supervisor = SupervisorHandle::new();
14784 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14785 let handler =
14786 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14787 .with_supervisor(supervisor);
14788
14789 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
14790 hello_via_sink(
14791 &handler,
14792 &target_ctx,
14793 &mut target_rx,
14794 hello_frame("target", PROTOCOL_VERSION, 1),
14795 )
14796 .await;
14797
14798 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
14799 let direct_handler = handler.clone();
14800 let direct_open = tokio::spawn(async move {
14801 direct_handler
14802 .handle_control_frame(
14803 &direct_ctx,
14804 route_open_frame(2, "target", unique_project_root("route-census-direct")),
14805 )
14806 .await
14807 .unwrap()
14808 });
14809 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14810 .await
14811 .expect("no direct route.bind within 5s")
14812 .expect("target control channel closed before direct route.bind");
14813 handler
14814 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
14815 .await
14816 .unwrap();
14817 assert!(direct_open.await.unwrap().is_empty());
14818 let _ = direct_rx.recv().await.unwrap();
14819
14820 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
14821 let reserved_handler = handler.clone();
14822 let reserved_open = tokio::spawn(async move {
14823 reserved_handler
14824 .handle_control_frame(
14825 &reserved_ctx,
14826 route_open_frame_with_admission_facts(
14827 3,
14828 "target",
14829 unique_project_root("admission-facts"),
14830 Some(ConsumerIdentity {
14831 module_id: "fed".to_string(),
14832 launch_nonce: "fed-nonce".to_string(),
14833 }),
14834 None,
14835 ),
14836 )
14837 .await
14838 .unwrap()
14839 });
14840 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14841 .await
14842 .expect("no reserved route.bind within 5s")
14843 .expect("target control channel closed before reserved route.bind");
14844 handler
14845 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
14846 .await
14847 .unwrap();
14848 assert!(reserved_open.await.unwrap().is_empty());
14849 let _ = reserved_rx.recv().await.unwrap();
14850
14851 forwarding
14852 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
14853 .unwrap();
14854 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
14855 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
14856 module_id: Some("target".to_string()),
14857 })
14858 .unwrap();
14859 let census_frame =
14860 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
14861 let response = handler
14862 .handle_control_frame(&census_ctx, census_frame)
14863 .await
14864 .unwrap()
14865 .pop()
14866 .unwrap();
14867 let actual: Value = serde_json::from_slice(&response.body).unwrap();
14868 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
14869 assert!(matches!(
14870 decoded,
14871 ClientControlResponse::SupervisorRoutes { .. }
14872 ));
14873 let routes = actual["modules"][0]["routes"].as_array().unwrap();
14874 assert_eq!(routes.len(), 2);
14875 assert!(routes.iter().all(|route| route["draining"] == true));
14876 assert!(
14879 routes.iter().all(|route| route["drain_reason"] == "reload"),
14880 "draining routes must name the drain's reason: {routes:?}"
14881 );
14882 assert!(routes.iter().any(|route| {
14883 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
14884 }));
14885 assert!(routes.iter().any(|route| {
14886 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
14887 }));
14888
14889 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14890 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
14891 if std::env::var_os("UPDATE_GOLDEN").is_some() {
14892 std::fs::write(
14893 &golden_path,
14894 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
14895 )
14896 .unwrap();
14897 }
14898 let expected: Value =
14899 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
14900 assert_eq!(actual, expected);
14901 }
14902
14903 async fn query_live_roots(
14904 handler: &ControlHandler,
14905 module_ctx: &RouteCtx,
14906 ) -> ModuleControlResponseToModule {
14907 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
14908 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
14909 let response = handler
14910 .handle_control_frame(module_ctx, frame)
14911 .await
14912 .unwrap()
14913 .pop()
14914 .unwrap();
14915 serde_json::from_slice(&response.body).unwrap()
14916 }
14917
14918 #[tokio::test(start_paused = true)]
14919 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
14920 let registry = Arc::new(Registry::default());
14921 let forwarding = Arc::new(ForwardingTable::default());
14922 let handler = ControlHandler::with_forwarding(registry, forwarding);
14923 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
14924 hello_via_sink(
14925 &handler,
14926 &target_ctx,
14927 &mut target_rx,
14928 hello_frame("target", PROTOCOL_VERSION, 1),
14929 )
14930 .await;
14931 let root = unique_project_root("live-roots-known");
14932 let path = ProjectRootId::from_path_allowing_missing(root.path())
14933 .unwrap()
14934 .as_path()
14935 .to_path_buf();
14936 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
14937 let open_handler = handler.clone();
14938 let opened = tokio::spawn(async move {
14939 open_handler
14940 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
14941 .await
14942 .unwrap()
14943 });
14944 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14945 .await
14946 .unwrap()
14947 .unwrap();
14948 handler
14949 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
14950 .await
14951 .unwrap();
14952 assert!(opened.await.unwrap().is_empty());
14953 let _ = client_rx.recv().await.unwrap();
14954
14955 let root = unique_project_root("live-roots-pending");
14956 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
14957 .unwrap()
14958 .as_path()
14959 .to_path_buf();
14960 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
14961 let open_handler = handler.clone();
14962 let pending = tokio::spawn(async move {
14963 open_handler
14964 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
14965 .await
14966 .unwrap()
14967 });
14968 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14969 .await
14970 .unwrap()
14971 .unwrap();
14972 let actual = query_live_roots(&handler, &target_ctx).await;
14973 let ModuleControlResponseToModule::LiveRoots {
14974 roots,
14975 unknown_root_bindings,
14976 total_bindings,
14977 } = actual
14978 else {
14979 panic!("expected live roots")
14980 };
14981 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
14982 assert_eq!(unknown_root_bindings, 0);
14983 assert_eq!(
14984 roots.len(),
14985 2,
14986 "root-known arm must retain each canonical root"
14987 );
14988 assert_eq!(
14989 total_bindings,
14990 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14991 );
14992 let counts = roots
14993 .iter()
14994 .map(|root| (root.project_root.clone(), root.bound, root.pending))
14995 .collect::<Vec<_>>();
14996 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
14997 expected.sort_by(|a, b| a.0.cmp(&b.0));
14998 assert_eq!(
14999 counts, expected,
15000 "roots must sort by path and count pending separately"
15001 );
15002 handler
15003 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
15004 .await
15005 .unwrap();
15006 assert!(pending.await.unwrap().is_empty());
15007 }
15008
15009 #[tokio::test(start_paused = true)]
15010 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
15011 let forwarding = Arc::new(ForwardingTable::default());
15012 let handler =
15013 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
15014 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
15015 hello_via_sink(
15016 &handler,
15017 &target_ctx,
15018 &mut target_rx,
15019 hello_frame("target", PROTOCOL_VERSION, 1),
15020 )
15021 .await;
15022 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
15023 let pending = forwarding
15024 .begin_route_bind_relay_for_test(
15025 client_ctx.connection_id,
15026 client_ctx.egress.clone(),
15027 2,
15028 "target",
15029 )
15030 .unwrap();
15031 forwarding
15032 .complete_pending_relay(
15033 target_ctx.connection_id,
15034 pending.corr,
15035 RouteBindRelayOutcome::Accepted,
15036 )
15037 .unwrap();
15038 let actual = query_live_roots(&handler, &target_ctx).await;
15039 let ModuleControlResponseToModule::LiveRoots {
15040 roots,
15041 unknown_root_bindings,
15042 total_bindings,
15043 } = actual
15044 else {
15045 panic!("expected live roots")
15046 };
15047 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
15048 assert_eq!(
15049 unknown_root_bindings, 1,
15050 "unknown-root arm must not read as no bindings"
15051 );
15052 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
15053 assert_eq!(
15054 total_bindings,
15055 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15056 );
15057 }
15058
15059 #[tokio::test(start_paused = true)]
15067 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
15068 let forwarding = Arc::new(ForwardingTable::default());
15069 let handler =
15070 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
15071 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
15072 let replies = handler
15073 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
15074 .await
15075 .unwrap();
15076 let queued_by_hello = module_rx.len();
15077
15078 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
15079 let open_handler = handler.clone();
15080 let open = tokio::spawn(async move {
15081 open_handler
15082 .handle_control_frame(
15083 &client_ctx,
15084 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
15085 )
15086 .await
15087 .unwrap()
15088 });
15089 let mut spins = 0;
15091 while module_rx.len() == queued_by_hello {
15092 spins += 1;
15093 assert!(spins < 10_000, "route.open never queued a route.bind");
15094 tokio::task::yield_now().await;
15095 }
15096
15097 for reply in replies {
15099 module_ctx.egress.send(reply).await.unwrap();
15100 }
15101
15102 let first = module_rx.recv().await.unwrap().frame;
15103 assert_eq!(
15104 first.header.ty,
15105 FrameType::HelloAck,
15106 "the first frame a registering module reads must be its HELLO_ACK"
15107 );
15108 assert_eq!(first.header.corr, 7);
15109 let second = module_rx.recv().await.unwrap().frame;
15110 assert_eq!(second.header.ty, FrameType::Request);
15111 assert!(
15112 matches!(
15113 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
15114 ModuleControlRequest::RouteBind { .. }
15115 ),
15116 "the route.bind follows the ack"
15117 );
15118 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
15119
15120 handler
15121 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
15122 .await
15123 .unwrap();
15124 assert!(open.await.unwrap().is_empty());
15125 let _ = client_rx.recv().await.unwrap();
15126 }
15127
15128 #[tokio::test(start_paused = true)]
15129 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
15130 let handler = ControlHandler::with_forwarding(
15131 Arc::new(Registry::default()),
15132 Arc::new(ForwardingTable::default()),
15133 );
15134 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
15135 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
15136 hello_via_sink(
15137 &handler,
15138 &first_ctx,
15139 &mut first_rx,
15140 hello_frame("first", PROTOCOL_VERSION, 1),
15141 )
15142 .await;
15143 hello_via_sink(
15144 &handler,
15145 &second_ctx,
15146 &mut second_rx,
15147 hello_frame("second", PROTOCOL_VERSION, 2),
15148 )
15149 .await;
15150 let root = unique_project_root("second-only");
15151 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
15152 let cloned = handler.clone();
15153 let open = tokio::spawn(async move {
15154 cloned
15155 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
15156 .await
15157 .unwrap()
15158 });
15159 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
15160 .await
15161 .unwrap()
15162 .unwrap();
15163 let first = query_live_roots(&handler, &first_ctx).await;
15164 let second = query_live_roots(&handler, &second_ctx).await;
15165 assert!(
15166 matches!(
15167 first,
15168 ModuleControlResponseToModule::LiveRoots {
15169 total_bindings: 0,
15170 ..
15171 }
15172 ),
15173 "cross-module scope must not expose another module's roots"
15174 );
15175 assert!(
15176 matches!(
15177 second,
15178 ModuleControlResponseToModule::LiveRoots {
15179 total_bindings: 1,
15180 ..
15181 }
15182 ),
15183 "second module must see its pending route"
15184 );
15185 handler
15186 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
15187 .await
15188 .unwrap();
15189 assert!(open.await.unwrap().is_empty());
15190 }
15191
15192 #[tokio::test(start_paused = true)]
15193 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
15194 let handler = ControlHandler::with_forwarding(
15195 Arc::new(Registry::default()),
15196 Arc::new(ForwardingTable::default()),
15197 );
15198 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
15199 hello_via_sink(
15200 &handler,
15201 &target_ctx,
15202 &mut target_rx,
15203 hello_frame("target", PROTOCOL_VERSION, 1),
15204 )
15205 .await;
15206 let actual = query_live_roots(&handler, &target_ctx).await;
15207 let ModuleControlResponseToModule::LiveRoots {
15208 roots,
15209 unknown_root_bindings,
15210 total_bindings,
15211 } = actual
15212 else {
15213 panic!("expected live roots")
15214 };
15215 assert!(roots.is_empty());
15216 assert_eq!(unknown_root_bindings, 0);
15217 assert_eq!(total_bindings, 0);
15218 assert_eq!(
15219 total_bindings,
15220 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15221 );
15222 }
15223
15224 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
15231 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
15232 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
15233 let text = std::fs::read_to_string(&path)
15234 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
15235 let vectors: Vec<(String, Value)> = text
15236 .lines()
15237 .filter(|line| !line.trim().is_empty())
15238 .map(|line| {
15239 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
15240 let id = entry["corpus_id"]
15241 .as_str()
15242 .expect("every vector carries a corpus_id")
15243 .to_string();
15244 (id, entry["package"].clone())
15245 })
15246 .collect();
15247 assert_eq!(
15251 vectors.len(),
15252 3,
15253 "vendored fed corpus changed size; re-sync from subc-federation"
15254 );
15255
15256 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
15271 let richest = vectors
15272 .iter()
15273 .filter_map(|(_, package)| package.as_object())
15274 .map(|object| {
15275 object
15276 .keys()
15277 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
15278 .count()
15279 })
15280 .max()
15281 .unwrap_or(0);
15282 assert!(
15283 richest >= 2,
15284 "vendored corpus no longer carries a package with unmodelled fields, \
15285 so the relay test can no longer distinguish a verbatim relay from a lossy one"
15286 );
15287
15288 vectors
15289 }
15290
15291 #[tokio::test]
15299 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
15300 for (corpus_id, package) in fed_admission_facts_vectors() {
15301 let registry = Arc::new(Registry::default());
15302 let forwarding = Arc::new(ForwardingTable::default());
15303 let supervisor = SupervisorHandle::new();
15304 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15305 let handler =
15306 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15307 .with_supervisor(supervisor)
15308 .with_admission_facts_config(
15309 Some("fed".to_string()),
15310 Some(vec!["target".to_string()]),
15311 );
15312
15313 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
15314 hello_via_sink(
15315 &handler,
15316 &target_ctx,
15317 &mut target_rx,
15318 hello_frame("target", PROTOCOL_VERSION, 1),
15319 )
15320 .await;
15321
15322 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
15323 let route_handler = handler.clone();
15324 let expected = package.clone();
15325 let route_task = tokio::spawn(async move {
15326 route_handler
15327 .handle_control_frame(
15328 &client_ctx,
15329 route_open_frame_with_admission_facts(
15330 20,
15331 "target",
15332 unique_project_root("admission-facts"),
15333 Some(subc_control::ConsumerIdentity {
15334 module_id: "fed".to_string(),
15335 launch_nonce: "fed-nonce".to_string(),
15336 }),
15337 Some(package),
15338 ),
15339 )
15340 .await
15341 .unwrap()
15342 });
15343
15344 let bind_frame = target_rx.recv().await.unwrap();
15345 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15346 let ModuleControlRequest::RouteBind {
15347 admission_facts, ..
15348 } = bind
15349 else {
15350 panic!("{corpus_id}: expected route.bind")
15351 };
15352 assert_eq!(
15353 admission_facts,
15354 Some(expected),
15355 "{corpus_id}: relay must not add, drop or reshape any field"
15356 );
15357
15358 handler
15359 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15360 .await
15361 .unwrap();
15362 route_task.await.unwrap();
15363 }
15364 }
15365
15366 #[tokio::test]
15367 async fn route_bind_never_relays_the_openers_build_class() {
15368 let registry = Arc::new(Registry::default());
15369 let forwarding = Arc::new(ForwardingTable::default());
15370 let handler =
15371 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15372 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
15373 hello_via_sink(
15374 &handler,
15375 &target_ctx,
15376 &mut target_rx,
15377 hello_frame("target", PROTOCOL_VERSION, 1),
15378 )
15379 .await;
15380
15381 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
15382 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
15383 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15384 busy_refusals: None,
15385 target: RouteTarget::ToolProvider {
15386 module_id: "target".to_string(),
15387 },
15388 identity,
15389 consumer_identity: None,
15390 consumer_capabilities: None,
15391 role_versions: None,
15392 admission_facts: None,
15393 scope: None,
15394 })
15395 .unwrap();
15396 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
15397
15398 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
15399 let route_handler = handler.clone();
15400 let route_task = tokio::spawn(async move {
15401 route_handler
15402 .handle_control_frame(&client_ctx, open)
15403 .await
15404 .unwrap()
15405 });
15406 let bind_frame = target_rx.recv().await.unwrap();
15407 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15408 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
15409 panic!("expected route.bind")
15410 };
15411 assert_eq!(
15412 identity.build_class, None,
15413 "route.bind carried the build class"
15414 );
15415 assert!(
15416 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
15417 "the route.bind bytes still name build_class"
15418 );
15419 handler
15420 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15421 .await
15422 .unwrap();
15423 route_task.await.unwrap();
15424 }
15425
15426 #[tokio::test]
15427 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
15428 let registry = Arc::new(Registry::default());
15429 let forwarding = Arc::new(ForwardingTable::default());
15430 let supervisor = SupervisorHandle::new();
15431 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15432 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
15433 let handler =
15434 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15435 .with_supervisor(supervisor)
15436 .with_admission_facts_config(
15437 Some("fed".to_string()),
15438 Some(vec!["target".to_string()]),
15439 );
15440
15441 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
15442 hello_via_sink(
15443 &handler,
15444 &target_ctx,
15445 &mut target_rx,
15446 hello_frame("target", PROTOCOL_VERSION, 1),
15447 )
15448 .await;
15449 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
15450 hello_via_sink(
15451 &handler,
15452 &other_ctx,
15453 &mut other_rx,
15454 hello_frame("other", PROTOCOL_VERSION, 2),
15455 )
15456 .await;
15457
15458 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
15459 let expected_facts = facts.clone();
15460 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
15461 let route_handler = handler.clone();
15462 let route_task = tokio::spawn(async move {
15463 route_handler
15464 .handle_control_frame(
15465 &client_ctx,
15466 route_open_frame_with_admission_facts(
15467 10,
15468 "target",
15469 unique_project_root("admission-facts"),
15470 Some(subc_control::ConsumerIdentity {
15471 module_id: "fed".to_string(),
15472 launch_nonce: "fed-nonce".to_string(),
15473 }),
15474 Some(facts.clone()),
15475 ),
15476 )
15477 .await
15478 .unwrap()
15479 });
15480 let bind_frame = target_rx.recv().await.unwrap();
15481 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15482 let ModuleControlRequest::RouteBind {
15483 admission_facts, ..
15484 } = bind
15485 else {
15486 panic!("expected route.bind")
15487 };
15488 assert_eq!(admission_facts, Some(expected_facts));
15489 handler
15490 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15491 .await
15492 .unwrap();
15493 assert!(route_task.await.unwrap().is_empty());
15494 assert!(matches!(
15495 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
15496 .unwrap(),
15497 ClientControlResponse::RouteOpen { .. }
15498 ));
15499
15500 let direct = handler
15501 .handle_control_frame(
15502 &route_ctx(ConnectionId::new(73)).0,
15503 route_open_frame_with_admission_facts(
15504 11,
15505 "target",
15506 unique_project_root("admission-facts"),
15507 None,
15508 Some(json!({"x": 1})),
15509 ),
15510 )
15511 .await
15512 .unwrap();
15513 assert_eq!(
15514 parse_error(&direct[0])["code"],
15515 "admission_facts_not_permitted"
15516 );
15517
15518 let different_reserved = handler
15519 .handle_control_frame(
15520 &route_ctx(ConnectionId::new(77)).0,
15521 route_open_frame_with_admission_facts(
15522 15,
15523 "target",
15524 unique_project_root("admission-facts"),
15525 Some(subc_control::ConsumerIdentity {
15526 module_id: "other".to_string(),
15527 launch_nonce: "other-nonce".to_string(),
15528 }),
15529 Some(json!({"x": 1})),
15530 ),
15531 )
15532 .await
15533 .unwrap();
15534 assert_eq!(
15535 parse_error(&different_reserved[0])["code"],
15536 "admission_facts_not_permitted"
15537 );
15538
15539 let other_target = handler
15540 .handle_control_frame(
15541 &route_ctx(ConnectionId::new(74)).0,
15542 route_open_frame_with_admission_facts(
15543 12,
15544 "other",
15545 unique_project_root("admission-facts"),
15546 Some(subc_control::ConsumerIdentity {
15547 module_id: "fed".to_string(),
15548 launch_nonce: "fed-nonce".to_string(),
15549 }),
15550 Some(json!({"x": 1})),
15551 ),
15552 )
15553 .await
15554 .unwrap();
15555 assert_eq!(
15556 parse_error(&other_target[0])["code"],
15557 "admission_facts_target_not_allowed"
15558 );
15559
15560 let nonexistent = handler
15561 .handle_control_frame(
15562 &route_ctx(ConnectionId::new(75)).0,
15563 route_open_frame_with_admission_facts(
15564 13,
15565 "missing",
15566 unique_project_root("admission-facts"),
15567 None,
15568 Some(json!({"x": 1})),
15569 ),
15570 )
15571 .await
15572 .unwrap();
15573 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
15574
15575 let described = handler
15576 .handle_control_frame(
15577 &route_ctx(ConnectionId::new(76)).0,
15578 Frame::build(
15579 FrameType::Request,
15580 control_flags(),
15581 0,
15582 0,
15583 14,
15584 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15585 )
15586 .unwrap(),
15587 )
15588 .await
15589 .unwrap();
15590 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15591 serde_json::from_slice(&described[0].body).unwrap()
15592 else {
15593 panic!("expected server.describe response")
15594 };
15595 assert!(capabilities
15596 .iter()
15597 .any(|cap| cap == "admission_facts_relay_v1"));
15598 }
15599
15600 #[tokio::test]
15601 async fn admission_facts_without_configured_carrier_are_rejected() {
15602 let registry = Arc::new(Registry::default());
15603 let forwarding = Arc::new(ForwardingTable::default());
15604 let handler = ControlHandler::with_forwarding(registry, forwarding);
15605 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
15606 hello_via_sink(
15607 &handler,
15608 &target_ctx,
15609 &mut target_rx,
15610 hello_frame("target", PROTOCOL_VERSION, 1),
15611 )
15612 .await;
15613
15614 let responses = handler
15615 .handle_control_frame(
15616 &route_ctx(ConnectionId::new(79)).0,
15617 route_open_frame_with_admission_facts(
15618 16,
15619 "target",
15620 unique_project_root("admission-facts"),
15621 None,
15622 Some(json!({"x": 1})),
15623 ),
15624 )
15625 .await
15626 .unwrap();
15627 assert_eq!(
15628 parse_error(&responses[0])["code"],
15629 "admission_facts_not_permitted"
15630 );
15631 }
15632
15633 #[tokio::test]
15634 async fn route_open_relays_consumer_capabilities_verbatim() {
15635 let registry = Arc::new(Registry::default());
15636 let forwarding = Arc::new(ForwardingTable::default());
15637 let handler =
15638 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15639 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
15640 hello_via_sink(
15641 &handler,
15642 &module_ctx,
15643 &mut module_rx,
15644 hello_frame("aft", PROTOCOL_VERSION, 7),
15645 )
15646 .await;
15647
15648 let expected = vec!["elicitation".to_string(), "roots".to_string()];
15649 let expected_for_request = expected.clone();
15650 let project_root = unique_project_root("consumer-capabilities-present");
15651 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
15652 let route_handler = handler.clone();
15653 let route_task = tokio::spawn(async move {
15654 route_handler
15655 .handle_control_frame(
15656 &client_ctx,
15657 route_open_frame_with_consumer_capabilities(
15658 401,
15659 "aft",
15660 project_root,
15661 Some(expected_for_request),
15662 ),
15663 )
15664 .await
15665 .unwrap()
15666 });
15667 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15668 .await
15669 .unwrap()
15670 .unwrap();
15671 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15672 let ModuleControlRequest::RouteBind {
15673 consumer_capabilities,
15674 ..
15675 } = bind
15676 else {
15677 panic!("expected route.bind request, got {bind:?}");
15678 };
15679 assert_eq!(consumer_capabilities, Some(expected.clone()));
15680
15681 handler
15682 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15683 .await
15684 .unwrap();
15685 let route_response = route_task.await.unwrap();
15686 assert!(route_response.is_empty());
15687 let published = client_rx.recv().await.unwrap();
15688 assert!(matches!(
15689 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15690 ClientControlResponse::RouteOpen { .. }
15691 ));
15692 }
15693
15694 #[tokio::test]
15695 async fn route_open_without_consumer_capabilities_relays_none() {
15696 let registry = Arc::new(Registry::default());
15697 let forwarding = Arc::new(ForwardingTable::default());
15698 let handler =
15699 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15700 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
15701 hello_via_sink(
15702 &handler,
15703 &module_ctx,
15704 &mut module_rx,
15705 hello_frame("aft", PROTOCOL_VERSION, 7),
15706 )
15707 .await;
15708
15709 let project_root = unique_project_root("consumer-capabilities-absent");
15710 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
15711 let route_handler = handler.clone();
15712 let route_task = tokio::spawn(async move {
15713 route_handler
15714 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
15715 .await
15716 .unwrap()
15717 });
15718 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15719 .await
15720 .unwrap()
15721 .unwrap();
15722 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15723 let ModuleControlRequest::RouteBind {
15724 consumer_capabilities,
15725 ..
15726 } = bind
15727 else {
15728 panic!("expected route.bind request, got {bind:?}");
15729 };
15730 assert_eq!(consumer_capabilities, None);
15731
15732 handler
15733 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15734 .await
15735 .unwrap();
15736 let route_response = route_task.await.unwrap();
15737 assert!(route_response.is_empty());
15738 let published = client_rx.recv().await.unwrap();
15739 assert!(matches!(
15740 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15741 ClientControlResponse::RouteOpen { .. }
15742 ));
15743 }
15744
15745 async fn bind_role_versions_for(
15749 sent: Option<BTreeMap<String, String>>,
15750 connection: u64,
15751 ) -> Option<BTreeMap<String, String>> {
15752 let registry = Arc::new(Registry::default());
15753 let forwarding = Arc::new(ForwardingTable::default());
15754 let handler =
15755 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15756 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
15757 hello_via_sink(
15758 &handler,
15759 &module_ctx,
15760 &mut module_rx,
15761 hello_frame("aft", PROTOCOL_VERSION, 7),
15762 )
15763 .await;
15764 let project_root = unique_project_root("role-versions");
15765 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
15766 let route_handler = handler.clone();
15767 let route_task = tokio::spawn(async move {
15768 route_handler
15769 .handle_control_frame(
15770 &client_ctx,
15771 route_open_frame_with_role_versions(403, "aft", project_root, sent),
15772 )
15773 .await
15774 .unwrap()
15775 });
15776 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15777 .await
15778 .expect("a well-formed route.open reaches the module as a bind")
15779 .unwrap();
15780 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15781 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
15782 panic!("expected route.bind request, got {bind:?}");
15783 };
15784 handler
15785 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15786 .await
15787 .unwrap();
15788 assert!(route_task.await.unwrap().is_empty());
15789 let published = client_rx.recv().await.unwrap();
15790 assert!(matches!(
15791 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15792 ClientControlResponse::RouteOpen { .. }
15793 ));
15794 role_versions
15795 }
15796
15797 #[tokio::test]
15798 async fn route_open_relays_role_versions_verbatim() {
15799 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
15800 assert_eq!(
15801 bind_role_versions_for(Some(sent.clone()), 141).await,
15802 Some(sent)
15803 );
15804 }
15805
15806 #[tokio::test]
15809 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
15810 assert_eq!(bind_role_versions_for(None, 143).await, None);
15811 assert_eq!(
15812 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
15813 None
15814 );
15815 }
15816
15817 #[tokio::test]
15818 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
15819 let registry = Arc::new(Registry::default());
15820 let forwarding = Arc::new(ForwardingTable::default());
15821 let handler =
15822 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15823 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
15824 hello_via_sink(
15825 &handler,
15826 &module_ctx,
15827 &mut module_rx,
15828 hello_frame("aft", PROTOCOL_VERSION, 7),
15829 )
15830 .await;
15831
15832 let nine: BTreeMap<String, String> = (0..9)
15833 .map(|index| (format!("role-{index}"), "v1".to_string()))
15834 .collect();
15835 for (label, malformed) in [
15836 (
15837 "invalid role name",
15838 role_versions(&[("Tool_Provider", "v1")]),
15839 ),
15840 ("invalid version", role_versions(&[("tool-provider", "v0")])),
15841 ("nine entries", nine),
15842 ] {
15843 let responses = tokio::time::timeout(
15846 Duration::from_secs(1),
15847 handler.handle_control_frame(
15848 &route_ctx(ConnectionId::new(148)).0,
15849 route_open_frame_with_role_versions(
15850 404,
15851 "aft",
15852 unique_project_root("role-versions-malformed"),
15853 Some(malformed),
15854 ),
15855 ),
15856 )
15857 .await
15858 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
15859 .unwrap();
15860 assert_eq!(responses.len(), 1, "{label}");
15861 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
15862 let error = parse_error(&responses[0]);
15863 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
15864 assert_eq!(
15865 error["detail"]["field"], "role_versions",
15866 "{label}: {error}"
15867 );
15868 assert!(
15869 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
15870 "{label}: a malformed declaration is terminal"
15871 );
15872 assert!(
15873 module_rx.try_recv().is_err(),
15874 "{label}: the module must never see a bind"
15875 );
15876 }
15877 }
15878
15879 #[tokio::test]
15883 async fn route_role_versions_capability_is_advertised() {
15884 let handler = ControlHandler::new(Arc::new(Registry::default()));
15885 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
15886 let ack = hello_via_sink(
15887 &handler,
15888 &ctx,
15889 &mut rx,
15890 hello_frame("m", PROTOCOL_VERSION, 1),
15891 )
15892 .await;
15893 let ack = parse_ack(&ack);
15894 assert!(
15895 ack.subc_capabilities
15896 .iter()
15897 .any(|c| c == "route-role-versions/v1"),
15898 "{:?}",
15899 ack.subc_capabilities
15900 );
15901
15902 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15903 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15904 let reply = handler
15905 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
15906 .await
15907 .unwrap()
15908 .pop()
15909 .unwrap();
15910 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15911 serde_json::from_slice(&reply.body).unwrap()
15912 else {
15913 panic!("not a server.describe reply");
15914 };
15915 assert!(
15916 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
15917 "{capabilities:?}"
15918 );
15919 }
15920
15921 #[tokio::test]
15925 async fn catalog_events_capability_is_advertised() {
15926 let handler = ControlHandler::new(Arc::new(Registry::default()));
15927 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15928 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15929 let reply = handler
15930 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
15931 .await
15932 .unwrap()
15933 .pop()
15934 .unwrap();
15935 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15936 serde_json::from_slice(&reply.body).unwrap()
15937 else {
15938 panic!("not a server.describe reply");
15939 };
15940 assert!(
15941 capabilities
15942 .iter()
15943 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
15944 "{capabilities:?}"
15945 );
15946 }
15947
15948 #[tokio::test]
15949 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
15950 let handler = ControlHandler::new(Arc::new(Registry::default()));
15951 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
15952 let ack = hello_via_sink(
15953 &handler,
15954 &module,
15955 &mut module_rx,
15956 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
15957 )
15958 .await;
15959 let ack = parse_ack(&ack);
15960 assert!(
15961 ack.subc_capabilities
15962 .iter()
15963 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15964 "{:?}",
15965 ack.subc_capabilities
15966 );
15967 assert!(ack
15968 .subc_capabilities
15969 .iter()
15970 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15971
15972 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
15973 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15974 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15975 let reply = handler
15976 .handle_control_frame(&client, frame)
15977 .await
15978 .unwrap()
15979 .pop()
15980 .unwrap();
15981 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15982 serde_json::from_slice(&reply.body).unwrap()
15983 else {
15984 panic!("not a server.describe reply");
15985 };
15986 assert!(
15987 capabilities
15988 .iter()
15989 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15990 "{capabilities:?}"
15991 );
15992 assert!(capabilities
15993 .iter()
15994 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15995 }
15996
15997 #[tokio::test]
15998 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
15999 let handler = ControlHandler::new(Arc::new(Registry::default()));
16000 let (module, mut rx) = route_ctx(ConnectionId::new(156));
16001 let ack = hello_via_sink(
16002 &handler,
16003 &module,
16004 &mut rx,
16005 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
16006 )
16007 .await;
16008 assert!(parse_ack(&ack)
16009 .subc_capabilities
16010 .iter()
16011 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
16012 assert!(parse_ack(&ack)
16013 .subc_capabilities
16014 .iter()
16015 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
16016 let (client, _rx) = route_ctx(ConnectionId::new(157));
16017 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16018 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16019 let reply = handler
16020 .handle_control_frame(&client, frame)
16021 .await
16022 .unwrap()
16023 .pop()
16024 .unwrap();
16025 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16026 serde_json::from_slice(&reply.body).unwrap()
16027 else {
16028 panic!("server.describe reply")
16029 };
16030 assert!(capabilities
16031 .iter()
16032 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
16033 assert!(capabilities
16034 .iter()
16035 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
16036 }
16037
16038 #[tokio::test]
16039 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
16040 let handler = ControlHandler::new(Arc::new(Registry::default()));
16041 let (module, mut rx) = route_ctx(ConnectionId::new(158));
16042 let ack = hello_via_sink(
16043 &handler,
16044 &module,
16045 &mut rx,
16046 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
16047 )
16048 .await;
16049 assert!(parse_ack(&ack)
16050 .subc_capabilities
16051 .iter()
16052 .any(|cap| cap == "scope-workdir/v1"));
16053 let (client, _rx) = route_ctx(ConnectionId::new(159));
16054 let frame = Frame::build(
16055 FrameType::Request,
16056 control_flags(),
16057 0,
16058 0,
16059 5,
16060 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16061 )
16062 .unwrap();
16063 let reply = handler
16064 .handle_control_frame(&client, frame)
16065 .await
16066 .unwrap()
16067 .remove(0);
16068 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16069 serde_json::from_slice(&reply.body).unwrap()
16070 else {
16071 panic!("server.describe reply")
16072 };
16073 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
16074 }
16075
16076 #[tokio::test]
16077 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
16078 let handler = ControlHandler::new(Arc::new(Registry::default()));
16079 let (module, mut rx) = route_ctx(ConnectionId::new(160));
16080 let ack = hello_via_sink(
16081 &handler,
16082 &module,
16083 &mut rx,
16084 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
16085 )
16086 .await;
16087 assert!(parse_ack(&ack)
16088 .subc_capabilities
16089 .iter()
16090 .any(|cap| cap == "worker-runs/v1"));
16091 let (client, _rx) = route_ctx(ConnectionId::new(161));
16092 let frame = Frame::build(
16093 FrameType::Request,
16094 control_flags(),
16095 0,
16096 0,
16097 5,
16098 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16099 )
16100 .unwrap();
16101 let reply = handler
16102 .handle_control_frame(&client, frame)
16103 .await
16104 .unwrap()
16105 .remove(0);
16106 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16107 serde_json::from_slice(&reply.body).unwrap()
16108 else {
16109 panic!("server.describe reply")
16110 };
16111 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
16112 }
16113
16114 #[tokio::test]
16115 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
16116 let handler = ControlHandler::new(Arc::new(Registry::default()));
16117 let (module, mut rx) = route_ctx(ConnectionId::new(154));
16118 let ack = hello_via_sink(
16119 &handler,
16120 &module,
16121 &mut rx,
16122 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
16123 )
16124 .await;
16125 assert!(parse_ack(&ack)
16126 .subc_capabilities
16127 .iter()
16128 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16129 let (client, _rx) = route_ctx(ConnectionId::new(155));
16130 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16131 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16132 let reply = handler
16133 .handle_control_frame(&client, frame)
16134 .await
16135 .unwrap()
16136 .pop()
16137 .unwrap();
16138 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16139 serde_json::from_slice(&reply.body).unwrap()
16140 else {
16141 panic!("not a server.describe reply")
16142 };
16143 assert!(capabilities
16144 .iter()
16145 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16146 }
16147
16148 #[tokio::test]
16149 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
16150 let registry = Arc::new(Registry::default());
16151 let forwarding = Arc::new(ForwardingTable::default());
16152 let handler =
16153 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16154 .with_health_probe_timeout(Duration::from_secs(5));
16155 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
16156 hello_via_sink(
16157 &handler,
16158 &module_ctx,
16159 &mut module_rx,
16160 non_routable_hello_frame_with_control_ops(
16161 "mcp",
16162 300,
16163 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16164 ),
16165 )
16166 .await;
16167 assert!(registry
16168 .get_module("mcp")
16169 .unwrap()
16170 .unwrap()
16171 .manifest
16172 .provides
16173 .is_empty());
16174
16175 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
16176 let route_response = handler
16177 .handle_control_frame(
16178 &route_client_ctx,
16179 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
16180 )
16181 .await
16182 .unwrap();
16183 assert_eq!(route_response[0].header.ty, FrameType::Error);
16184 assert_eq!(
16185 parse_error(&route_response[0])["code"],
16186 "target_unavailable"
16187 );
16188 assert!(parse_error(&route_response[0])["message"]
16189 .as_str()
16190 .unwrap()
16191 .contains("does not provide the requested target"));
16192 assert!(module_rx.try_recv().is_err());
16193
16194 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
16195 let health_handler = handler.clone();
16196 let health_task = tokio::spawn(async move {
16197 health_handler
16198 .handle_control_frame(
16199 &health_client_ctx,
16200 supervisor_health_probe_frame(302, "mcp"),
16201 )
16202 .await
16203 .unwrap()
16204 });
16205 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16206 .await
16207 .unwrap()
16208 .unwrap();
16209 assert_eq!(
16210 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
16211 ModuleControlRequest::HealthCheck {}
16212 );
16213 handler
16214 .handle_control_frame(
16215 &module_ctx,
16216 health_response(health_frame.header.corr, HealthStatus::Ok),
16217 )
16218 .await
16219 .unwrap();
16220 let health_response = health_task.await.unwrap();
16221 assert_eq!(health_response[0].header.ty, FrameType::Response);
16222 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
16223 ClientControlResponse::SupervisorHealthProbe {
16224 module_id, status, ..
16225 } => {
16226 assert_eq!(module_id, "mcp");
16227 assert_eq!(status, HealthStatus::Ok);
16228 }
16229 other => panic!("unexpected health response: {other:?}"),
16230 }
16231
16232 forwarding
16237 .cleanup_connection(module_ctx.connection_id)
16238 .unwrap();
16239 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
16240 let cleanup_response = tokio::time::timeout(
16241 Duration::from_millis(200),
16242 handler.handle_control_frame(
16243 &cleanup_probe_ctx,
16244 supervisor_health_probe_frame(303, "mcp"),
16245 ),
16246 )
16247 .await
16248 .expect("probe should fail immediately when the control lane is gone")
16249 .unwrap();
16250 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
16251 assert_eq!(
16252 parse_error(&cleanup_response[0])["code"],
16253 "target_unavailable"
16254 );
16255 assert!(parse_error(&cleanup_response[0])["message"]
16256 .as_str()
16257 .unwrap()
16258 .contains("no module connection"));
16259
16260 handler
16261 .cleanup_connection(module_ctx.connection_id)
16262 .unwrap();
16263 }
16264
16265 #[tokio::test]
16266 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
16267 let registry = Arc::new(Registry::default());
16268 let supervisor_handle = SupervisorHandle::new();
16269 let supervisor =
16270 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
16271 .with_handle(supervisor_handle.clone())
16272 .with_connection_file_path(
16273 std::env::temp_dir()
16274 .join(format!("subc-route-open-warming-{}", std::process::id())),
16275 );
16276 let module = supervisor
16277 .supervise_configured(
16278 ModuleSpec {
16279 script: None,
16280 carry_containment: false,
16281 module_id: "warming".to_string(),
16282 program: fake_aft_stub_path(),
16283 args: Vec::new(),
16284 env: Vec::new(),
16285 reserved: false,
16286 reserved_prefixes: Vec::new(),
16287 protocol: ModuleProtocol::Subc,
16288 overlap: Default::default(),
16289 },
16290 true,
16291 )
16292 .unwrap();
16293 assert_eq!(module.state().unwrap(), ModuleState::Running);
16294
16295 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16296 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
16297 let response = handler
16298 .handle_control_frame(
16299 &ctx,
16300 route_open_frame(304, "warming", unique_project_root("warming")),
16301 )
16302 .await
16303 .unwrap();
16304 module.stop().await.unwrap();
16305
16306 assert_eq!(response[0].header.ty, FrameType::Error);
16307 let error = parse_error(&response[0]);
16308 assert_eq!(error["code"], "module_warming");
16309 assert!(error["message"]
16310 .as_str()
16311 .unwrap()
16312 .contains("state=running, enabled=true, live=false"));
16313 }
16314
16315 #[test]
16316 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
16317 let handler = ControlHandler::new(Arc::new(Registry::default()));
16318 let capture = EventCapture::default();
16319 let _subscriber =
16320 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16321 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
16322 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
16323 let pending = (0..limit).collect::<Vec<_>>();
16324 let response = handler
16325 .route_open_capacity_refusal(
16326 &ctx,
16327 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
16328 "busy",
16329 pending.len(),
16330 limit,
16331 )
16332 .unwrap();
16333 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16334 let event = capture
16335 .events()
16336 .into_iter()
16337 .find(|event| {
16338 event.target == "control"
16339 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
16340 })
16341 .expect("connection admission refusal event");
16342 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16343 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16344 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
16347 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
16348 }
16349
16350 #[test]
16351 fn route_open_target_cap_logs_admission_reason_and_capacity() {
16352 let handler = ControlHandler::new(Arc::new(Registry::default()));
16353 let capture = EventCapture::default();
16354 let _subscriber =
16355 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16356 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
16357 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
16358 let guards = (0..limit)
16359 .map(|_| {
16360 handler
16361 .route_bind_concurrency
16362 .try_admit("busy", limit)
16363 .unwrap()
16364 })
16365 .collect::<Vec<_>>();
16366 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
16367 Err(in_flight) => in_flight,
16368 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
16369 };
16370 let response = handler
16371 .route_open_target_capacity_refusal(
16372 &ctx,
16373 &RouteOpenCaller::unknown(),
16374 &route_open_frame(397, "busy", unique_project_root("target-cap")),
16375 "busy",
16376 in_flight,
16377 )
16378 .unwrap();
16379 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16380 let event = capture
16381 .events()
16382 .into_iter()
16383 .find(|event| {
16384 event.target == "control"
16385 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
16386 })
16387 .expect("target admission refusal event");
16388 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16389 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16390 drop(guards);
16391 }
16392
16393 #[cfg(unix)]
16394 #[tokio::test(start_paused = true)]
16395 async fn supervisor_list_shows_a_failed_modules_next_retry() {
16396 let registry = Arc::new(Registry::default());
16397 let handle = SupervisorHandle::new();
16398 let supervisor =
16399 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
16400 .with_handle(handle.clone());
16401 let module = supervisor
16402 .spawn(crate::ModuleSpec {
16403 script: None,
16404 carry_containment: false,
16405 module_id: "retry-list".into(),
16406 program: PathBuf::from("/bin/sh"),
16407 args: vec!["-c".into(), "exit 17".into()],
16408 env: vec![],
16409 reserved: false,
16410 reserved_prefixes: vec![],
16411 protocol: ModuleProtocol::None,
16412 overlap: Default::default(),
16413 })
16414 .unwrap();
16415 let deadline = std::time::Instant::now() + Duration::from_secs(10);
16416 while module.state().unwrap() != crate::ModuleState::Failed {
16417 assert!(
16418 std::time::Instant::now() < deadline,
16419 "waiting for budget failure"
16420 );
16421 tokio::task::yield_now().await;
16422 }
16423 let handler = ControlHandler::new(registry).with_supervisor(handle);
16424 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16425 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16426 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16427 assert_eq!(body["modules"][0]["state"], "failed");
16428 let retry = body["modules"][0]["next_retry_at_ms"]
16429 .as_u64()
16430 .expect("scheduled retry on the wire");
16431 let now = std::time::SystemTime::now()
16432 .duration_since(std::time::UNIX_EPOCH)
16433 .unwrap()
16434 .as_millis() as u64;
16435 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
16436 module.set_enabled(false).await.unwrap();
16437 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16438 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16439 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16440 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
16441 }
16442
16443 #[tokio::test]
16444 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
16445 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
16446 let supervisor =
16447 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
16448 let module = supervisor
16449 .spawn(crate::ModuleSpec {
16450 script: None,
16451 carry_containment: false,
16452 module_id: "pid-read-budget".into(),
16453 program: std::env::current_exe().unwrap(),
16454 args: vec![
16455 "--exact".into(),
16456 "control::tests::pid_resource_read_fixture".into(),
16457 ],
16458 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16459 .into_iter()
16460 .map(|key| (key.into(), dir.display().to_string()))
16461 .chain(std::iter::once((
16462 "SUBC_SUBTREE_PID_FIXTURE".into(),
16463 "1".into(),
16464 )))
16465 .collect(),
16466 reserved: false,
16467 reserved_prefixes: Vec::new(),
16468 protocol: ModuleProtocol::None,
16469 overlap: Default::default(),
16470 })
16471 .unwrap();
16472 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16473 while !matches!(
16474 module.child_resource_usage(),
16475 subc_control::ChildResourceUsage::Measured(_)
16476 ) {
16477 assert!(
16478 tokio::time::Instant::now() < ready_deadline,
16479 "fixture process was not measurable"
16480 );
16481 tokio::time::sleep(Duration::from_millis(10)).await;
16482 }
16483 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
16484 let reading = {
16485 let _guard = crate::subtree_resources::fault_read_guard().await;
16486 crate::subtree_resources::read_process(module.clone()).await
16487 };
16488 stop_resource_fixture(&module, &dir).await;
16489 assert!(
16490 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
16491 "{reading:?}"
16492 );
16493 }
16494
16495 #[test]
16496 fn pid_resource_read_fixture() {
16497 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
16498 let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
16499 let deadline = std::time::Instant::now() + Duration::from_secs(60);
16500 while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
16501 std::thread::sleep(Duration::from_millis(5));
16502 }
16503 std::fs::write(dir.join("module-stopped"), "").unwrap();
16504 }
16505 }
16506
16507 async fn stop_resource_fixture(
16508 module: &crate::supervise::SupervisedModule,
16509 dir: &std::path::Path,
16510 ) {
16511 let stopping = module.stop();
16512 tokio::pin!(stopping);
16513 tokio::select! {
16514 result = &mut stopping => result.unwrap(),
16515 () = async {
16516 while module.state().unwrap() != crate::supervise::ModuleState::Draining {
16517 tokio::time::sleep(Duration::from_millis(5)).await;
16518 }
16519 std::fs::write(dir.join("stop-module"), "").unwrap();
16520 } => stopping.await.unwrap(),
16521 }
16522 #[cfg(windows)]
16523 assert!(
16524 dir.join("module-stopped").exists(),
16525 "the resource fixture must exit cooperatively, not wait for the drain kill"
16526 );
16527 assert_eq!(
16528 module.state().unwrap(),
16529 crate::supervise::ModuleState::Stopped
16530 );
16531 }
16532
16533 #[tokio::test]
16534 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
16535 let registry = Arc::new(Registry::default());
16536 let handle = SupervisorHandle::new();
16537 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16538 .with_handle(handle.clone());
16539 supervisor
16540 .supervise_configured(
16541 crate::ModuleSpec {
16542 script: None,
16543 module_id: "disabled-resource-reading".into(),
16544 program: std::env::current_exe().unwrap(),
16545 args: Vec::new(),
16546 env: Vec::new(),
16547 reserved: false,
16548 reserved_prefixes: Vec::new(),
16549 protocol: ModuleProtocol::None,
16550 overlap: Default::default(),
16551 carry_containment: false,
16552 },
16553 false,
16554 )
16555 .unwrap();
16556 let handler = ControlHandler::new(registry).with_supervisor(handle);
16557 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16558 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16559 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16560 assert!(body["modules"][0].get("tree").is_none());
16561 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
16562 }
16563
16564 #[tokio::test]
16565 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
16566 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
16567 let registry = Arc::new(Registry::default());
16568 let handle = SupervisorHandle::new();
16569 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16570 .with_handle(handle.clone());
16571 let module = supervisor
16572 .spawn(crate::ModuleSpec {
16573 script: None,
16574 carry_containment: false,
16575 module_id: "list-process-priority".into(),
16576 program: std::env::current_exe().unwrap(),
16577 args: vec![
16578 "--exact".into(),
16579 "control::tests::pid_resource_read_fixture".into(),
16580 ],
16581 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16582 .into_iter()
16583 .map(|key| (key.into(), dir.display().to_string()))
16584 .chain(std::iter::once((
16585 "SUBC_SUBTREE_PID_FIXTURE".into(),
16586 "1".into(),
16587 )))
16588 .collect(),
16589 reserved: false,
16590 reserved_prefixes: Vec::new(),
16591 protocol: ModuleProtocol::None,
16592 overlap: Default::default(),
16593 })
16594 .unwrap();
16595 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16596 loop {
16597 let image = module.running_image_agreement().await;
16598 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
16599 break;
16600 }
16601 assert!(
16602 tokio::time::Instant::now() < ready_deadline,
16603 "the fixture process never became observable: {image:?}"
16604 );
16605 tokio::time::sleep(Duration::from_millis(10)).await;
16606 }
16607 let guard = crate::subtree_resources::fault_read_guard().await;
16608 let (target, release, _) = crate::subtree_resources::held();
16609 module.set_subtree_target_for_test(target);
16610 let handler = ControlHandler::new(registry).with_supervisor(handle);
16611 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16612 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16613 drop(release);
16614 drop(guard);
16615 stop_resource_fixture(&module, &dir).await;
16616 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16617 assert_eq!(
16618 body["modules"][0]["resources"]["status"], "measured",
16619 "the optional tree timeout must not discard the process reading: {body}"
16620 );
16621 assert!(body["modules"][0].get("tree").is_none());
16622 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
16623 assert_eq!(
16624 body["modules"][0]["pending_reload"]["image"]["status"], "match",
16625 "the tree timeout must not hide a completed image observation"
16626 );
16627 }
16628
16629 #[cfg(any(target_os = "linux", target_os = "macos"))]
16630 #[tokio::test]
16631 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
16632 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
16633 .with_daemon_provenance(
16634 std::process::id(),
16635 0,
16636 Some(std::env::current_exe().unwrap()),
16637 None,
16638 None,
16639 );
16640 let (probe, release, began) = ExecutableIdentityProbe::held();
16641 handler.daemon_provenance.probe = probe;
16642 let observe = |handler: ControlHandler, corr| {
16643 tokio::spawn(async move {
16644 let (ctx, _) = route_ctx(ConnectionId::new(corr));
16645 let frame =
16646 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
16647 .unwrap();
16648 handler
16649 .handle_supervisor_provenance(&ctx, frame, None)
16650 .await
16651 .unwrap()
16652 })
16653 };
16654 let explicit = observe(handler.clone(), 114);
16655 began.await.unwrap();
16656 let concurrent = observe(handler.clone(), 115);
16657 tokio::time::sleep(Duration::from_millis(300)).await;
16660 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16661 let list = tokio::time::timeout(
16662 Duration::from_secs(1),
16663 handler.handle_supervisor_list(frame),
16664 )
16665 .await;
16666 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
16667 drop(release);
16668 let replies = explicit.await.unwrap();
16669 let concurrent_replies = concurrent.await.unwrap();
16670 assert!(
16671 explicit_was_pending,
16672 "explicit provenance must await the full observation"
16673 );
16674 let list = list
16675 .expect("a concurrent explicit probe must not stall list")
16676 .unwrap();
16677 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
16678 assert_eq!(list_body["op"], "supervisor.list");
16679 for replies in [replies, concurrent_replies] {
16680 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16681 assert_eq!(
16682 body["daemon"]["daemon_observed"]["running_image"]["status"],
16683 "match"
16684 );
16685 }
16686 }
16687
16688 #[tokio::test]
16689 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
16690 let registry = Arc::new(Registry::default());
16691 let handle = SupervisorHandle::new();
16692 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16693 .with_handle(handle.clone());
16694 let module = supervisor
16695 .supervise_configured(
16696 crate::ModuleSpec {
16697 script: None,
16698 carry_containment: false,
16699 module_id: "memory-read".into(),
16700 program: PathBuf::from("disabled-module"),
16701 args: Vec::new(),
16702 env: Vec::new(),
16703 reserved: false,
16704 reserved_prefixes: Vec::new(),
16705 protocol: ModuleProtocol::Subc,
16706 overlap: Default::default(),
16707 },
16708 true,
16709 )
16710 .unwrap();
16711 let _guard = crate::subtree_resources::fault_read_guard().await;
16712 let (target, release, began) = crate::subtree_resources::held();
16713 let stalled_target = target.clone();
16716 let stalled = tokio::task::spawn_blocking(move || {
16717 stalled_target.tree(&subc_os::ProcessTable::default())
16718 });
16719 began.await.unwrap();
16720 module.set_subtree_target_for_test(target);
16721 let handler = ControlHandler::new(registry).with_supervisor(handle);
16722 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
16723 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16724 let replies = handler
16725 .handle_client_control_request(
16726 &ctx,
16727 frame,
16728 ClientControlRequest::SupervisorList {},
16729 &mut None,
16730 )
16731 .await
16732 .unwrap();
16733 assert!(
16734 replies.is_empty(),
16735 "list reply must be detached from the connection reader"
16736 );
16737 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16738 let replies = handler
16739 .handle_client_control_request(
16740 &ctx,
16741 frame,
16742 ClientControlRequest::SupervisorSpawnSnapshot {},
16743 &mut None,
16744 )
16745 .await
16746 .unwrap();
16747 assert_eq!(
16748 replies.len(),
16749 1,
16750 "another request must complete while the subtree read stalls"
16751 );
16752 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16753 .await
16754 .unwrap()
16755 .unwrap();
16756 release.send(()).unwrap();
16757 drop(release);
16758 stalled.await.unwrap();
16759 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
16760 assert!(
16761 body["modules"][0].get("tree").is_none(),
16762 "an unknown tree reading must be omitted after the bounded read expires"
16763 );
16764 handler
16768 .resource_tracker
16769 .read(handler.supervisor.list(), false, "released_tree_fixture")
16770 .await;
16771 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16772 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16773 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16774 assert_eq!(
16775 body["modules"][0]["tree"]["source"], "process_tree",
16776 "the next request must export the completed refresh"
16777 );
16778 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
16779 }
16780
16781 #[tokio::test]
16782 async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
16783 use subc_control::{
16784 AgentCommandResources, ChildMemoryKind, ChildResourceReading,
16785 ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
16786 TreeReading, TreeSource,
16787 };
16788 let registry = Arc::new(Registry::default());
16789 let handle = SupervisorHandle::new();
16790 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16791 .with_handle(handle.clone());
16792 supervisor
16793 .supervise_configured(
16794 crate::ModuleSpec {
16795 script: None,
16796 carry_containment: false,
16797 module_id: "budget-worker".into(),
16798 program: PathBuf::from("disabled-module"),
16799 args: Vec::new(),
16800 env: Vec::new(),
16801 reserved: false,
16802 reserved_prefixes: Vec::new(),
16803 protocol: ModuleProtocol::Subc,
16804 overlap: Default::default(),
16805 },
16806 true,
16807 )
16808 .unwrap();
16809 let handler = ControlHandler::new(registry).with_supervisor(handle);
16810 let now = u64::try_from(
16811 std::time::SystemTime::now()
16812 .duration_since(std::time::UNIX_EPOCH)
16813 .unwrap()
16814 .as_millis(),
16815 )
16816 .unwrap();
16817 let measured = now - 1_000;
16818 let hour = CpuLastHour::new(100, 60_000);
16819 let daemon = DaemonResources::new(7, 1, "test")
16820 .with_measured_at_ms(Some(measured))
16821 .with_resources(Some(ChildResourceUsage::Measured(
16822 ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
16823 )))
16824 .with_cpu_last_hour(Some(hour.clone()));
16825 let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
16826 .with_measured_at_ms(Some(measured))
16827 .with_cpu_last_hour(Some(hour.clone()));
16828 let tree = TreeReading::new(
16829 150,
16830 50,
16831 ChildMemoryKind::ResidentSet,
16832 1,
16833 TreeSource::ProcessTree,
16834 )
16835 .with_measured_at_ms(Some(measured));
16836 handler.resource_tracker.seed_committed_for_test(
16837 &handler.supervisor.list(),
16838 crate::resource_totals::Readings {
16839 modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
16840 daemon: Some(daemon),
16841 agent_commands: Some(vec![agents]),
16842 ..Default::default()
16843 },
16844 now,
16845 );
16846 list_diagnostics::scope(async {
16847 tokio::time::sleep(Duration::from_millis(110)).await;
16848 assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
16849 let request =
16850 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16851 let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
16852 let ClientControlResponse::SupervisorList {
16853 modules,
16854 daemon,
16855 agent_commands,
16856 ..
16857 } = serde_json::from_slice(&replies[0].body).unwrap()
16858 else {
16859 panic!("expected a list response")
16860 };
16861 assert!(matches!(
16862 modules[0].resources,
16863 Some(ChildResourceUsage::Unavailable {
16864 reason: ChildResourceUnavailableReason::Unreadable
16865 })
16866 ));
16867 assert_eq!(
16868 (
16869 modules[0].tree.as_ref().unwrap().cpu_ms,
16870 modules[0].tree.as_ref().unwrap().measured_at_ms
16871 ),
16872 (150, Some(measured))
16873 );
16874 let daemon = daemon.unwrap();
16875 let agents = agent_commands.unwrap();
16876 assert_eq!(daemon.measured_at_ms, Some(measured));
16877 assert_eq!(
16878 (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
16879 (1, 50, Some(measured))
16880 );
16881 for hour in [
16882 modules[0].cpu_last_hour.as_ref(),
16883 daemon.cpu_last_hour.as_ref(),
16884 agents[0].cpu_last_hour.as_ref(),
16885 ] {
16886 let hour = hour.unwrap();
16887 assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
16888 }
16889 })
16890 .await;
16891 }
16892
16893 #[tokio::test]
16894 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
16895 let registry = Arc::new(Registry::default());
16896 let handle = SupervisorHandle::new();
16897 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16898 .with_handle(handle.clone());
16899 supervisor
16900 .supervise_configured(
16901 crate::ModuleSpec {
16902 script: None,
16903 carry_containment: false,
16904 module_id: "drift".into(),
16905 program: PathBuf::from("disabled-module"),
16906 args: Vec::new(),
16907 env: Vec::new(),
16908 reserved: false,
16909 reserved_prefixes: Vec::new(),
16910 protocol: ModuleProtocol::Subc,
16911 overlap: Default::default(),
16912 },
16913 false,
16914 )
16915 .unwrap();
16916 let handler = ControlHandler::new(registry).with_supervisor(handle);
16917 for configured in [None, Some(false), Some(true)] {
16918 handler.capability_evaluator.configure(
16919 configured.map(|enabled| ("drift".to_string(), enabled)),
16920 BTreeMap::new(),
16921 );
16922 let frame =
16923 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16924 let replies = handler
16925 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
16926 .await
16927 .unwrap();
16928 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16929 assert!(body.get("configured_enabled").is_none());
16930 assert_eq!(body["op"], "supervisor.ack");
16931 let frame =
16932 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16933 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16934 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16935 assert_eq!(
16936 body["modules"][0].get("configured_enabled"),
16937 configured.map(Value::Bool).as_ref()
16938 );
16939 assert!(body["modules"][0].get("tree").is_none());
16940 assert!(body["modules"][0].get("subtree").is_none());
16941 }
16942 }
16943
16944 #[tokio::test(flavor = "current_thread")]
16945 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
16946 let handler = ControlHandler::new(Arc::new(Registry::default()));
16947 let capture = EventCapture::default();
16948 let _subscriber =
16949 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16950 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
16951 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16952
16953 let _ = handler
16954 .handle_client_control_request(
16955 &ctx,
16956 frame,
16957 ClientControlRequest::SupervisorSetEnabled {
16958 module_id: "broca".to_string(),
16959 enabled: false,
16960 },
16961 &mut None,
16962 )
16963 .await
16964 .unwrap();
16965
16966 let events = capture
16967 .events()
16968 .into_iter()
16969 .filter(|event| {
16970 event.target == "control"
16971 && event.fields.get("message").map(String::as_str)
16972 == Some("supervisor request received")
16973 })
16974 .collect::<Vec<_>>();
16975 assert_eq!(events.len(), 1, "one supervisor request log line");
16976 let event = &events[0];
16977 assert_eq!(event.level, tracing::Level::INFO);
16978 assert_eq!(
16979 event.fields.get("op"),
16980 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
16981 );
16982 assert_eq!(
16983 event.fields.get("module_id"),
16984 Some(&"\"broca\"".to_string())
16985 );
16986 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
16987 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
16988 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
16989 }
16990
16991 #[tokio::test(flavor = "current_thread")]
16992 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
16993 let registry = Arc::new(Registry::default());
16994 let module_connection = ConnectionId::new(303);
16995 registry
16996 .register_with_control_ops(
16997 manifest("aft", PROTOCOL_VERSION),
16998 PROTOCOL_VERSION,
16999 module_connection,
17000 module_baseline_control_ops(),
17001 )
17002 .unwrap();
17003 let handler = ControlHandler::new(registry);
17004 let capture = EventCapture::default();
17005 let _subscriber =
17006 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17007 let (ctx, _rx) = route_ctx(module_connection);
17008 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
17009
17010 let _ = handler
17011 .handle_client_control_request(
17012 &ctx,
17013 frame,
17014 ClientControlRequest::SupervisorSetEnabled {
17015 module_id: "broca".to_string(),
17016 enabled: false,
17017 },
17018 &mut None,
17019 )
17020 .await
17021 .unwrap();
17022
17023 let callers = capture
17024 .events()
17025 .into_iter()
17026 .filter(|event| {
17027 event.fields.get("message").map(String::as_str)
17028 == Some("supervisor request received")
17029 })
17030 .map(|event| event.fields.get("caller").cloned())
17031 .collect::<Vec<_>>();
17032 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
17033 }
17034
17035 #[tokio::test(flavor = "current_thread")]
17036 async fn supervisor_list_does_not_log_request_received() {
17037 let handler = ControlHandler::new(Arc::new(Registry::default()));
17038 let capture = EventCapture::default();
17039 let _subscriber =
17040 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17041 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
17042 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
17043
17044 handler
17045 .handle_client_control_request(
17046 &ctx,
17047 frame,
17048 ClientControlRequest::SupervisorList {},
17049 &mut None,
17050 )
17051 .await
17052 .unwrap();
17053
17054 assert!(capture.events().into_iter().all(|event| {
17055 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17056 }));
17057 }
17058
17059 #[tokio::test(flavor = "current_thread")]
17060 async fn supervisor_rescan_preview_does_not_log_request_received() {
17061 let handler = ControlHandler::new(Arc::new(Registry::default()));
17062 let capture = EventCapture::default();
17063 let _subscriber =
17064 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17065 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
17066 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
17067
17068 handler
17069 .handle_client_control_request(
17070 &ctx,
17071 frame,
17072 ClientControlRequest::SupervisorRescan { preview: true },
17073 &mut None,
17074 )
17075 .await
17076 .unwrap();
17077
17078 assert!(capture.events().into_iter().all(|event| {
17079 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17080 }));
17081 }
17082
17083 struct ApproveOperator;
17084 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
17085 fn prompt(
17086 &self,
17087 _: &str,
17088 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
17089 ) -> crate::operator_confirm::ProviderResult {
17090 crate::operator_confirm::ProviderResult::Approved
17091 }
17092 }
17093
17094 #[tokio::test]
17095 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
17096 assert_eq!(
17097 ControlHandler::default().privileged_operator_prompt,
17098 PrivilegedOperatorPrompt::Log
17099 );
17100 for (initial, changed, running) in [
17101 ("log", "enforce", PrivilegedOperatorPrompt::Log),
17102 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
17103 ] {
17104 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
17105 let path = dir.join("subc.jsonc");
17106 std::fs::write(
17107 &path,
17108 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
17109 )
17110 .unwrap();
17111 let config = crate::daemon_config::load(&path).unwrap().unwrap();
17112 let registry = Arc::new(Registry::default());
17113 let handle = SupervisorHandle::new();
17114 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
17115 .with_handle(handle.clone());
17116 let handler = ControlHandler::new(registry)
17117 .with_supervisor(handle)
17118 .with_privileged_operator_prompt(config.privileged_operator_prompt)
17119 .with_supervisor_rescan(supervisor, &path, config.port);
17120 handler.forwarding.operator_confirms().configure(
17121 Arc::new(ApproveOperator),
17122 crate::operator_confirm::OperatorLimits::default(),
17123 );
17124 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
17125
17126 for (disk_mode, needs_restart) in [
17127 (Some(initial), false),
17128 (Some(changed), true),
17129 (Some(changed), true),
17130 (None, running == PrivilegedOperatorPrompt::Enforce),
17131 (Some(initial), false),
17132 ] {
17133 let mut doc = json!({"version":1});
17134 if let Some(mode) = disk_mode {
17135 doc["privileged_operator_prompt"] = json!(mode);
17136 }
17137 std::fs::write(&path, doc.to_string()).unwrap();
17138 for preview in [true, false] {
17139 let request = ClientControlRequest::SupervisorRescan { preview };
17140 let frame = Frame::build(
17141 FrameType::Request,
17142 control_flags(),
17143 0,
17144 0,
17145 1,
17146 serde_json::to_vec(&request).unwrap(),
17147 )
17148 .unwrap();
17149 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17150 assert_eq!(frames.len(), 1);
17151 let ClientControlResponse::SupervisorRescan { result } =
17152 serde_json::from_slice(&frames[0].body).unwrap()
17153 else {
17154 panic!("expected a rescan response");
17155 };
17156 let expected = if needs_restart {
17157 vec!["privileged_operator_prompt"]
17158 } else {
17159 Vec::new()
17160 };
17161 assert_eq!(result.restart_required, expected);
17162 assert_eq!(handler.privileged_operator_prompt, running);
17163 assert_eq!(
17164 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
17165 running
17166 );
17167 }
17168 }
17169 }
17170 }
17171
17172 #[tokio::test]
17173 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
17174 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
17175 let path = dir.join("subc.jsonc");
17176 let registry = Arc::new(Registry::default());
17177 let handle = SupervisorHandle::new();
17178 let supervisor =
17179 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
17180 let handler = ControlHandler::new(registry)
17181 .with_supervisor(handle)
17182 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
17183 .with_supervisor_rescan(supervisor, &path, None);
17184 handler.forwarding.operator_confirms().configure(
17185 Arc::new(ApproveOperator),
17186 crate::operator_confirm::OperatorLimits::default(),
17187 );
17188 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
17189 for preview in [true, false] {
17190 std::fs::write(
17191 &path,
17192 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
17193 )
17194 .unwrap();
17195 let request = ClientControlRequest::SupervisorRescan { preview };
17196 let frame = Frame::build(
17197 FrameType::Request,
17198 control_flags(),
17199 0,
17200 0,
17201 1,
17202 serde_json::to_vec(&request).unwrap(),
17203 )
17204 .unwrap();
17205 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17206 assert_eq!(frames.len(), 1);
17207 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
17208 assert_eq!(
17209 handler.privileged_operator_prompt,
17210 PrivilegedOperatorPrompt::Enforce
17211 );
17212 }
17213 }
17214
17215 #[tokio::test]
17220 async fn route_open_refusal_names_the_check_that_refused() {
17221 let handler = ControlHandler::new(Arc::new(Registry::default()));
17222 let capture = EventCapture::default();
17223 let _subscriber =
17224 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17225 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17226 let response = handler
17227 .handle_control_frame(
17228 &ctx,
17229 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
17230 )
17231 .await
17232 .unwrap();
17233
17234 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17235 let event = capture
17236 .events()
17237 .into_iter()
17238 .find(|event| {
17239 event.target == "control"
17240 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17241 })
17242 .expect("route.open refusal event");
17243 assert_eq!(
17244 event.fields.get("reason"),
17245 Some(&"\"not_registered\"".to_string())
17246 );
17247 }
17248
17249 #[tokio::test]
17250 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
17251 let registry = Arc::new(Registry::default());
17252 let supervisor_handle = SupervisorHandle::new();
17253 let supervisor =
17254 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17255 .with_handle(supervisor_handle.clone())
17256 .with_connection_file_path(std::env::temp_dir().join(format!(
17257 "subc-route-open-refusal-info-{}",
17258 std::process::id()
17259 )));
17260 let module = supervisor
17261 .supervise_configured(
17262 ModuleSpec {
17263 script: None,
17264 carry_containment: false,
17265 module_id: "warming".to_string(),
17266 program: fake_aft_stub_path(),
17267 args: Vec::new(),
17268 env: Vec::new(),
17269 reserved: false,
17270 reserved_prefixes: Vec::new(),
17271 protocol: ModuleProtocol::Subc,
17272 overlap: Default::default(),
17273 },
17274 true,
17275 )
17276 .unwrap();
17277 assert_eq!(module.state().unwrap(), ModuleState::Running);
17278
17279 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17280 assert!(handler
17281 .counters()
17282 .snapshot()
17283 .get("route_open_refused_by_code")
17284 .is_none());
17285 let capture = EventCapture::default();
17286 let _subscriber =
17287 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17288 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
17289 let response = handler
17290 .handle_control_frame(
17291 &ctx,
17292 route_open_frame(394, "warming", unique_project_root("refusal-info")),
17293 )
17294 .await
17295 .unwrap();
17296 module.stop().await.unwrap();
17297
17298 assert_eq!(parse_error(&response[0])["code"], "module_warming");
17299 let event = capture
17300 .events()
17301 .into_iter()
17302 .find(|event| {
17303 event.target == "control"
17304 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
17305 })
17306 .expect("route.open refusal event");
17307 assert_eq!(
17308 event.fields.get("module_id"),
17309 Some(&"\"warming\"".to_string())
17310 );
17311 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
17312 assert_eq!(
17313 event.fields.get("reason"),
17314 Some(&"\"supervised_not_registered\"".to_string())
17315 );
17316 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
17317 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
17318 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
17319 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
17320 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
17321 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
17322 assert_eq!(
17323 handler.counters().snapshot()["route_open_refused_by_code"],
17324 json!({ "module_warming": 1 })
17325 );
17326 }
17327
17328 const OUTAGE_START: &str = "route.open refusing module: not serving";
17329 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
17330
17331 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
17332 capture
17333 .events()
17334 .into_iter()
17335 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
17336 .collect()
17337 }
17338
17339 fn supervise_stub(
17340 registry: &Arc<Registry>,
17341 module_id: &str,
17342 enabled: bool,
17343 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
17344 let supervisor_handle = SupervisorHandle::new();
17345 let supervisor =
17346 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
17347 .with_handle(supervisor_handle.clone())
17348 .with_connection_file_path(std::env::temp_dir().join(format!(
17349 "subc-route-outage-{module_id}-{}",
17350 std::process::id()
17351 )));
17352 let module = supervisor
17353 .supervise_configured(
17354 ModuleSpec {
17355 script: None,
17356 carry_containment: false,
17357 module_id: module_id.to_string(),
17358 program: fake_aft_stub_path(),
17359 args: Vec::new(),
17360 env: Vec::new(),
17361 reserved: false,
17362 reserved_prefixes: Vec::new(),
17363 protocol: ModuleProtocol::Subc,
17364 overlap: Default::default(),
17365 },
17366 enabled,
17367 )
17368 .unwrap();
17369 (supervisor_handle, module)
17370 }
17371
17372 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
17373 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
17374 module_id: module_id.to_string(),
17375 drain_timeout_ms: Some(50),
17376 })
17377 .unwrap();
17378 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
17379 }
17380
17381 #[test]
17385 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
17386 let registry = Arc::new(Registry::default());
17387 let forwarding = Arc::new(ForwardingTable::default());
17388 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17389 let second = ControlHandler::with_forwarding(registry, forwarding);
17390 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
17391 }
17392
17393 #[tokio::test(flavor = "current_thread")]
17397 async fn route_open_unknown_module_refusals_add_no_outage_state() {
17398 let handler = ControlHandler::new(Arc::new(Registry::default()));
17399 let capture = EventCapture::default();
17400 let _subscriber =
17401 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17402 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
17403 for corr in 0..8 {
17404 let response = handler
17405 .handle_control_frame(
17406 &ctx,
17407 route_open_frame(
17408 380 + corr,
17409 &format!("nobody-{corr}"),
17410 unique_project_root("outage-unknown"),
17411 ),
17412 )
17413 .await
17414 .unwrap();
17415 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17416 }
17417
17418 assert_eq!(handler.route_outages.tracked_module_count(), 0);
17419 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
17420 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
17421 }
17422
17423 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17428 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
17429 let registry = Arc::new(Registry::default());
17430 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
17431 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17432 let capture = EventCapture::default();
17433 let _subscriber =
17434 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17435 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
17436 handler.route_outages.record_accepted("outage-restart");
17439
17440 let response = handler
17441 .handle_control_frame(
17442 &ctx,
17443 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
17444 )
17445 .await
17446 .unwrap();
17447 assert_eq!(response[0].header.ty, FrameType::Error);
17448 let starts = outage_lines(&capture, OUTAGE_START);
17449 assert_eq!(starts.len(), 1, "{starts:?}");
17450 assert_eq!(starts[0].level, tracing::Level::WARN);
17451 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
17452 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
17453 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
17454 handler.route_outages.record_accepted("outage-restart");
17455 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
17456
17457 tokio::time::timeout(Duration::from_secs(10), async {
17461 while module.state().unwrap() != crate::ModuleState::Failed {
17462 tokio::task::yield_now().await;
17463 }
17464 })
17465 .await
17466 .expect("stub exit must be recorded before the operator restart");
17467
17468 let restart = handler
17469 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
17470 .await
17471 .unwrap();
17472 assert_eq!(
17473 restart[0].header.ty,
17474 FrameType::Response,
17475 "{:?}",
17476 parse_error(&restart[0])
17477 );
17478 handler
17479 .handle_control_frame(
17480 &ctx,
17481 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
17482 )
17483 .await
17484 .unwrap();
17485 module.stop().await.unwrap();
17486
17487 let starts = outage_lines(&capture, OUTAGE_START);
17488 assert_eq!(starts.len(), 2, "{starts:?}");
17489 assert_eq!(starts[1].level, tracing::Level::INFO);
17490 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
17491 }
17492
17493 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17497 async fn failed_operator_restart_leaves_no_operator_mark() {
17498 let registry = Arc::new(Registry::default());
17499 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
17500 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17501 let capture = EventCapture::default();
17502 let _subscriber =
17503 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17504 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
17505 handler.route_outages.record_accepted("outage-disabled");
17506
17507 let restart = handler
17508 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
17509 .await
17510 .unwrap();
17511 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
17512 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
17513
17514 handler
17515 .handle_control_frame(
17516 &ctx,
17517 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
17518 )
17519 .await
17520 .unwrap();
17521 let starts = outage_lines(&capture, OUTAGE_START);
17522 assert_eq!(starts.len(), 1, "{starts:?}");
17523 assert_eq!(starts[0].level, tracing::Level::WARN);
17524 }
17525
17526 #[tokio::test(flavor = "current_thread")]
17527 async fn route_open_unknown_module_escapes_target_module_id() {
17528 let handler = ControlHandler::new(Arc::new(Registry::default()));
17529 let capture = EventCapture::default();
17530 let _subscriber =
17531 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17532 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
17533 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17534 let response = handler
17535 .handle_control_frame(
17536 &ctx,
17537 route_open_frame(
17538 395,
17539 hostile_module_id,
17540 unique_project_root("hostile-target-module-id"),
17541 ),
17542 )
17543 .await
17544 .unwrap();
17545
17546 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17547 let event = capture
17548 .events()
17549 .into_iter()
17550 .find(|event| {
17551 event.target == "control"
17552 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17553 })
17554 .expect("route.open unknown-module refusal event");
17555 let logged = event.fields.get("module_id").expect("module_id field");
17556 assert!(!logged.bytes().any(|byte| byte < 0x20));
17557 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17558 }
17559
17560 #[tokio::test(flavor = "current_thread")]
17563 async fn route_open_refusal_carries_daemon_origin() {
17564 let handler = ControlHandler::new(Arc::new(Registry::default()));
17565 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17566 let response = handler
17567 .handle_control_frame(
17568 &ctx,
17569 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
17570 )
17571 .await
17572 .unwrap();
17573
17574 assert_eq!(response.len(), 1);
17575 assert_eq!(response[0].header.ty, FrameType::Error);
17576 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17577 assert!(
17578 response[0].header.flags.is_daemon_origin(),
17579 "route.open refusal must carry daemon origin: {:?}",
17580 response[0].header.flags
17581 );
17582 }
17583
17584 #[tokio::test(flavor = "current_thread")]
17588 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
17589 let registry = Arc::new(Registry::default());
17590 let forwarding = Arc::new(ForwardingTable::default());
17591 let handler =
17592 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17593 let module_connection = ConnectionId::new(95);
17594 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17595 hello_via_sink(
17596 &handler,
17597 &module_ctx,
17598 &mut module_rx,
17599 hello_frame("aft", PROTOCOL_VERSION, 395),
17600 )
17601 .await;
17602 forwarding
17603 .begin_module_drain("aft", RouteCloseReason::Restart)
17604 .unwrap()
17605 .expect("the module endpoint drains");
17606
17607 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
17608 let response = handler
17609 .handle_control_frame(
17610 &client_ctx,
17611 route_open_frame(396, "aft", unique_project_root("drain-origin")),
17612 )
17613 .await
17614 .unwrap();
17615
17616 assert_eq!(response.len(), 1);
17617 assert_eq!(response[0].header.ty, FrameType::Error);
17618 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17619 assert!(
17620 response[0].header.flags.is_daemon_origin(),
17621 "drain module_reloading must carry daemon origin: {:?}",
17622 response[0].header.flags
17623 );
17624 }
17625
17626 #[tokio::test(flavor = "current_thread")]
17630 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
17631 let registry = Arc::new(Registry::default());
17632 let forwarding = Arc::new(ForwardingTable::default());
17633 let handler =
17634 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17635 let module_connection = ConnectionId::new(95);
17636 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17637 hello_via_sink(
17638 &handler,
17639 &module_ctx,
17640 &mut module_rx,
17641 hello_frame("aft", PROTOCOL_VERSION, 395),
17642 )
17643 .await;
17644
17645 let client_connection = ConnectionId::new(96);
17646 let (client_ctx, _client_rx) = route_ctx(client_connection);
17647 let (route_task, bind) = relay_route_open(
17648 &handler,
17649 client_connection,
17650 &client_ctx.egress,
17651 &mut module_rx,
17652 396,
17653 "aft",
17654 "spoofed-reloading",
17655 )
17656 .await;
17657 let rejection = Frame::build(
17660 FrameType::Error,
17661 crate::router::daemon_error_flags(),
17662 0,
17663 0,
17664 bind.header.corr,
17665 serde_json::to_vec(&ErrorBody::new(
17666 "module_reloading",
17667 "module refused route.bind",
17668 ))
17669 .unwrap(),
17670 )
17671 .unwrap();
17672 handler
17673 .handle_control_frame(&module_ctx, rejection)
17674 .await
17675 .unwrap();
17676
17677 let response = route_task.await.unwrap();
17678 assert_eq!(response.len(), 1);
17679 assert_eq!(response[0].header.ty, FrameType::Error);
17680 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17681 assert!(
17682 !response[0].header.flags.is_daemon_origin(),
17683 "a module's own rejection must not carry daemon origin: {:?}",
17684 response[0].header.flags
17685 );
17686 }
17687
17688 #[tokio::test(flavor = "current_thread")]
17689 async fn route_open_module_rejection_uses_daemon_counter_key() {
17690 let registry = Arc::new(Registry::default());
17691 let forwarding = Arc::new(ForwardingTable::default());
17692 let handler =
17693 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17694 let module_connection = ConnectionId::new(95);
17695 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17696 hello_via_sink(
17697 &handler,
17698 &module_ctx,
17699 &mut module_rx,
17700 hello_frame("aft", PROTOCOL_VERSION, 395),
17701 )
17702 .await;
17703
17704 let client_connection = ConnectionId::new(96);
17705 let (client_ctx, _client_rx) = route_ctx(client_connection);
17706 let capture = EventCapture::default();
17707 static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
17714 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
17715 let _subscriber =
17716 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17717 let (route_task, bind) = relay_route_open(
17718 &handler,
17719 client_connection,
17720 &client_ctx.egress,
17721 &mut module_rx,
17722 396,
17723 "aft",
17724 "hostile-module-code",
17725 )
17726 .await;
17727 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
17728 let rejection = Frame::build(
17729 FrameType::Error,
17730 control_flags(),
17731 0,
17732 0,
17733 bind.header.corr,
17734 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
17735 )
17736 .unwrap();
17737 handler
17738 .handle_control_frame(&module_ctx, rejection)
17739 .await
17740 .unwrap();
17741
17742 let response = route_task.await.unwrap();
17743 assert_eq!(parse_error(&response[0])["code"], hostile_code);
17744 let counters = handler.counters().snapshot();
17745 assert_eq!(
17746 counters["route_open_refused_by_code"],
17747 json!({ "module_rejected": 1 })
17748 );
17749 assert!(counters["route_open_refused_by_code"]
17750 .get(hostile_code)
17751 .is_none());
17752
17753 let event = capture
17754 .events()
17755 .into_iter()
17756 .find(|event| {
17757 event.target == "control"
17758 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
17759 })
17760 .expect("route.open module-rejection refusal event");
17761 let logged = event.fields.get("module_code").expect("module_code field");
17762 assert!(!logged.bytes().any(|byte| byte < 0x20));
17763 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17764 }
17765
17766 #[tokio::test]
17767 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
17768 let registry = Arc::new(Registry::default());
17769 let supervisor_handle = SupervisorHandle::new();
17770 let missing_program = std::env::temp_dir().join(format!(
17771 "subc-route-open-missing-program-{}",
17772 std::process::id()
17773 ));
17774 let supervisor =
17775 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17776 .with_handle(supervisor_handle.clone());
17777 let module = supervisor
17778 .supervise_configured(
17779 ModuleSpec {
17780 script: None,
17781 carry_containment: false,
17782 module_id: "failed".to_string(),
17783 program: missing_program,
17784 args: Vec::new(),
17785 env: Vec::new(),
17786 reserved: false,
17787 reserved_prefixes: Vec::new(),
17788 protocol: ModuleProtocol::Subc,
17789 overlap: Default::default(),
17790 },
17791 true,
17792 )
17793 .unwrap();
17794 assert_eq!(module.state().unwrap(), ModuleState::Failed);
17795
17796 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17797 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
17798 let response = handler
17799 .handle_control_frame(
17800 &ctx,
17801 route_open_frame(305, "failed", unique_project_root("failed")),
17802 )
17803 .await
17804 .unwrap();
17805
17806 assert_eq!(response[0].header.ty, FrameType::Error);
17807 let error = parse_error(&response[0]);
17808 assert_eq!(error["code"], "target_unavailable");
17809 assert!(error["message"]
17810 .as_str()
17811 .unwrap()
17812 .contains("state=failed, enabled=true, live=false"));
17813 }
17814
17815 #[tokio::test]
17816 async fn route_open_role_mismatch_remains_target_unavailable() {
17817 let registry = Arc::new(Registry::default());
17818 let handler = ControlHandler::new(Arc::clone(®istry));
17819 handler
17820 .handle_control(
17821 ConnectionId::new(41),
17822 non_routable_hello_frame_with_control_ops("health-only", 306, None),
17823 )
17824 .unwrap();
17825
17826 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
17827 let response = handler
17828 .handle_control_frame(
17829 &ctx,
17830 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
17831 )
17832 .await
17833 .unwrap();
17834
17835 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17836 assert!(parse_error(&response[0])["message"]
17837 .as_str()
17838 .unwrap()
17839 .contains("does not provide the requested target"));
17840 }
17841
17842 #[tokio::test]
17843 async fn route_open_inactive_registration_remains_target_unavailable() {
17844 let registry = Arc::new(Registry::default());
17845 let handler = ControlHandler::new(Arc::clone(®istry));
17846 handler
17847 .handle_control(
17848 ConnectionId::new(43),
17849 hello_frame("inactive", PROTOCOL_VERSION, 308),
17850 )
17851 .unwrap();
17852 assert!(registry
17853 .set_module_state_for_test("inactive", ChannelState::Closed)
17854 .unwrap());
17855
17856 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
17857 let response = handler
17858 .handle_control_frame(
17859 &ctx,
17860 route_open_frame(309, "inactive", unique_project_root("inactive")),
17861 )
17862 .await
17863 .unwrap();
17864
17865 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17866 assert!(parse_error(&response[0])["message"]
17867 .as_str()
17868 .unwrap()
17869 .contains("is not active"));
17870 }
17871
17872 #[tokio::test]
17873 async fn late_health_reply_is_recorded_through_the_module_response_path() {
17874 let registry = Arc::new(Registry::default());
17875 let forwarding = Arc::new(ForwardingTable::default());
17876 let supervisor_handle = SupervisorHandle::new();
17877 let supervisor =
17878 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
17879 .with_forwarding(Arc::clone(&forwarding))
17880 .with_handle(supervisor_handle.clone());
17881 let module = supervisor
17882 .supervise_configured(
17883 crate::ModuleSpec {
17884 script: None,
17885 carry_containment: false,
17886 module_id: "late-health-response".to_string(),
17887 program: PathBuf::from("disabled-module"),
17888 args: Vec::new(),
17889 env: Vec::new(),
17890 reserved: false,
17891 reserved_prefixes: Vec::new(),
17892 protocol: ModuleProtocol::Subc,
17893 overlap: Default::default(),
17894 },
17895 false,
17896 )
17897 .unwrap();
17898 let handler =
17899 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17900 .with_supervisor(supervisor_handle);
17901 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
17902 handler
17903 .handle_control_frame(
17904 &module_ctx,
17905 hello_frame_with_control_ops(
17906 "late-health-response",
17907 PROTOCOL_VERSION,
17908 7,
17909 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17910 ),
17911 )
17912 .await
17913 .unwrap();
17914 let probe_started_at = Instant::now() - Duration::from_millis(80);
17915 let pending = forwarding
17916 .begin_health_probe_rpc_for(
17917 "late-health-response",
17918 MODULE_CONTROL_OP_HEALTH_CHECK,
17919 probe_started_at,
17920 Instant::now() - Duration::from_millis(1),
17921 )
17922 .unwrap();
17923 assert!(forwarding
17924 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
17925 .unwrap());
17926
17927 let responses = handler
17928 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
17929 .await
17930 .unwrap();
17931
17932 assert!(responses.is_empty());
17933 let health = module.status().unwrap().health;
17934 assert_eq!(health.late_answer_count, 1);
17935 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
17936 }
17937
17938 #[tokio::test]
17939 async fn health_probe_timeout_and_module_death_are_typed() {
17940 let registry = Arc::new(Registry::default());
17941 let forwarding = Arc::new(ForwardingTable::default());
17942 let handler =
17943 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17944 .with_health_probe_timeout(Duration::from_millis(50));
17945 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
17946 hello_via_sink(
17947 &handler,
17948 &module_ctx,
17949 &mut module_rx,
17950 hello_frame_with_control_ops(
17951 "aft",
17952 PROTOCOL_VERSION,
17953 7,
17954 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17955 ),
17956 )
17957 .await;
17958
17959 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
17960 let responses = handler
17961 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
17962 .await
17963 .unwrap();
17964 assert_eq!(responses[0].header.ty, FrameType::Error);
17965 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
17966 let _ = module_rx.try_recv();
17967
17968 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
17969 let health_handler = handler.clone();
17970 let death_task = tokio::spawn(async move {
17971 health_handler
17972 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
17973 .await
17974 .unwrap()
17975 });
17976 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
17977 .await
17978 .unwrap()
17979 .unwrap();
17980 handler
17981 .cleanup_connection(module_ctx.connection_id)
17982 .unwrap();
17983 let responses = death_task.await.unwrap();
17984 assert_eq!(responses[0].header.ty, FrameType::Error);
17985 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
17986 }
17987
17988 #[test]
17989 fn hello_requires_exact_protocol_version() {
17990 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
17991 let registry = Arc::new(Registry::default());
17992 let handler = ControlHandler::new(Arc::clone(®istry));
17993 let responses = handler
17994 .handle_control(
17995 ConnectionId::new(connection),
17996 hello_frame("aft", offered, 9),
17997 )
17998 .unwrap();
17999
18000 assert_eq!(responses.len(), 1);
18001 assert_eq!(responses[0].header.ty, FrameType::Error);
18002 let error = parse_error(&responses[0]);
18003 assert_eq!(error["code"], "version_unsupported");
18004 assert!(registry.get_module("aft").unwrap().is_none());
18005 assert_eq!(registry.active_registration_count().unwrap(), 0);
18006 }
18007 }
18008
18009 #[test]
18010 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
18011 let registry = Arc::new(Registry::default());
18012 let forwarding = Arc::new(ForwardingTable::default());
18013 let handler =
18014 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18015 let module_connection = ConnectionId::new(301);
18016 let registration = registry
18017 .register_with_control_ops(
18018 manifest("aft-push", PROTOCOL_VERSION),
18019 PROTOCOL_VERSION,
18020 module_connection,
18021 module_baseline_control_ops(),
18022 )
18023 .unwrap();
18024 let (module_tx, _module_rx) = mpsc::channel(8);
18025 let endpoint = forwarding
18026 .register_module_connection(
18027 module_connection,
18028 "aft-push".to_string(),
18029 PROTOCOL_VERSION,
18030 manifest_concurrency(®istration.manifest),
18031 FrameSink::new(module_tx),
18032 )
18033 .unwrap();
18034
18035 let unknown = Frame::build(
18037 FrameType::Push,
18038 control_flags(),
18039 0,
18040 0,
18041 5,
18042 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
18043 )
18044 .unwrap();
18045 let out = handler.handle_status_update(endpoint, unknown).unwrap();
18046 assert!(
18047 out.is_empty(),
18048 "unknown push op must be ignored, got {out:?}"
18049 );
18050
18051 let malformed = Frame::build(
18053 FrameType::Push,
18054 control_flags(),
18055 0,
18056 0,
18057 6,
18058 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
18059 )
18060 .unwrap();
18061 let out = handler.handle_status_update(endpoint, malformed).unwrap();
18062 assert_eq!(out.len(), 1);
18063 assert_eq!(out[0].header.ty, FrameType::Error);
18064 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
18065 }
18066
18067 #[test]
18068 fn hello_rejected_when_connection_already_owns_client_routes() {
18069 let registry = Arc::new(Registry::default());
18070 let forwarding = Arc::new(ForwardingTable::default());
18071 let handler =
18072 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18073 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
18075 let client_connection = ConnectionId::new(202);
18076
18077 let responses = handler
18080 .handle_control(
18081 client_connection,
18082 hello_frame("aft-second", PROTOCOL_VERSION, 9),
18083 )
18084 .unwrap();
18085 assert_eq!(responses[0].header.ty, FrameType::Error);
18086 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
18087 assert!(registry.get_module("aft-second").unwrap().is_none());
18088 }
18089
18090 #[tokio::test]
18091 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
18092 let registry = Arc::new(Registry::default());
18093 let forwarding = Arc::new(ForwardingTable::default());
18094 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
18095 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
18096 hello_via_sink(
18097 &handler,
18098 &module_ctx,
18099 &mut module_rx,
18100 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
18101 )
18102 .await;
18103 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
18104 let pending = forwarding
18105 .begin_route_bind_relay_for_test(
18106 client_ctx.connection_id,
18107 client_ctx.egress.clone(),
18108 2,
18109 "alpha",
18110 )
18111 .unwrap();
18112 forwarding
18113 .complete_pending_relay(
18114 module_ctx.connection_id,
18115 pending.corr,
18116 RouteBindRelayOutcome::Accepted,
18117 )
18118 .unwrap();
18119 client_rx.try_recv().unwrap();
18120 for module_id in ["beta", "alpha"] {
18121 let replies = handler
18122 .handle_control_frame(
18123 &module_ctx,
18124 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
18125 )
18126 .await
18127 .unwrap();
18128 assert_eq!(replies.len(), 1, "second HELLO must be refused");
18129 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
18130 }
18131 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
18132 assert!(registry.get_module("beta").unwrap().is_none());
18133 assert!(matches!(
18134 forwarding
18135 .lookup_data_route(
18136 client_ctx.connection_id,
18137 pending.client_channel,
18138 pending.client_epoch,
18139 )
18140 .unwrap(),
18141 DataRoute::Client(DataRouteState::Bound(_))
18142 ));
18143 assert!(handler
18144 .hello_launch_nonces
18145 .lock()
18146 .unwrap()
18147 .presented(module_ctx.connection_id, Some("alpha-nonce")));
18148 assert!(module_rx.try_recv().is_err());
18149 }
18150
18151 #[test]
18152 fn reserved_module_hello_requires_matching_launch_nonce() {
18153 let registry = Arc::new(Registry::default());
18154 let supervisor = SupervisorHandle::new();
18155 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
18158 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18159
18160 let no_nonce = handler
18162 .handle_control(
18163 ConnectionId::new(1),
18164 hello_frame("vault", PROTOCOL_VERSION, 1),
18165 )
18166 .unwrap();
18167 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
18168 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
18169 assert!(registry.get_module("vault").unwrap().is_none());
18170
18171 let wrong = handler
18173 .handle_control(
18174 ConnectionId::new(2),
18175 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
18176 )
18177 .unwrap();
18178 assert_eq!(wrong[0].header.ty, FrameType::Error);
18179 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
18180 assert!(registry.get_module("vault").unwrap().is_none());
18181
18182 let ok = handler
18184 .handle_control(
18185 ConnectionId::new(3),
18186 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
18187 )
18188 .unwrap();
18189 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
18190 assert!(registry.get_module("vault").unwrap().is_some());
18191 }
18192
18193 #[test]
18194 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
18195 let registry = Arc::new(Registry::default());
18196 let supervisor = SupervisorHandle::new();
18197 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18198 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18199 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18200
18201 let squat = handler
18202 .handle_control(
18203 ConnectionId::new(1),
18204 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
18205 )
18206 .unwrap();
18207 assert_eq!(squat[0].header.ty, FrameType::Error);
18208 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
18209 assert!(parse_error(&squat[0])["message"]
18210 .as_str()
18211 .unwrap()
18212 .contains("fed:"));
18213
18214 let accepted_peer = handler
18215 .handle_control(
18216 ConnectionId::new(2),
18217 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
18218 )
18219 .unwrap();
18220 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
18221
18222 let accepted_short = handler
18223 .handle_control(
18224 ConnectionId::new(3),
18225 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
18226 )
18227 .unwrap();
18228 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
18229
18230 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
18231 let response = handler
18232 .handle_control(
18233 ConnectionId::new(conn),
18234 hello_frame(module_id, PROTOCOL_VERSION, conn),
18235 )
18236 .unwrap();
18237 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
18238 }
18239 }
18240
18241 #[test]
18242 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
18243 let registry = Arc::new(Registry::default());
18244 let supervisor = SupervisorHandle::new();
18245 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18246 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18247 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
18248 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18249
18250 let owner_nonce = handler
18251 .handle_control(
18252 ConnectionId::new(1),
18253 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
18254 )
18255 .unwrap();
18256 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
18257 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
18258 assert!(registry.get_module("fed:special").unwrap().is_none());
18259
18260 let exact_nonce = handler
18261 .handle_control(
18262 ConnectionId::new(2),
18263 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
18264 )
18265 .unwrap();
18266 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
18267 assert!(registry.get_module("fed:special").unwrap().is_some());
18268 }
18269
18270 #[test]
18271 fn non_reserved_module_ignores_launch_nonce() {
18272 let registry = Arc::new(Registry::default());
18273 let handler = ControlHandler::new(Arc::clone(®istry));
18276 let no_nonce = handler
18277 .handle_control(
18278 ConnectionId::new(1),
18279 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
18280 )
18281 .unwrap();
18282 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
18283 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
18284
18285 let echoed_nonce = handler
18286 .handle_control(
18287 ConnectionId::new(2),
18288 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
18289 )
18290 .unwrap();
18291 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
18292 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
18293 }
18294
18295 #[test]
18296 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
18297 let handler = ControlHandler::default();
18298 let conn = ConnectionId::new(1);
18299 let malformed = Frame::build(
18300 FrameType::Hello,
18301 control_flags(),
18302 0,
18303 0,
18304 3,
18305 b"{not json".to_vec(),
18306 )
18307 .unwrap();
18308
18309 let error = handler.handle_control(conn, malformed).unwrap();
18310 assert_eq!(error[0].header.ty, FrameType::Error);
18311 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
18312
18313 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18314 let pong = handler.handle_control(conn, ping).unwrap();
18315 assert_eq!(pong[0].header.ty, FrameType::Pong);
18316 assert_eq!(pong[0].header.corr, 4);
18317 }
18318
18319 #[test]
18320 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
18321 let registry = Arc::new(Registry::default());
18322 let handler = ControlHandler::new(Arc::clone(®istry));
18323
18324 handler
18325 .handle_control(
18326 ConnectionId::new(1),
18327 hello_frame("aft", PROTOCOL_VERSION, 1),
18328 )
18329 .unwrap();
18330 let duplicate = handler
18331 .handle_control(
18332 ConnectionId::new(2),
18333 hello_frame("aft", PROTOCOL_VERSION, 2),
18334 )
18335 .unwrap();
18336
18337 assert_eq!(duplicate[0].header.ty, FrameType::Error);
18338 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18339 let registration = registry.get_module("aft").unwrap().unwrap();
18340 assert_eq!(registration.connection_id, ConnectionId::new(1));
18341 }
18342
18343 #[test]
18344 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
18345 let registry = Arc::new(Registry::default());
18346 let forwarding = Arc::new(ForwardingTable::default());
18347 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
18348 let handler =
18349 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18350 .with_process_liveness(process_liveness);
18351 let (ctx, route_channel, route_epoch) =
18352 bind_liveness_route(®istry, &forwarding, "aft-dead");
18353 let responses = handler
18354 .handle_route_poll(
18355 &ctx,
18356 route_poll_frame(41, PollKind::Liveness, route_channel),
18357 route_channel,
18358 route_epoch,
18359 PollKind::Liveness,
18360 )
18361 .unwrap();
18362
18363 assert_eq!(responses.len(), 1);
18364 assert_eq!(responses[0].header.ty, FrameType::Response);
18365 assert_route_poll_liveness(&responses[0], false);
18366 }
18367
18368 #[test]
18369 fn liveness_poll_without_process_source_uses_bound_route() {
18370 let registry = Arc::new(Registry::default());
18371 let forwarding = Arc::new(ForwardingTable::default());
18372 let handler =
18373 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18374 let (ctx, route_channel, route_epoch) =
18375 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
18376 let responses = handler
18377 .handle_route_poll(
18378 &ctx,
18379 route_poll_frame(42, PollKind::Liveness, route_channel),
18380 route_channel,
18381 route_epoch,
18382 PollKind::Liveness,
18383 )
18384 .unwrap();
18385
18386 assert_route_poll_liveness(&responses[0], true);
18387 }
18388
18389 #[test]
18390 fn liveness_poll_untracked_process_source_uses_bound_route() {
18391 let registry = Arc::new(Registry::default());
18392 let forwarding = Arc::new(ForwardingTable::default());
18393 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
18394 let handler =
18395 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18396 .with_process_liveness(process_liveness);
18397 let (ctx, route_channel, route_epoch) =
18398 bind_liveness_route(®istry, &forwarding, "aft-untracked");
18399 let responses = handler
18400 .handle_route_poll(
18401 &ctx,
18402 route_poll_frame(43, PollKind::Liveness, route_channel),
18403 route_channel,
18404 route_epoch,
18405 PollKind::Liveness,
18406 )
18407 .unwrap();
18408
18409 assert_route_poll_liveness(&responses[0], true);
18410 }
18411
18412 #[tokio::test]
18413 async fn unknown_op_returns_unknown_control_op() {
18414 let handler = ControlHandler::default();
18415 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
18416 let request = Frame::build(
18417 FrameType::Request,
18418 control_flags(),
18419 0,
18420 0,
18421 55,
18422 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
18423 )
18424 .unwrap();
18425
18426 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18427
18428 assert_eq!(response.len(), 1);
18429 assert_eq!(response[0].header.ty, FrameType::Error);
18430 assert_eq!(response[0].header.corr, 55);
18431 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
18432 }
18433
18434 #[tokio::test]
18435 async fn supervisor_provenance_rejects_unknown_exact_module() {
18436 let handler = ControlHandler::default();
18437 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
18438 let request = Frame::build(
18439 FrameType::Request,
18440 control_flags(),
18441 0,
18442 0,
18443 57,
18444 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
18445 )
18446 .unwrap();
18447
18448 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18449
18450 assert_eq!(response.len(), 1);
18451 assert_eq!(response[0].header.ty, FrameType::Error);
18452 assert_eq!(response[0].header.corr, 57);
18453 let error = parse_error(&response[0]);
18454 assert_eq!(error["code"], "unknown_module");
18455 assert_eq!(error["message"], "module_id 'missing' is not supervised");
18456 }
18457
18458 #[test]
18459 fn provenance_probe_override_keeps_handler_tests_deterministic() {
18460 let expected = subc_control::RunningImageAgreement::Unavailable {
18461 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18462 };
18463 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
18464 assert_eq!(handler.provenance_probe_override, Some(expected));
18465 }
18466
18467 #[test]
18468 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
18469 let verdict = reload_verdict(
18470 std::path::Path::new("/bin/new"),
18471 Some(std::path::Path::new("/bin/old")),
18472 subc_control::RunningImageAgreement::Unavailable {
18473 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18474 },
18475 );
18476 assert!(matches!(
18477 verdict.path,
18478 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
18479 if configured == std::path::Path::new("/bin/new")
18480 && spawned_from == std::path::Path::new("/bin/old")
18481 ));
18482 }
18483
18484 #[test]
18485 fn launch_spec_without_running_digest_is_unavailable_not_match() {
18486 use subc_control::{
18487 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
18488 };
18489 let configured = crate::supervise::ModuleSpec {
18490 script: None,
18491 carry_containment: false,
18492 module_id: "legacy".into(),
18493 program: "/module".into(),
18494 args: Vec::new(),
18495 env: Vec::new(),
18496 reserved: false,
18497 reserved_prefixes: Vec::new(),
18498 protocol: subc_control::ModuleProtocol::Subc,
18499 overlap: Default::default(),
18500 };
18501 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
18502 assert_eq!(
18503 launch_spec_agreement(&configured, false, Some(&digest)),
18504 Agreement::Unavailable {
18505 reason: Reason::NotRunning
18506 }
18507 );
18508 assert_eq!(
18509 launch_spec_agreement(&configured, true, None),
18510 Agreement::Unavailable {
18511 reason: Reason::SpawnedSpecUnavailable
18512 }
18513 );
18514 assert_eq!(
18515 launch_spec_agreement(&configured, true, Some(&digest)),
18516 Agreement::Match
18517 );
18518 }
18519
18520 #[cfg(windows)]
18521 #[test]
18522 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
18523 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
18524 let configured = dir.join("module.exe");
18525 std::fs::write(&configured, b"executable path fixture").unwrap();
18526 let spawned = std::fs::canonicalize(&configured).unwrap();
18527 assert_ne!(
18528 configured, spawned,
18529 "fixture must exercise different path spellings"
18530 );
18531 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
18532 let lower_case = configured.to_string_lossy().to_lowercase();
18533 for spelling in [
18534 &configured,
18535 Path::new(&forward_slashes),
18536 Path::new(&lower_case),
18537 ] {
18538 let verdict = reload_verdict(
18539 spelling,
18540 Some(&spawned),
18541 subc_control::RunningImageAgreement::Unavailable {
18542 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18543 },
18544 );
18545 assert_eq!(
18546 verdict.path,
18547 ReloadPathAgreement::Match,
18548 "configured {spelling:?}, spawned {spawned:?}"
18549 );
18550 }
18551 }
18552
18553 #[cfg(windows)]
18554 #[test]
18555 fn windows_reload_path_keeps_different_existing_program_pending() {
18556 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
18557 let old = dir.join("old.exe");
18558 let new = dir.join("new.exe");
18559 std::fs::write(&old, b"same executable content").unwrap();
18560 std::fs::copy(&old, &new).unwrap();
18561 let spawned = std::fs::canonicalize(&old).unwrap();
18562 let image = subc_control::RunningImageAgreement::Unavailable {
18563 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18564 };
18565 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
18566 assert!(
18567 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
18568 if configured == new && spawned_from == spawned)
18569 );
18570 let missing = dir.join("missing.exe");
18571 assert!(matches!(
18572 reload_verdict(&missing, Some(&spawned), image).path,
18573 ReloadPathAgreement::Mismatch { .. }
18574 ));
18575 }
18576
18577 #[test]
18578 fn reload_verdict_detects_replaced_image_at_same_path() {
18579 let image = subc_control::RunningImageAgreement::Mismatch {
18580 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
18581 digest: "old".into(),
18582 },
18583 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
18584 digest: "new".into(),
18585 },
18586 };
18587 let verdict = reload_verdict(
18588 std::path::Path::new("/bin/same"),
18589 Some(std::path::Path::new("/bin/same")),
18590 image.clone(),
18591 );
18592 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18593 assert_eq!(verdict.image, image);
18594 }
18595
18596 #[test]
18597 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
18598 let image = subc_control::RunningImageAgreement::Unavailable {
18599 reason: subc_control::RunningImageUnavailableReason::NotRunning,
18600 };
18601 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
18602 assert_eq!(
18603 verdict.path,
18604 subc_control::ReloadPathAgreement::Unavailable {
18605 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
18606 }
18607 );
18608 assert_eq!(verdict.image, image);
18609
18610 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
18611 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18612 };
18613 let verdict = reload_verdict(
18614 std::path::Path::new("/bin/same"),
18615 Some(std::path::Path::new("/bin/same")),
18616 unconfirmed.clone(),
18617 );
18618 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18619 assert_eq!(verdict.image, unconfirmed);
18620 }
18621
18622 #[test]
18623 fn reload_verdict_preserves_each_image_unavailability_reason() {
18624 use subc_control::RunningImageUnavailableReason as Reason;
18625
18626 for reason in [
18627 Reason::NotRunning,
18628 Reason::UnsupportedPlatform,
18629 Reason::RunningExecutableUnreadable,
18630 Reason::SpawnedPathUnreadable,
18631 Reason::HashFailed,
18632 Reason::ProcessIdentityUnconfirmed,
18633 Reason::Unknown("future_probe_reason".to_string()),
18634 ] {
18635 let image = subc_control::RunningImageAgreement::Unavailable {
18636 reason: reason.clone(),
18637 };
18638 let verdict = reload_verdict(
18639 std::path::Path::new("/bin/same"),
18640 Some(std::path::Path::new("/bin/same")),
18641 image.clone(),
18642 );
18643 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18644 assert_eq!(verdict.image, image, "{reason:?}");
18645 }
18646 }
18647
18648 #[tokio::test]
18649 async fn malformed_control_bodies_return_invalid_control_body() {
18650 let handler = ControlHandler::default();
18651 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
18652
18653 for (corr, body) in [
18654 (56, br#"{"route_channel":1}"#.as_slice()),
18655 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
18656 (
18657 58,
18658 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
18659 ),
18660 ] {
18661 let request = Frame::build(
18662 FrameType::Request,
18663 control_flags(),
18664 0,
18665 0,
18666 corr,
18667 body.to_vec(),
18668 )
18669 .unwrap();
18670 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18671
18672 assert_eq!(response.len(), 1);
18673 assert_eq!(response[0].header.ty, FrameType::Error);
18674 assert_eq!(response[0].header.corr, corr);
18675 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
18676 }
18677 }
18678
18679 #[tokio::test]
18680 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
18681 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18682 let registry = Arc::new(Registry::default());
18683 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18684 let router = Router::with_control_handler(Arc::clone(&control));
18685 let connection = router.begin_connection();
18686 let (ctx, mut rx) = route_ctx(connection.id());
18687
18688 router
18689 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
18690 .await
18691 .unwrap();
18692 let response = rx.recv().await.unwrap();
18693 let ack = parse_ack(&response);
18694 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18695 let channel = 1;
18696
18697 let goodbye =
18698 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
18699 router.route_for_connection(&ctx, goodbye).await.unwrap();
18700 assert!(rx.try_recv().is_err());
18701 assert!(registry.get_module("aft").unwrap().is_none());
18702
18703 router
18704 .route_for_connection(&ctx, channel_request(channel, 13))
18705 .await
18706 .unwrap();
18707 let error_frame = rx.recv().await.unwrap();
18708 assert_eq!(error_frame.header.ty, FrameType::Error);
18709 assert_eq!(error_frame.header.channel, channel);
18710 let captured = crate::router::test_log::captured_logs(&logs);
18711 assert_eq!(
18712 captured
18713 .lines()
18714 .filter(|line| {
18715 line.contains("module_id=aft")
18716 && line.contains("reason=explicit_goodbye")
18717 && line.contains("module registration ended")
18718 })
18719 .count(),
18720 1,
18721 "unexpected GOODBYE registry log: {captured}"
18722 );
18723 }
18724
18725 #[tokio::test]
18726 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
18727 let registry = Arc::new(Registry::default());
18728 let handler = ControlHandler::new(registry).with_capability_config(
18729 [("prov".to_string(), true), ("cons".to_string(), true)],
18730 BTreeMap::new(),
18731 );
18732 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
18733 register_capability_manifest(
18734 &handler,
18735 &provider_ctx,
18736 &mut provider_rx,
18737 capability_manifest("prov", &["thing/v1"], &[]),
18738 1,
18739 )
18740 .await;
18741 let mut consumer = capability_manifest("cons", &[], &[]);
18742 consumer.capabilities.as_mut().unwrap().requires.push(
18743 subc_protocol::manifest::CapabilityRequirement {
18744 capability: "thing/v1".to_string(),
18745 need: subc_protocol::manifest::CapabilityNeed::Required,
18746 },
18747 );
18748 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
18749 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
18750 assert_eq!(
18751 handler.capability_evaluator.verdict("cons", "thing/v1"),
18752 Some(CapabilityVerdict::Provided)
18753 );
18754 let (mut client_rx, _) = open_route_for_capability_test(
18755 &handler,
18756 &provider_ctx,
18757 &mut provider_rx,
18758 703,
18759 3,
18760 "prov",
18761 None,
18762 )
18763 .await;
18764 let goodbye =
18765 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18766 handler
18767 .handle_control_frame(&provider_ctx, goodbye)
18768 .await
18769 .unwrap();
18770 assert_eq!(
18771 handler.capability_evaluator.verdict("cons", "thing/v1"),
18772 Some(CapabilityVerdict::NeverProvided)
18773 );
18774 let closed = client_rx
18775 .try_recv()
18776 .expect("GOODBYE pushes route.closed before route GOODBYE");
18777 assert!(
18778 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
18779 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
18780 );
18781 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
18782 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
18783 }
18784
18785 struct RouteClosedFixture {
18786 handler: ControlHandler,
18787 #[cfg(unix)]
18788 supervisor: Supervisor,
18789 module_ctx: RouteCtx,
18790 #[cfg(unix)]
18791 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18792 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18793 route: Arc<crate::forwarding::RouteBinding>,
18794 }
18795
18796 async fn route_closed_fixture() -> RouteClosedFixture {
18797 let registry = Arc::new(Registry::default());
18798 let handler = ControlHandler::new(Arc::clone(®istry));
18799 #[cfg(unix)]
18800 let supervisor = Supervisor::new(registry, RestartPolicy::default())
18801 .with_forwarding(Arc::clone(&handler.forwarding));
18802 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
18803 register_capability_manifest(
18804 &handler,
18805 &module_ctx,
18806 &mut module_rx,
18807 capability_manifest("prov", &[], &[]),
18808 1,
18809 )
18810 .await;
18811 let (client_rx, bind) = open_route_for_capability_test(
18812 &handler,
18813 &module_ctx,
18814 &mut module_rx,
18815 702,
18816 2,
18817 "prov",
18818 None,
18819 )
18820 .await;
18821 let ModuleControlRequest::RouteBind {
18822 route_channel: channel,
18823 epoch: route_epoch,
18824 ..
18825 } = bind
18826 else {
18827 panic!("expected route.bind");
18828 };
18829 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
18830 handler
18831 .forwarding
18832 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
18833 .unwrap()
18834 else {
18835 panic!("the acknowledged route must be bound");
18836 };
18837 RouteClosedFixture {
18838 handler,
18839 #[cfg(unix)]
18840 supervisor,
18841 module_ctx,
18842 #[cfg(unix)]
18843 module_rx,
18844 client_rx,
18845 route,
18846 }
18847 }
18848
18849 impl RouteClosedFixture {
18850 #[cfg(unix)]
18851 async fn cut(&mut self) {
18852 let notices = async {
18853 let mut draining = self.module_rx.recv().await.unwrap();
18854 assert!(matches!(
18855 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
18856 &draining.body
18857 )
18858 .unwrap(),
18859 subc_protocol::session::ModuleControlCommand::Draining {
18860 reason: RouteCloseReason::Disable,
18861 ..
18862 }
18863 ));
18864 draining.flushed.take().unwrap().send(()).unwrap();
18865 let mut closing = self.client_rx.recv().await.unwrap();
18866 assert!(matches!(
18867 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
18868 ClientControlPush::RouteClosing {
18869 reason: RouteCloseReason::Restart,
18870 ..
18871 }
18872 ));
18873 closing.flushed.take().unwrap().send(()).unwrap();
18874 };
18875 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
18876 result.unwrap();
18877 }
18878
18879 fn disconnect(&mut self) -> ClientControlPush {
18880 self.handler
18881 .cleanup_connection(self.module_ctx.connection_id)
18882 .unwrap();
18883 let closed = self.client_rx.try_recv().unwrap();
18884 assert_eq!(closed.header.ty, FrameType::Push);
18885 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
18886 assert!(matches!(
18887 &push,
18888 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
18889 if channels == &[self.route.client_channel]
18890 ));
18891 assert_eq!(
18892 self.client_rx.try_recv().unwrap().header.ty,
18893 FrameType::Goodbye
18894 );
18895 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
18896 push
18897 }
18898 }
18899
18900 #[cfg(unix)]
18901 #[tokio::test(start_paused = true)]
18902 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
18903 let mut fixture = route_closed_fixture().await;
18904 fixture.cut().await;
18905 assert!(matches!(
18906 fixture.disconnect(),
18907 ClientControlPush::RouteClosed {
18908 reason: RouteCloseReason::Restart,
18909 drained: true,
18910 ..
18911 }
18912 ));
18913 }
18914
18915 #[cfg(unix)]
18916 #[tokio::test(start_paused = true)]
18917 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
18918 let mut fixture = route_closed_fixture().await;
18919 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
18920 let started = tokio::time::Instant::now();
18921 fixture.cut().await;
18922 assert!(started.elapsed() >= Duration::from_secs(2));
18923 assert_eq!(fixture.route.flow.in_flight(), 1);
18924 assert!(matches!(
18925 fixture.disconnect(),
18926 ClientControlPush::RouteClosed {
18927 reason: RouteCloseReason::Restart,
18928 drained: false,
18929 ..
18930 }
18931 ));
18932 }
18933
18934 #[tokio::test]
18935 async fn module_connection_crash_reports_crash_and_not_drained() {
18936 let mut fixture = route_closed_fixture().await;
18937 assert!(matches!(
18941 fixture.disconnect(),
18942 ClientControlPush::RouteClosed {
18943 reason: RouteCloseReason::Crash,
18944 drained: false,
18945 ..
18946 }
18947 ));
18948 }
18949
18950 #[tokio::test]
18951 async fn dropping_router_connection_releases_registration() {
18952 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18953 let registry = Arc::new(Registry::default());
18954 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18955 let router = Router::with_control_handler(Arc::clone(&control));
18956 let connection = router.begin_connection();
18957 let connection_id = connection.id();
18958 let (ctx, mut rx) = route_ctx(connection_id);
18959
18960 router
18961 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
18962 .await
18963 .unwrap();
18964 let response = rx.recv().await.unwrap();
18965 let ack = parse_ack(&response);
18966 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18967 assert!(registry.get_module("aft").unwrap().is_some());
18968
18969 drop(connection);
18970
18971 assert!(registry.get_module("aft").unwrap().is_none());
18972 assert_eq!(registry.active_registration_count().unwrap(), 0);
18973
18974 control
18975 .cleanup_connection(ConnectionId::new(u64::MAX))
18976 .unwrap();
18977 let captured = crate::router::test_log::captured_logs(&logs);
18978 println!("captured registry lifecycle logs:\n{captured}");
18979 let events: Vec<_> = captured
18980 .lines()
18981 .filter(|line| {
18982 line.contains("module registered module_id=aft ")
18983 || line.contains("module registration ended")
18984 })
18985 .collect();
18986 assert_eq!(
18987 events.len(),
18988 2,
18989 "unexpected registry lifecycle logs: {captured}"
18990 );
18991 assert!(events[0].contains("module registered module_id=aft "));
18992 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
18993 assert!(events[1].contains(&format!(
18994 "module_id=aft connection_id={}",
18995 connection_id.get()
18996 )));
18997 assert!(events[1].contains("reason=connection_closed"));
18998 assert!(events[1].contains("module registration ended"));
18999 assert_eq!(
19000 captured
19001 .lines()
19002 .filter(|line| line.contains("module registered module_id=aft "))
19003 .count(),
19004 1,
19005 "legacy registration admission line must appear once: {captured}"
19006 );
19007 }
19008
19009 #[tokio::test]
19010 async fn hello_registration_keeps_legacy_module_registered_line_once() {
19011 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19012 let registry = Arc::new(Registry::default());
19013 let control = ControlHandler::new(Arc::clone(®istry));
19014 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
19015 hello_via_sink(
19016 &control,
19017 &ctx,
19018 &mut rx,
19019 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
19020 )
19021 .await;
19022
19023 let captured = crate::router::test_log::captured_logs(&logs);
19024 let admissions: Vec<_> = captured
19025 .lines()
19026 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
19027 .collect();
19028 assert_eq!(
19029 admissions.len(),
19030 1,
19031 "legacy admission line must remain exactly once: {captured}"
19032 );
19033 assert!(admissions[0].contains("routable_provider=true"));
19034 assert!(admissions[0].contains("connection_id=777"));
19035 }
19036
19037 fn capability_manifest(
19038 module_id: &str,
19039 provides: &[&str],
19040 must_never_reach: &[&str],
19041 ) -> ModuleManifest {
19042 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
19043 manifest.capabilities = Some(CapabilityDeclarations {
19044 provides: provides
19045 .iter()
19046 .map(|capability| (*capability).to_string())
19047 .collect(),
19048 requires: Vec::new(),
19049 must_never_reach: must_never_reach
19050 .iter()
19051 .map(|capability| (*capability).to_string())
19052 .collect(),
19053 });
19054 manifest
19055 }
19056
19057 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
19058 Frame::build(
19059 FrameType::Hello,
19060 control_flags(),
19061 0,
19062 0,
19063 corr,
19064 serde_json::to_vec(&ModuleHelloBody {
19065 protocol_ver: manifest.protocol_ver,
19066 manifest,
19067 control_ops: None,
19068 launch_nonce: None,
19069 })
19070 .expect("capability test HELLO serializes"),
19071 )
19072 .expect("capability test HELLO frame builds")
19073 }
19074
19075 fn catalog_update_with_capabilities_frame(
19076 corr: u64,
19077 capabilities: CapabilityDeclarations,
19078 ) -> Frame {
19079 Frame::build(
19080 FrameType::Request,
19081 control_flags(),
19082 0,
19083 0,
19084 corr,
19085 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
19086 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
19087 capabilities: Some(capabilities),
19088 ready: None,
19089 })
19090 .expect("capability catalog.update serializes"),
19091 )
19092 .expect("capability catalog.update frame builds")
19093 }
19094
19095 async fn register_capability_manifest(
19096 handler: &ControlHandler,
19097 ctx: &RouteCtx,
19098 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19099 manifest: ModuleManifest,
19100 corr: u64,
19101 ) {
19102 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
19103 }
19104
19105 async fn open_route_for_capability_test(
19106 handler: &ControlHandler,
19107 target_ctx: &RouteCtx,
19108 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19109 client_connection_id: u64,
19110 corr: u64,
19111 target_module_id: &str,
19112 consumer_identity: Option<ConsumerIdentity>,
19113 ) -> (
19114 mpsc::Receiver<crate::router::OutboundFrame>,
19115 ModuleControlRequest,
19116 ) {
19117 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
19118 let route_handler = handler.clone();
19119 let target_module_id = target_module_id.to_string();
19120 let route_task = tokio::spawn(async move {
19121 route_handler
19122 .handle_control_frame(
19123 &client_ctx,
19124 route_open_frame_with_admission_facts(
19125 corr,
19126 &target_module_id,
19127 unique_project_root("admission-facts"),
19128 consumer_identity,
19129 None,
19130 ),
19131 )
19132 .await
19133 .expect("capability test route.open succeeds")
19134 });
19135 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
19136 .await
19137 .expect("capability test route.open must reach route.bind")
19138 .expect("target control receiver stays open");
19139 let bind_request: ModuleControlRequest =
19140 serde_json::from_slice(&bind.body).expect("route.bind decodes");
19141 handler
19142 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
19143 .await
19144 .expect("capability test route.bind ACK succeeds");
19145 assert!(route_task.await.expect("route.open task joins").is_empty());
19146 let opened = client_rx
19147 .recv()
19148 .await
19149 .expect("successful route.open publishes a response");
19150 assert!(matches!(
19151 serde_json::from_slice::<ClientControlResponse>(&opened.body),
19152 Ok(ClientControlResponse::RouteOpen { .. })
19153 ));
19154 (client_rx, bind_request)
19155 }
19156
19157 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
19158 assert_eq!(frame.header.ty, FrameType::Push);
19159 assert_eq!(frame.header.channel, 0);
19160 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
19161 .expect("route.closed control push decodes");
19162 let ClientControlPush::RouteClosed { channels, .. } = &push else {
19163 panic!("expected route.closed");
19164 };
19165 assert_eq!(channels.len(), 1, "exactly one violating route closed");
19166 let closed_channel = channels[0];
19167 let channels = channels.clone();
19168 assert_eq!(
19169 push,
19170 ClientControlPush::RouteClosed {
19171 module_id: target_module_id.to_string(),
19172 channels,
19173 reason: RouteCloseReason::CapabilityDenied,
19174 drained: false,
19175 abandoned: 0,
19176 excluded_subscriptions: 0,
19177 terminal: Some(false),
19178 }
19179 );
19180 closed_channel
19181 }
19182
19183 #[tokio::test]
19184 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
19185 let registry = Arc::new(Registry::default());
19186 let forwarding = Arc::new(ForwardingTable::default());
19187 let supervisor = SupervisorHandle::new();
19188 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19189 let handler =
19190 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19191 .with_supervisor(supervisor);
19192 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
19193 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
19194 register_capability_manifest(
19195 &handler,
19196 &target_ctx,
19197 &mut target_rx,
19198 capability_manifest("target", &["credentials-provider/v1"], &[]),
19199 1,
19200 )
19201 .await;
19202 register_capability_manifest(
19203 &handler,
19204 &opener_ctx,
19205 &mut opener_rx,
19206 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19207 2,
19208 )
19209 .await;
19210
19211 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
19212 let replies = handler
19213 .handle_control_frame(
19214 &client_ctx,
19215 route_open_frame_with_admission_facts(
19216 3,
19217 "target",
19218 unique_project_root("admission-facts"),
19219 Some(ConsumerIdentity {
19220 module_id: "opener".to_string(),
19221 launch_nonce: "opener-nonce".to_string(),
19222 }),
19223 None,
19224 ),
19225 )
19226 .await
19227 .expect("denied route.open returns a typed frame");
19228 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19229 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19230 assert!(
19231 target_rx.try_recv().is_err(),
19232 "forbidden route.open must not relay route.bind"
19233 );
19234 }
19235
19236 #[tokio::test]
19237 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
19238 let registry = Arc::new(Registry::default());
19239 let forwarding = Arc::new(ForwardingTable::default());
19240 let supervisor = SupervisorHandle::new();
19241 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19242 let handler =
19243 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19244 .with_supervisor(supervisor);
19245 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
19246 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
19247 register_capability_manifest(
19248 &handler,
19249 &target_ctx,
19250 &mut target_rx,
19251 capability_manifest("target", &["credentials-provider/v1"], &[]),
19252 1,
19253 )
19254 .await;
19255 register_capability_manifest(
19256 &handler,
19257 &old_opener_ctx,
19258 &mut old_opener_rx,
19259 capability_manifest("opener", &[], &[]),
19260 2,
19261 )
19262 .await;
19263 let (mut client_rx, _) = open_route_for_capability_test(
19264 &handler,
19265 &target_ctx,
19266 &mut target_rx,
19267 712,
19268 3,
19269 "target",
19270 Some(ConsumerIdentity {
19271 module_id: "opener".to_string(),
19272 launch_nonce: "opener-nonce".to_string(),
19273 }),
19274 )
19275 .await;
19276 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19277
19278 handler
19279 .cleanup_connection(old_opener_ctx.connection_id)
19280 .expect("old opener registration cleans up");
19281 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
19282 register_capability_manifest(
19283 &handler,
19284 &new_opener_ctx,
19285 &mut new_opener_rx,
19286 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19287 4,
19288 )
19289 .await;
19290
19291 let closed_channel = assert_capability_denied_push(
19292 client_rx
19293 .try_recv()
19294 .expect("HELLO deny addition must emit route.closed")
19295 .frame,
19296 "target",
19297 );
19298 let goodbye = client_rx
19299 .try_recv()
19300 .expect("HELLO deny addition must send the client a route GOODBYE")
19301 .frame;
19302 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19303 assert_eq!(goodbye.header.channel, closed_channel);
19304 assert_ne!(goodbye.header.channel, 0);
19305 assert_ne!(goodbye.header.epoch, 0);
19306 assert!(client_rx.try_recv().is_err());
19307 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19308 assert!(matches!(
19309 target_rx.try_recv(),
19310 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19311 ));
19312 }
19313
19314 #[tokio::test]
19315 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
19316 let registry = Arc::new(Registry::default());
19317 let forwarding = Arc::new(ForwardingTable::default());
19318 let supervisor = SupervisorHandle::new();
19319 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19320 let handler =
19321 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19322 .with_supervisor(supervisor);
19323 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
19324 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
19325 register_capability_manifest(
19326 &handler,
19327 &target_ctx,
19328 &mut target_rx,
19329 capability_manifest("target", &[], &[]),
19330 1,
19331 )
19332 .await;
19333 register_capability_manifest(
19334 &handler,
19335 &opener_ctx,
19336 &mut opener_rx,
19337 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19338 2,
19339 )
19340 .await;
19341 let (mut client_rx, _) = open_route_for_capability_test(
19342 &handler,
19343 &target_ctx,
19344 &mut target_rx,
19345 722,
19346 3,
19347 "target",
19348 Some(ConsumerIdentity {
19349 module_id: "opener".to_string(),
19350 launch_nonce: "opener-nonce".to_string(),
19351 }),
19352 )
19353 .await;
19354 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19355
19356 let replies = handler
19357 .handle_control_frame(
19358 &target_ctx,
19359 catalog_update_with_capabilities_frame(
19360 4,
19361 CapabilityDeclarations {
19362 provides: vec!["credentials-provider/v1".to_string()],
19363 requires: Vec::new(),
19364 must_never_reach: Vec::new(),
19365 },
19366 ),
19367 )
19368 .await
19369 .expect("claim catalog.update succeeds");
19370 assert!(matches!(
19371 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
19372 Ok(ModuleControlResponseToModule::CatalogUpdate {})
19373 ));
19374 let closed_channel = assert_capability_denied_push(
19375 client_rx
19376 .try_recv()
19377 .expect("claim addition must emit route.closed")
19378 .frame,
19379 "target",
19380 );
19381 let goodbye = client_rx
19382 .try_recv()
19383 .expect("claim addition must send the client a route GOODBYE")
19384 .frame;
19385 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19386 assert_eq!(goodbye.header.channel, closed_channel);
19387 assert_ne!(goodbye.header.channel, 0);
19388 assert_ne!(goodbye.header.epoch, 0);
19389 assert!(client_rx.try_recv().is_err());
19390 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19391 assert!(matches!(
19392 target_rx.try_recv(),
19393 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19394 ));
19395 }
19396
19397 #[tokio::test]
19398 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
19399 let registry = Arc::new(Registry::default());
19400 let forwarding = Arc::new(ForwardingTable::default());
19401 let supervisor = SupervisorHandle::new();
19402 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19403 let handler =
19404 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19405 .with_supervisor(supervisor);
19406 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
19407 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
19408 register_capability_manifest(
19409 &handler,
19410 &target_ctx,
19411 &mut target_rx,
19412 capability_manifest("target", &["credentials-provider/v1"], &[]),
19413 1,
19414 )
19415 .await;
19416 register_capability_manifest(
19417 &handler,
19418 &opener_ctx,
19419 &mut opener_rx,
19420 capability_manifest("opener", &[], &[]),
19421 2,
19422 )
19423 .await;
19424 let (mut client_rx, _) = open_route_for_capability_test(
19425 &handler,
19426 &target_ctx,
19427 &mut target_rx,
19428 732,
19429 3,
19430 "target",
19431 Some(ConsumerIdentity {
19432 module_id: "opener".to_string(),
19433 launch_nonce: "opener-nonce".to_string(),
19434 }),
19435 )
19436 .await;
19437 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19438
19439 handler
19440 .handle_control_frame(
19441 &target_ctx,
19442 catalog_update_with_capabilities_frame(
19443 4,
19444 CapabilityDeclarations {
19445 provides: Vec::new(),
19446 requires: Vec::new(),
19447 must_never_reach: Vec::new(),
19448 },
19449 ),
19450 )
19451 .await
19452 .expect("claim removal catalog.update succeeds");
19453 assert_eq!(
19454 forwarding.active_binding_count().unwrap(),
19455 1,
19456 "removing an attested target claim must leave the route census unchanged"
19457 );
19458 assert!(
19459 client_rx.try_recv().is_err(),
19460 "claim removal must not emit route.closed or a client route GOODBYE"
19461 );
19462 assert!(
19463 target_rx.try_recv().is_err(),
19464 "claim removal must not send the target a route GOODBYE"
19465 );
19466 }
19467
19468 #[tokio::test]
19471 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
19472 let registry = Arc::new(Registry::default());
19473 let forwarding = Arc::new(ForwardingTable::default());
19474 let supervisor = SupervisorHandle::new();
19475 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19476 let handler =
19477 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19478 .with_supervisor(supervisor);
19479 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
19480 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
19481 register_capability_manifest(
19482 &handler,
19483 &target_ctx,
19484 &mut target_rx,
19485 capability_manifest("target", &["credentials-provider/v1"], &[]),
19486 1,
19487 )
19488 .await;
19489 register_capability_manifest(
19490 &handler,
19491 &opener_ctx,
19492 &mut opener_rx,
19493 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19494 2,
19495 )
19496 .await;
19497
19498 let (_client_rx, bind) = open_route_for_capability_test(
19499 &handler,
19500 &target_ctx,
19501 &mut target_rx,
19502 742,
19503 3,
19504 "target",
19505 None,
19506 )
19507 .await;
19508 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
19509 panic!("direct scope-honesty route must bind");
19510 };
19511 assert_eq!(principal, Some(Principal::Direct));
19512 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19513 }
19514
19515 #[tokio::test]
19518 async fn must_never_reach_self_route_is_capability_forbidden() {
19519 let registry = Arc::new(Registry::default());
19520 let forwarding = Arc::new(ForwardingTable::default());
19521 let supervisor = SupervisorHandle::new();
19522 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
19523 let handler =
19524 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19525 .with_supervisor(supervisor);
19526 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
19527 register_capability_manifest(
19528 &handler,
19529 &self_ctx,
19530 &mut self_rx,
19531 capability_manifest(
19532 "self-provider",
19533 &["credentials-provider/v1"],
19534 &["credentials-provider/v1"],
19535 ),
19536 1,
19537 )
19538 .await;
19539
19540 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
19541 let replies = handler
19542 .handle_control_frame(
19543 &client_ctx,
19544 route_open_frame_with_admission_facts(
19545 2,
19546 "self-provider",
19547 unique_project_root("admission-facts"),
19548 Some(ConsumerIdentity {
19549 module_id: "self-provider".to_string(),
19550 launch_nonce: "self-nonce".to_string(),
19551 }),
19552 None,
19553 ),
19554 )
19555 .await
19556 .expect("self-route refusal returns a typed frame");
19557 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19558 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19559 assert!(
19560 self_rx.try_recv().is_err(),
19561 "self denial must not relay route.bind"
19562 );
19563 }
19564
19565 #[test]
19566 fn unsupported_channel_zero_frame_returns_error() {
19567 let handler = ControlHandler::default();
19568 let request = Frame::build(
19569 FrameType::Request,
19570 control_flags(),
19571 0,
19572 0,
19573 21,
19574 b"opaque".to_vec(),
19575 )
19576 .unwrap();
19577
19578 let response = handler
19579 .handle_control(ConnectionId::new(1), request)
19580 .unwrap();
19581
19582 assert_eq!(response[0].header.ty, FrameType::Error);
19583 assert_eq!(
19584 parse_error(&response[0])["code"],
19585 "unsupported_control_frame"
19586 );
19587 }
19588
19589 mod swap {
19594 use super::*;
19595
19596 const INCUMBENT: ConnectionId = ConnectionId::new(30);
19597 const CANDIDATE: ConnectionId = ConnectionId::new(40);
19598
19599 struct Swap {
19600 registry: Arc<Registry>,
19601 forwarding: Arc<ForwardingTable>,
19602 handler: ControlHandler,
19603 incumbent_ctx: RouteCtx,
19604 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19605 candidate_ctx: RouteCtx,
19606 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19607 }
19608
19609 async fn swap_with_incumbent() -> Swap {
19610 let registry = Arc::new(Registry::default());
19611 let forwarding = Arc::new(ForwardingTable::default());
19612 let handler =
19613 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
19614 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
19615 hello_via_sink(
19616 &handler,
19617 &incumbent_ctx,
19618 &mut incumbent_rx,
19619 hello_frame("aft", PROTOCOL_VERSION, 7),
19620 )
19621 .await;
19622 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
19623 Swap {
19624 registry,
19625 forwarding,
19626 handler,
19627 incumbent_ctx,
19628 incumbent_rx,
19629 candidate_ctx,
19630 candidate_rx,
19631 }
19632 }
19633
19634 fn register_candidate(swap: &Swap, ready: Option<bool>) {
19635 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
19636 candidate_manifest.ready = ready;
19637 let registration = swap
19638 .registry
19639 .register_candidate_with_control_ops(
19640 candidate_manifest,
19641 PROTOCOL_VERSION,
19642 CANDIDATE,
19643 module_baseline_control_ops(),
19644 )
19645 .unwrap();
19646 swap.forwarding
19647 .register_candidate_module_connection(
19648 CANDIDATE,
19649 "aft".to_string(),
19650 PROTOCOL_VERSION,
19651 manifest_concurrency(®istration.manifest),
19652 swap.candidate_ctx.egress.clone(),
19653 )
19654 .unwrap();
19655 }
19656
19657 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
19658 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
19659 swap.registry.promote_candidate("aft").unwrap().unwrap();
19660 cutover.incumbent.unwrap()
19661 }
19662
19663 #[tokio::test]
19672 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
19673 let mut swap = swap_with_incumbent().await;
19674 let hold = Arc::new(std::sync::Barrier::new(2));
19675 let handler = swap
19676 .handler
19677 .clone()
19678 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
19679 handler
19680 .supervisor
19681 .open_swap("aft", "candidate-nonce".to_string());
19682 let hello = {
19683 let handler = handler.clone();
19684 let candidate_ctx = swap.candidate_ctx.clone();
19685 std::thread::spawn(move || {
19686 handler
19687 .handle_hello(
19688 candidate_ctx.connection_id,
19689 Some(candidate_ctx.egress.clone()),
19690 hello_frame_with_nonce(
19691 "aft",
19692 PROTOCOL_VERSION,
19693 8,
19694 Some("candidate-nonce"),
19695 ),
19696 )
19697 .unwrap()
19698 })
19699 };
19700
19701 hold.wait();
19702 assert!(swap
19705 .registry
19706 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
19707 .unwrap()
19708 .is_some());
19709 assert!(swap
19710 .registry
19711 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19712 .unwrap()
19713 .is_none());
19714 assert!(swap
19715 .forwarding
19716 .module_endpoint_for_connection(CANDIDATE)
19717 .unwrap()
19718 .is_none());
19719
19720 hold.wait();
19721 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
19722 assert_eq!(
19723 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19724 FrameType::HelloAck
19725 );
19726 let published = swap
19727 .registry
19728 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19729 .unwrap()
19730 .expect("candidate is published once its HELLO completes");
19731 assert_eq!(published.connection_id, CANDIDATE);
19732 assert!(swap
19733 .forwarding
19734 .module_endpoint_for_connection(CANDIDATE)
19735 .unwrap()
19736 .is_some());
19737 }
19738
19739 #[tokio::test]
19740 async fn candidate_lookup_after_hello_ack_waits_for_publication() {
19741 let mut swap = swap_with_incumbent().await;
19742 let hold = Arc::new(std::sync::Barrier::new(2));
19743 swap.handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
19744 swap.handler
19745 .supervisor
19746 .open_swap("aft", "candidate-nonce".to_string());
19747 let hello = {
19748 let handler = swap.handler.clone();
19749 let candidate_ctx = swap.candidate_ctx.clone();
19750 std::thread::spawn(move || {
19751 handler
19752 .handle_hello(
19753 CANDIDATE,
19754 Some(candidate_ctx.egress),
19755 hello_frame_with_nonce(
19756 "aft",
19757 PROTOCOL_VERSION,
19758 8,
19759 Some("candidate-nonce"),
19760 ),
19761 )
19762 .unwrap()
19763 })
19764 };
19765 hold.wait();
19766 assert_eq!(
19767 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19768 FrameType::HelloAck
19769 );
19770 let reader_registry = Arc::clone(&swap.registry);
19771 let published = lookup_during_hello_publication(&swap.registry, &hold, move || {
19772 reader_registry
19773 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19774 .unwrap()
19775 });
19776 assert!(hello.join().unwrap().is_empty());
19777 assert_eq!(
19778 published
19779 .expect("ACK makes candidate publication visible")
19780 .connection_id,
19781 CANDIDATE
19782 );
19783 assert_eq!(
19784 swap.registry.generation().unwrap(),
19785 1,
19786 "candidate is not a second catalog entry"
19787 );
19788 }
19789
19790 #[tokio::test]
19791 async fn audit4_ungated_removed_candidate_publication_rolls_back_admission() {
19792 let mut swap = swap_with_incumbent().await;
19793 let hold = Arc::new(std::sync::Barrier::new(2));
19794 let handler = swap
19795 .handler
19796 .clone()
19797 .with_hello_registry_to_forwarding_hold(hold.clone());
19798 handler
19799 .supervisor
19800 .open_swap("aft", "candidate-nonce".into());
19801 let hello = {
19802 let handler = handler.clone();
19803 let ctx = swap.candidate_ctx.clone();
19804 std::thread::spawn(move || {
19805 handler
19806 .handle_hello(
19807 CANDIDATE,
19808 Some(ctx.egress),
19809 hello_frame_with_nonce(
19810 "aft",
19811 PROTOCOL_VERSION,
19812 8,
19813 Some("candidate-nonce"),
19814 ),
19815 )
19816 .unwrap()
19817 })
19818 };
19819 hold.wait();
19820 assert_eq!(
19821 swap.registry
19822 .deregister_connection(CANDIDATE)
19823 .unwrap()
19824 .len(),
19825 1
19826 );
19827 hold.wait();
19828 assert!(hello.join().unwrap().is_empty());
19829 assert_eq!(
19830 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19831 FrameType::HelloAck
19832 );
19833 assert!(swap.registry.get_candidate("aft").unwrap().is_none());
19834 assert_eq!(
19835 handler
19836 .supervisor
19837 .swap_hello_admission("aft", Some("candidate-nonce")),
19838 SwapHelloAdmission::Candidate,
19839 "failed publication must roll back candidate_admitted for an ungated nonce swap"
19840 );
19841 handler.supervisor.close_swap("aft");
19842 }
19843
19844 fn keyed_total(counters: &Value, key: &str) -> u64 {
19845 counters[key]
19846 .as_object()
19847 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
19848 .unwrap_or(0)
19849 }
19850
19851 #[tokio::test]
19852 async fn replacement_logs_old_end_and_new_admission_once() {
19853 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19854 let mut swap = swap_with_incumbent().await;
19855 swap.handler
19856 .supervisor
19857 .open_swap("aft", "candidate-nonce".to_string());
19858 hello_via_sink(
19859 &swap.handler,
19860 &swap.candidate_ctx,
19861 &mut swap.candidate_rx,
19862 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
19863 )
19864 .await;
19865 cutover(&swap);
19866 swap.handler.cleanup_connection(INCUMBENT).unwrap();
19867
19868 let captured = crate::router::test_log::captured_logs(&logs);
19869 println!("captured replacement registry logs:\n{captured}");
19870 let new_admissions: Vec<_> = captured
19871 .lines()
19872 .filter(|line| {
19873 line.contains("connection_id=40")
19874 && line.contains("swap candidate registered; not routable until cutover")
19875 })
19876 .collect();
19877 assert_eq!(
19878 new_admissions.len(),
19879 1,
19880 "unexpected admission logs: {captured}"
19881 );
19882 assert!(new_admissions[0].contains("module_id=aft"));
19883 assert!(new_admissions[0].contains("connection_id=40"));
19884 assert!(
19885 new_admissions[0].contains("swap candidate registered; not routable until cutover")
19886 );
19887 assert!(new_admissions[0].contains("ready=true"));
19888
19889 let old_admissions: Vec<_> = captured
19890 .lines()
19891 .filter(|line| {
19892 line.contains("module registered module_id=aft ")
19893 && line.contains("connection_id=30")
19894 })
19895 .collect();
19896 assert_eq!(
19897 old_admissions.len(),
19898 1,
19899 "unexpected incumbent admission: {captured}"
19900 );
19901
19902 let old_ends: Vec<_> = captured
19903 .lines()
19904 .filter(|line| {
19905 line.contains("connection_id=30") && line.contains("module registration ended")
19906 })
19907 .collect();
19908 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
19909 assert!(old_ends[0].contains("module_id=aft"));
19910 assert!(old_ends[0].contains("connection_id=30"));
19911 assert!(old_ends[0].contains("reason=replaced"));
19912 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
19913 assert_eq!(
19914 captured
19915 .lines()
19916 .filter(|line| line.contains("module registration promoted"))
19917 .count(),
19918 1,
19919 "unexpected promotion logs: {captured}"
19920 );
19921 }
19922
19923 #[tokio::test]
19929 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
19930 let mut swap = swap_with_incumbent().await;
19931 let handler = swap.handler.clone();
19932
19933 let cotenant = ConnectionId::new(31);
19935 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
19936 let (cotenant_task, cotenant_bind) = relay_route_open(
19937 &handler,
19938 cotenant,
19939 &cotenant_ctx.egress,
19940 &mut swap.incumbent_rx,
19941 100,
19942 "aft",
19943 "swap-cotenant",
19944 )
19945 .await;
19946 handler
19947 .handle_control_frame(
19948 &swap.incumbent_ctx,
19949 route_bind_ack(cotenant_bind.header.corr),
19950 )
19951 .await
19952 .unwrap();
19953 assert!(cotenant_task.await.unwrap().is_empty());
19954 let (cotenant_channel, cotenant_epoch) =
19955 published_route(&cotenant_rx.recv().await.unwrap());
19956
19957 let caller = ConnectionId::new(32);
19959 let (caller_ctx, mut caller_rx) = route_ctx(caller);
19960 let (caller_task, caller_bind) = relay_route_open(
19961 &handler,
19962 caller,
19963 &caller_ctx.egress,
19964 &mut swap.incumbent_rx,
19965 101,
19966 "aft",
19967 "swap-caller",
19968 )
19969 .await;
19970 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
19971
19972 register_candidate(&swap, None);
19973 cutover(&swap);
19974
19975 let ack = handler
19977 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
19978 .await;
19979 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
19980 if module_loop_error.is_some() {
19981 handler.cleanup_connection(INCUMBENT).unwrap();
19984 }
19985
19986 assert!(
19988 cotenant_rx.try_recv().is_err(),
19989 "the co-tenant route on the incumbent was torn down by one late ack: \
19990 {module_loop_error:?}"
19991 );
19992 assert!(matches!(
19993 swap.forwarding
19994 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
19995 .unwrap(),
19996 DataRoute::Client(DataRouteState::Bound(_))
19997 ));
19998 assert_eq!(module_loop_error, None);
19999 assert!(swap
20000 .registry
20001 .get_module_by_connection(INCUMBENT)
20002 .unwrap()
20003 .is_some());
20004
20005 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
20007 .await
20008 .expect("the incumbent is told to drop the abandoned binding")
20009 .unwrap()
20010 .frame;
20011 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
20012 assert_eq!(goodbye.header.channel, abandoned_channel);
20013 assert_eq!(goodbye.header.epoch, abandoned_epoch);
20014 assert!(swap.incumbent_rx.try_recv().is_err());
20015
20016 let response = caller_task.await.unwrap();
20018 assert_eq!(response.len(), 1);
20019 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
20020 assert!(caller_rx.try_recv().is_err());
20021
20022 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
20026 let counters = handler.counters().snapshot();
20027 assert_eq!(
20028 keyed_total(&counters, "route_open_accepted_by_principal"),
20029 1
20030 );
20031 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
20032 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
20033 }
20034
20035 #[tokio::test]
20039 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
20040 let mut swap = swap_with_incumbent().await;
20041 register_candidate(&swap, None);
20042 let incumbent = cutover(&swap);
20043 swap.forwarding
20044 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
20045 .unwrap()
20046 .expect("the incumbent is still registered");
20047
20048 let client = ConnectionId::new(33);
20049 let (client_ctx, mut client_rx) = route_ctx(client);
20050 let route_handler = swap.handler.clone();
20051 let open_ctx = RouteCtx {
20052 connection_id: client,
20053 egress: client_ctx.egress.clone(),
20054 };
20055 let mut route_task = tokio::spawn(async move {
20056 route_handler
20057 .handle_control_frame(
20058 &open_ctx,
20059 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
20060 )
20061 .await
20062 .unwrap()
20063 });
20064 let bind = tokio::select! {
20065 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
20066 response = &mut route_task => {
20067 let response = response.unwrap();
20068 panic!(
20069 "post-cutover route.open was refused instead of relayed to the candidate: {}",
20070 parse_error(&response[0])["code"]
20071 );
20072 }
20073 };
20074 swap.handler
20075 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
20076 .await
20077 .unwrap();
20078 assert!(route_task.await.unwrap().is_empty());
20079 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
20080 match swap
20081 .forwarding
20082 .lookup_data_route(client, channel, epoch)
20083 .unwrap()
20084 {
20085 DataRoute::Client(DataRouteState::Bound(route)) => {
20086 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
20087 }
20088 other => panic!("expected a bound route on the candidate, got {other:?}"),
20089 }
20090 assert!(swap.incumbent_rx.try_recv().is_err());
20091 }
20092
20093 #[tokio::test]
20098 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
20099 let swap = swap_with_incumbent().await;
20100 register_candidate(&swap, Some(false));
20101 let update = Frame::build(
20102 FrameType::Request,
20103 control_flags(),
20104 0,
20105 0,
20106 55,
20107 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
20108 provides: manifest("aft", PROTOCOL_VERSION).provides,
20109 capabilities: None,
20110 ready: Some(true),
20111 })
20112 .unwrap(),
20113 )
20114 .unwrap();
20115
20116 let replies = swap
20117 .handler
20118 .handle_control_frame(&swap.candidate_ctx, update)
20119 .await
20120 .unwrap();
20121
20122 assert_eq!(replies.len(), 1);
20123 assert_eq!(
20124 replies[0].header.ty,
20125 FrameType::Response,
20126 "candidate catalog.update was refused: {:?}",
20127 serde_json::from_slice::<Value>(&replies[0].body).ok()
20128 );
20129 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
20130 assert_eq!(
20131 swap.registry
20132 .get_module("aft")
20133 .unwrap()
20134 .unwrap()
20135 .connection_id,
20136 INCUMBENT
20137 );
20138 }
20139 }
20140
20141 mod swap_admission {
20145 use super::*;
20146
20147 const INCUMBENT_NONCE: &str = "incumbent-nonce";
20148 const CANDIDATE_NONCE: &str = "candidate-nonce";
20149
20150 fn handler_with_incumbent(
20151 module_id: &str,
20152 reserved: bool,
20153 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
20154 let registry = Arc::new(Registry::default());
20155 let supervisor = SupervisorHandle::new();
20156 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
20157 if reserved {
20158 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
20159 }
20160 let handler =
20161 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
20162 let incumbent = handler
20163 .handle_control(
20164 ConnectionId::new(1),
20165 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
20166 )
20167 .unwrap();
20168 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
20169 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
20170 (registry, supervisor, handler)
20171 }
20172
20173 #[test]
20179 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
20180 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
20181
20182 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
20183 let replies = handler
20184 .handle_control(
20185 ConnectionId::new(connection),
20186 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
20187 )
20188 .unwrap();
20189 assert_eq!(replies[0].header.ty, FrameType::Error);
20190 assert_eq!(
20191 parse_error(&replies[0])["code"],
20192 "swap_token_invalid",
20193 "nonce {nonce:?}"
20194 );
20195 }
20196 assert!(registry.get_candidate("aft").unwrap().is_none());
20197 assert_eq!(
20198 registry.get_module("aft").unwrap().unwrap().connection_id,
20199 ConnectionId::new(1)
20200 );
20201
20202 let admitted = handler
20205 .handle_control(
20206 ConnectionId::new(4),
20207 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
20208 )
20209 .unwrap();
20210 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
20211 assert_eq!(
20212 registry
20213 .get_candidate("aft")
20214 .unwrap()
20215 .unwrap()
20216 .connection_id,
20217 ConnectionId::new(4)
20218 );
20219 assert_eq!(
20220 registry.get_module("aft").unwrap().unwrap().connection_id,
20221 ConnectionId::new(1),
20222 "the candidate must not take the active slot"
20223 );
20224 let replayed = handler
20225 .handle_control(
20226 ConnectionId::new(5),
20227 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
20228 )
20229 .unwrap();
20230 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
20231
20232 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
20236 let squatter = handler
20237 .handle_control(
20238 ConnectionId::new(6),
20239 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
20240 )
20241 .unwrap();
20242 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
20243 assert!(
20244 registry.get_module("aft").unwrap().is_none(),
20245 "a squatter took the active slot of an id being swapped"
20246 );
20247 }
20248
20249 #[test]
20254 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
20255 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
20256
20257 let replies = handler
20258 .handle_control(
20259 ConnectionId::new(2),
20260 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20261 )
20262 .unwrap();
20263
20264 assert_eq!(
20265 replies[0].header.ty,
20266 FrameType::HelloAck,
20267 "reserved candidate refused: {:?}",
20268 serde_json::from_slice::<Value>(&replies[0].body).ok()
20269 );
20270 assert_eq!(
20271 registry
20272 .get_candidate("vault")
20273 .unwrap()
20274 .unwrap()
20275 .connection_id,
20276 ConnectionId::new(2)
20277 );
20278 }
20279
20280 #[test]
20283 fn without_an_open_swap_the_ordinary_gates_decide() {
20284 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
20285 supervisor.close_swap("vault");
20286
20287 let candidate = handler
20288 .handle_control(
20289 ConnectionId::new(2),
20290 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20291 )
20292 .unwrap();
20293 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
20294 let duplicate = handler
20295 .handle_control(
20296 ConnectionId::new(3),
20297 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
20298 )
20299 .unwrap();
20300 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
20301 assert!(registry.get_candidate("vault").unwrap().is_none());
20302 }
20303 }
20304
20305 mod scopes {
20309 mod close_reasons {
20310 include!("route_close_reason_tests.rs");
20311 }
20312 mod idle_tests {
20313 use super::*;
20314 include!("scope_idle_tests.rs");
20315 }
20316 mod route_open_timings {
20317 use super::*;
20318 include!("route_open_timing_tests.rs");
20319 }
20320 mod route_open_refusal_log {
20321 use super::*;
20322 include!("route_open_refusal_log_tests.rs");
20323 }
20324 mod activity {
20325 include!("activity_scope_tests.rs");
20326 }
20327 mod delta {
20328 include!("scope_handler_delta_tests.rs");
20329 }
20330 mod watch {
20331 include!("scope_watch_tests.rs");
20332 }
20333 mod call_modes {
20334 include!("scope_call_mode_tests.rs");
20335 }
20336 mod federated {
20337 include!("scope_federated_tests.rs");
20338 }
20339 mod authorize {
20340 include!("route_authorize_tests.rs");
20341 }
20342 #[tokio::test]
20343 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
20344 let rig = rig().await;
20345 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
20346 "s".into(),
20347 1,
20348 "tool".into(),
20349 ))
20350 .unwrap();
20351 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
20352 let (direct, _rx) = wide_ctx(999);
20353 for (ctx, expected) in [
20354 (&rig.owner, FrameType::Response),
20355 (&direct, FrameType::Error),
20356 ] {
20357 let frame =
20358 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
20359 .unwrap();
20360 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
20361 assert_eq!(replies[0].header.ty, expected);
20362 if expected == FrameType::Response {
20363 let reply: subc_control::ScopeServedByReply =
20364 serde_json::from_slice(&replies[0].body).unwrap();
20365 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
20366 } else {
20367 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
20368 }
20369 }
20370 let reply = call(
20371 &rig.handler,
20372 &rig.owner,
20373 &ModuleControlRequestFromModule::ScopeDescribe {
20374 owner: Principal::Reserved {
20375 module_id: OWNER.into(),
20376 },
20377 scope_ref: "s".into(),
20378 },
20379 )
20380 .await;
20381 assert!(matches!(
20382 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
20383 ModuleControlResponseToModule::ScopeDescribe { .. }
20384 ));
20385 }
20386
20387 use subc_protocol::scope::{
20388 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
20389 ScopeStatus,
20390 };
20391
20392 use super::*;
20393
20394 const OWNER: &str = "prefrontal-core";
20395
20396 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
20397 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
20398 }
20399
20400 async fn call(
20401 handler: &ControlHandler,
20402 ctx: &RouteCtx,
20403 request: &ModuleControlRequestFromModule,
20404 ) -> Frame {
20405 let body = serde_json::to_vec(request).unwrap();
20406 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
20407 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
20408 assert_eq!(replies.len(), 1, "{replies:?}");
20409 replies.pop().unwrap()
20410 }
20411
20412 async fn sync(
20413 handler: &ControlHandler,
20414 ctx: &RouteCtx,
20415 generation: u64,
20416 scopes: Vec<ScopeRecord>,
20417 ) -> Result<ModuleControlResponseToModule, String> {
20418 let reply = call(
20419 handler,
20420 ctx,
20421 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
20422 )
20423 .await;
20424 match reply.header.ty {
20425 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
20426 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
20427 }
20428 }
20429
20430 async fn describe(
20431 handler: &ControlHandler,
20432 ctx: &RouteCtx,
20433 owner: &str,
20434 scope_ref: &str,
20435 ) -> ModuleControlResponseToModule {
20436 let reply = call(
20437 handler,
20438 ctx,
20439 &ModuleControlRequestFromModule::ScopeDescribe {
20440 owner: Principal::Reserved {
20441 module_id: owner.to_string(),
20442 },
20443 scope_ref: scope_ref.to_string(),
20444 },
20445 )
20446 .await;
20447 assert_eq!(
20448 reply.header.ty,
20449 FrameType::Response,
20450 "{:?}",
20451 parse_error(&reply)
20452 );
20453 serde_json::from_slice(&reply.body).unwrap()
20454 }
20455
20456 #[tokio::test]
20457 async fn scope_describe_never_echoes_bind_proof_or_hash() {
20458 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
20459 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
20460 let mut rig = rig().await;
20461 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
20462 .await;
20463 let _bound = rig
20464 .bound(
20465 Some(AFT),
20466 PLEXUS,
20467 Some(
20468 rig.selector("s", Some(1))
20469 .with_bind_proof(Some(PROOF.into())),
20470 ),
20471 )
20472 .await;
20473 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
20474 let output = serde_json::to_string(&reply).unwrap();
20475 assert!(output.contains("\"status\":\"live\""));
20476 assert!(!output.contains(PROOF));
20477 assert!(!output.contains(HASH));
20478 }
20479
20480 async fn module(
20482 handler: &ControlHandler,
20483 connection: u64,
20484 module_id: &str,
20485 nonce: Option<&str>,
20486 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20487 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
20488 hello_via_sink(
20489 handler,
20490 &ctx,
20491 &mut rx,
20492 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
20493 )
20494 .await;
20495 (ctx, rx)
20496 }
20497
20498 #[tokio::test]
20501 async fn a_client_connection_cannot_sync_or_describe() {
20502 let handler = ControlHandler::new(Arc::new(Registry::default()));
20503 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
20504 for request in [
20505 ModuleControlRequestFromModule::ScopeSync {
20506 generation: 1,
20507 scopes: vec![head("s", 1)],
20508 },
20509 ModuleControlRequestFromModule::ScopeDescribe {
20510 owner: Principal::Direct,
20511 scope_ref: "s".to_string(),
20512 },
20513 ] {
20514 let reply = call(&handler, &ctx, &request).await;
20515 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
20516 }
20517 assert!(
20518 !handler
20519 .scopes
20520 .read()
20521 .unwrap()
20522 .describe(
20523 &Principal::Reserved {
20524 module_id: OWNER.to_string()
20525 },
20526 "s"
20527 )
20528 .owner_synced
20529 );
20530 }
20531
20532 #[tokio::test]
20535 async fn a_module_without_a_supervised_launch_cannot_sync() {
20536 let handler = ControlHandler::new(Arc::new(Registry::default()));
20537 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
20538 assert_eq!(
20539 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
20540 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20541 );
20542 }
20543
20544 #[tokio::test]
20545 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
20546 let supervisor = SupervisorHandle::new();
20547 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20548 let handler = ControlHandler::new(Arc::new(Registry::default()))
20549 .with_supervisor(supervisor.clone());
20550 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20551 sync(&handler, &incumbent, 1, vec![head("s", 1)])
20552 .await
20553 .expect("the current launch syncs");
20554
20555 supervisor.open_swap(OWNER, "n2".to_string());
20558 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
20559 assert_eq!(
20560 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20561 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20562 );
20563 sync(&handler, &incumbent, 2, vec![head("s", 1)])
20564 .await
20565 .expect("the serving owner syncs during the swap");
20566
20567 supervisor.close_swap(OWNER);
20570 assert_eq!(
20571 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20572 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20573 );
20574 sync(&handler, &incumbent, 3, vec![head("s", 1)])
20575 .await
20576 .expect("the serving owner syncs after the rollback");
20577 handler.cleanup_connection(candidate.connection_id).unwrap();
20578
20579 supervisor.open_swap(OWNER, "n3".to_string());
20584 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
20585 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
20586 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
20587 .await
20588 .expect("the promoted launch takes authority");
20589 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20590 panic!("unexpected reply {reply:?}");
20591 };
20592 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
20593 assert_eq!(
20594 sync(&handler, &incumbent, 4, Vec::new()).await,
20595 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20596 );
20597 }
20598
20599 #[tokio::test]
20602 async fn closing_the_authority_connection_frees_sync_authority() {
20603 let supervisor = SupervisorHandle::new();
20604 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20605 let handler = ControlHandler::new(Arc::new(Registry::default()))
20606 .with_supervisor(supervisor.clone());
20607 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20608 sync(&handler, &first, 10, vec![head("s", 1)])
20609 .await
20610 .unwrap();
20611 handler.cleanup_connection(first.connection_id).unwrap();
20612
20613 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
20614 sync(&handler, &second, 1, vec![head("s", 1)])
20615 .await
20616 .expect("the next connection takes the released authority");
20617 }
20618
20619 #[tokio::test]
20620 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
20621 let supervisor = SupervisorHandle::new();
20622 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20623 let handler =
20624 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
20625 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
20626 sync(&handler, &first, 10, vec![head("s", 1)])
20627 .await
20628 .unwrap();
20629 handler
20630 .handle_control_frame(
20631 &first,
20632 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
20633 )
20634 .await
20635 .unwrap();
20636 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
20637 sync(&handler, &second, 1, vec![head("s", 1)])
20638 .await
20639 .expect("GOODBYE releases authority even if the old socket remains open");
20640 }
20641
20642 #[tokio::test]
20643 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
20644 let registry = Arc::new(Registry::default());
20645 let supervisor_handle = SupervisorHandle::new();
20646 let supervisor =
20647 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20648 .with_handle(supervisor_handle.clone())
20649 .with_daemon_incarnation("incarnation-7".to_string());
20650 supervisor
20653 .supervise_configured(
20654 ModuleSpec {
20655 script: None,
20656 carry_containment: false,
20657 module_id: OWNER.to_string(),
20658 program: PathBuf::from("/nonexistent/prefrontal-core"),
20659 args: Vec::new(),
20660 env: Vec::new(),
20661 reserved: false,
20662 reserved_prefixes: Vec::new(),
20663 protocol: ModuleProtocol::Subc,
20664 overlap: Default::default(),
20665 },
20666 false,
20667 )
20668 .unwrap();
20669 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
20670 let handler =
20671 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
20672 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
20673
20674 let ModuleControlResponseToModule::ScopeDescribe {
20676 status,
20677 daemon_incarnation,
20678 owner_synced,
20679 owner_configured,
20680 scope,
20681 ..
20682 } = describe(&handler, &reader, OWNER, "s").await
20683 else {
20684 panic!("not a describe reply");
20685 };
20686 assert_eq!(status, ScopeStatus::NotLive);
20687 assert_eq!(daemon_incarnation, "incarnation-7");
20688 assert!(!owner_synced);
20689 assert!(owner_configured);
20690 assert!(scope.is_none());
20691
20692 let ModuleControlResponseToModule::ScopeDescribe {
20695 status,
20696 owner_configured,
20697 ..
20698 } = describe(&handler, &reader, "ghost", "s").await
20699 else {
20700 panic!("not a describe reply");
20701 };
20702 assert_eq!(status, ScopeStatus::NotLive);
20703 assert!(!owner_configured);
20704
20705 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
20707 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
20708 let ModuleControlResponseToModule::ScopeDescribe {
20709 status,
20710 scope_epoch,
20711 owner_synced,
20712 scope,
20713 ..
20714 } = describe(&handler, &reader, OWNER, "s").await
20715 else {
20716 panic!("not a describe reply");
20717 };
20718 assert_eq!(status, ScopeStatus::Live);
20719 assert_eq!(scope_epoch, Some(4));
20720 assert!(owner_synced);
20721 let stamp = scope.expect("a live scope carries its stamp");
20722 assert!(
20723 stamp.owner_authorized,
20724 "prefrontal-core is the default authority"
20725 );
20726 assert_eq!(stamp.kind, ScopeKind::Head);
20727 }
20728
20729 #[tokio::test]
20730 async fn scope_authority_owners_decides_owner_authorized() {
20731 let supervisor = SupervisorHandle::new();
20732 supervisor.set_spawn_nonce("broca", "b1".to_string());
20733 let handler = ControlHandler::new(Arc::new(Registry::default()))
20734 .with_supervisor(supervisor)
20735 .with_scope_authority_owners(vec!["broca".to_string()]);
20736 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
20737 let mut gated = head("s", 1);
20738 gated.attributes.agent_id = Some("agent".to_string());
20739 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
20740 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
20741 describe(&handler, &broca, "broca", "s").await
20742 else {
20743 panic!("not a describe reply");
20744 };
20745 assert!(scope.unwrap().owner_authorized);
20746 }
20747
20748 #[tokio::test]
20752 async fn scope_ops_and_the_scopes_capability_are_advertised() {
20753 let handler = ControlHandler::new(Arc::new(Registry::default()));
20754 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
20755 let ack = hello_via_sink(
20756 &handler,
20757 &ctx,
20758 &mut rx,
20759 hello_frame("m", PROTOCOL_VERSION, 1),
20760 )
20761 .await;
20762 let ack = parse_ack(&ack);
20763 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
20764 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
20765 }
20766 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
20767
20768 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
20769 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
20770 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
20771 let reply = handler
20772 .handle_control_frame(&client, frame)
20773 .await
20774 .unwrap()
20775 .pop()
20776 .unwrap();
20777 let ClientControlResponse::ServerDescribe { capabilities, .. } =
20778 serde_json::from_slice(&reply.body).unwrap()
20779 else {
20780 panic!("not a server.describe reply");
20781 };
20782 assert!(
20783 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
20784 "{capabilities:?}"
20785 );
20786 }
20787
20788 const PLEXUS: &str = "plexus";
20791 const OTHER: &str = "other";
20792 const AFT: &str = "aft";
20793 const BROCA: &str = "broca";
20794 const MAGIC: &str = "magic-context";
20795
20796 fn nonce(module_id: &str) -> String {
20797 format!("nonce-{module_id}")
20798 }
20799
20800 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20801 let (tx, rx) = mpsc::channel(64);
20802 (
20803 RouteCtx {
20804 connection_id: ConnectionId::new(connection),
20805 egress: FrameSink::new(tx),
20806 },
20807 rx,
20808 )
20809 }
20810
20811 struct Rig {
20815 handler: ControlHandler,
20816 forwarding: Arc<ForwardingTable>,
20817 owner: RouteCtx,
20818 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20819 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
20820 generation: u64,
20821 next_connection: u64,
20822 _supervisor: Supervisor,
20823 }
20824
20825 async fn rig() -> Rig {
20826 rig_with_flow_support(true).await
20827 }
20828
20829 async fn rig_with_flow_support(flow_support: bool) -> Rig {
20830 rig_with_scope_capabilities(flow_support, false).await
20831 }
20832
20833 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
20834 let mut provides = Vec::new();
20835 if flow_support {
20836 provides.push("flow-scopes/v1");
20837 }
20838 if run_support {
20839 provides.push("agent-run-scopes/v1");
20840 }
20841 rig_with_target_capabilities(&provides).await
20842 }
20843
20844 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
20845 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
20846 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20847 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20850 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
20851 Frame::build(
20852 FrameType::Hello,
20853 control_flags(),
20854 0,
20855 0,
20856 connection,
20857 serde_json::to_vec(&body).unwrap(),
20858 )
20859 .unwrap()
20860 }
20861
20862 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
20863 let registry = Arc::new(Registry::default());
20864 let forwarding = Arc::new(ForwardingTable::default());
20865 let supervisor_handle = SupervisorHandle::new();
20866 let supervisor =
20867 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20868 .with_handle(supervisor_handle.clone());
20869 supervisor
20870 .supervise_configured(
20871 ModuleSpec {
20872 script: None,
20873 carry_containment: false,
20874 module_id: OWNER.to_string(),
20875 program: PathBuf::from("/nonexistent/prefrontal-core"),
20876 args: Vec::new(),
20877 env: Vec::new(),
20878 reserved: false,
20879 reserved_prefixes: Vec::new(),
20880 protocol: ModuleProtocol::Subc,
20881 overlap: Default::default(),
20882 },
20883 false,
20884 )
20885 .unwrap();
20886 for module_id in [OWNER, AFT, BROCA, MAGIC] {
20887 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
20888 }
20889 let handler =
20890 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
20891 .with_supervisor(supervisor_handle);
20892 let (owner, mut owner_rx) = wide_ctx(1);
20893 hello_via_sink(
20894 &handler,
20895 &owner,
20896 &mut owner_rx,
20897 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
20898 )
20899 .await;
20900 let mut modules = BTreeMap::new();
20901 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
20902 let (ctx, mut rx) = wide_ctx(connection);
20903 let hello = hello_with_capabilities(module_id, connection, provides);
20904 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
20905 modules.insert(module_id.to_string(), (ctx, rx));
20906 }
20907 Rig {
20908 handler,
20909 forwarding,
20910 owner,
20911 _owner_rx: owner_rx,
20912 modules,
20913 generation: 0,
20914 next_connection: 100,
20915 _supervisor: supervisor,
20916 }
20917 }
20918
20919 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
20920 ScopeCarrier::new(Principal::Reserved {
20921 module_id: module_id.to_string(),
20922 })
20923 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
20924 }
20925
20926 fn session(scope_epoch: u64) -> ScopeRecord {
20929 let mut record = head("s", scope_epoch);
20930 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
20931 record.attributes.agent_id = Some("agent-1".to_string());
20932 record.attributes.delegates = true;
20933 record
20934 }
20935
20936 impl Rig {
20937 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
20938 self.generation += 1;
20939 sync(&self.handler, &self.owner, self.generation, scopes)
20940 .await
20941 .expect("the owner's sync is accepted");
20942 }
20943
20944 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
20945 ScopeSelector::new(
20946 Principal::Reserved {
20947 module_id: OWNER.to_string(),
20948 },
20949 scope_ref.to_string(),
20950 scope_epoch,
20951 )
20952 }
20953
20954 fn open_frame(
20955 &mut self,
20956 opener: Option<&str>,
20957 target: &str,
20958 scope: Option<ScopeSelector>,
20959 ) -> (
20960 RouteCtx,
20961 mpsc::Receiver<crate::router::OutboundFrame>,
20962 Frame,
20963 ) {
20964 self.next_connection += 1;
20965 let (ctx, rx) = wide_ctx(self.next_connection);
20966 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
20967 busy_refusals: None,
20968 target: RouteTarget::ToolProvider {
20969 module_id: target.to_string(),
20970 },
20971 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
20972 consumer_identity: opener.map(|module_id| ConsumerIdentity {
20973 module_id: module_id.to_string(),
20974 launch_nonce: nonce(module_id),
20975 }),
20976 consumer_capabilities: None,
20977 role_versions: None,
20978 admission_facts: None,
20979 scope,
20980 })
20981 .unwrap();
20982 let frame = Frame::build(
20983 FrameType::Request,
20984 control_flags(),
20985 0,
20986 0,
20987 self.next_connection,
20988 body,
20989 )
20990 .unwrap();
20991 (ctx, rx, frame)
20992 }
20993
20994 async fn refused(
20996 &mut self,
20997 opener: Option<&str>,
20998 target: &str,
20999 scope: Option<ScopeSelector>,
21000 ) -> String {
21001 self.refusal_body(opener, target, scope).await["code"]
21002 .as_str()
21003 .unwrap()
21004 .to_string()
21005 }
21006
21007 async fn refusal_body(
21008 &mut self,
21009 opener: Option<&str>,
21010 target: &str,
21011 scope: Option<ScopeSelector>,
21012 ) -> Value {
21013 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
21014 let replies = tokio::time::timeout(
21015 Duration::from_secs(2),
21016 self.handler.handle_control_frame(&ctx, frame),
21017 )
21018 .await
21019 .expect("the open must be refused before waiting for a bind ack")
21020 .unwrap();
21021 assert_eq!(replies.len(), 1, "{replies:?}");
21022 assert_eq!(replies[0].header.ty, FrameType::Error);
21023 let (_, module_rx) = self.modules.get_mut(target).unwrap();
21024 assert!(
21025 module_rx.try_recv().is_err(),
21026 "a refused open relays nothing"
21027 );
21028 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
21029 parse_error(&replies[0])
21030 }
21031
21032 async fn relayed(
21034 &mut self,
21035 opener: Option<&str>,
21036 target: &str,
21037 scope: Option<ScopeSelector>,
21038 ) -> Relayed {
21039 self.relayed_with_legacy_root(opener, target, scope, None)
21040 .await
21041 }
21042
21043 async fn relayed_with_legacy_root(
21044 &mut self,
21045 opener: Option<&str>,
21046 target: &str,
21047 scope: Option<ScopeSelector>,
21048 legacy_root: Option<&std::path::Path>,
21049 ) -> Relayed {
21050 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
21051 if let Some(root) = legacy_root {
21052 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
21053 body["identity"]["project_root"] = serde_json::json!(root);
21054 frame.body = serde_json::to_vec(&body).unwrap();
21055 }
21056 let handler = self.handler.clone();
21057 let task_ctx = ctx.clone();
21058 let task = tokio::spawn(async move {
21059 handler
21060 .handle_control_frame(&task_ctx, frame)
21061 .await
21062 .unwrap()
21063 });
21064 let (_, module_rx) = self.modules.get_mut(target).unwrap();
21065 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
21066 .await
21067 .expect("the target receives the relayed route.bind")
21068 .unwrap()
21069 .frame;
21070 Relayed {
21071 target: target.to_string(),
21072 client: ctx,
21073 client_rx: rx,
21074 task,
21075 bind,
21076 }
21077 }
21078
21079 async fn ack(&self, relayed: &Relayed) {
21080 let (module, _) = &self.modules[&relayed.target];
21081 self.handler
21082 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
21083 .await
21084 .unwrap();
21085 }
21086
21087 async fn bound(
21089 &mut self,
21090 opener: Option<&str>,
21091 target: &str,
21092 scope: Option<ScopeSelector>,
21093 ) -> Bound {
21094 let relayed = self.relayed(opener, target, scope).await;
21095 self.commit_relayed(relayed).await
21096 }
21097
21098 async fn bound_with_legacy_root(
21099 &mut self,
21100 opener: Option<&str>,
21101 target: &str,
21102 scope: Option<ScopeSelector>,
21103 root: &std::path::Path,
21104 ) -> Bound {
21105 let relayed = self
21106 .relayed_with_legacy_root(opener, target, scope, Some(root))
21107 .await;
21108 self.commit_relayed(relayed).await
21109 }
21110
21111 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
21112 self.ack(&relayed).await;
21113 let Relayed {
21114 target,
21115 client,
21116 mut client_rx,
21117 task,
21118 bind,
21119 } = relayed;
21120 assert!(
21121 task.await.unwrap().is_empty(),
21122 "the open is answered by commit"
21123 );
21124 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
21125 Bound {
21126 target,
21127 client,
21128 client_rx,
21129 channel,
21130 epoch,
21131 bind,
21132 }
21133 }
21134
21135 fn live(&self, route: &Bound) -> bool {
21136 matches!(
21137 self.forwarding
21138 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21139 .unwrap(),
21140 DataRoute::Client(DataRouteState::Bound(_))
21141 )
21142 }
21143 }
21144
21145 struct Relayed {
21146 target: String,
21147 client: RouteCtx,
21148 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21149 task: tokio::task::JoinHandle<Vec<Frame>>,
21150 bind: Frame,
21151 }
21152
21153 struct Bound {
21154 target: String,
21155 client: RouteCtx,
21156 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21157 channel: u16,
21158 epoch: u32,
21159 bind: Frame,
21160 }
21161
21162 impl Bound {
21163 fn closed_reason(&mut self) -> RouteCloseReason {
21166 let mut reason = None;
21167 let mut goodbye = false;
21168 while let Ok(outbound) = self.client_rx.try_recv() {
21169 let frame = outbound.frame;
21170 match frame.header.ty {
21171 FrameType::Goodbye => {
21172 assert_eq!(
21173 (frame.header.channel, frame.header.epoch),
21174 (self.channel, self.epoch)
21175 );
21176 goodbye = true;
21177 }
21178 FrameType::Push => {
21179 let ClientControlPush::RouteClosed {
21180 reason: r,
21181 module_id,
21182 ..
21183 } = serde_json::from_slice(&frame.body).unwrap()
21184 else {
21185 panic!("unexpected push");
21186 };
21187 assert_eq!(module_id, self.target);
21188 reason = Some(r);
21189 }
21190 other => panic!("unexpected frame {other:?}"),
21191 }
21192 }
21193 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
21194 reason.expect("the client is told why the route closed")
21195 }
21196
21197 fn untouched(&mut self) -> bool {
21198 self.client_rx.try_recv().is_err()
21199 }
21200
21201 fn stamp(&self) -> Option<ScopeStamp> {
21202 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
21203 ModuleControlRequest::RouteBind { scope, .. } => scope,
21204 other => panic!("expected a route.bind, got {other:?}"),
21205 }
21206 }
21207 }
21208
21209 #[tokio::test]
21210 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
21211 ) {
21212 let mut rig = rig().await;
21213 let mut record = session(1);
21214 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
21215 record.child_owners = vec![Principal::Reserved {
21216 module_id: MAGIC.to_string(),
21217 }];
21218 rig.sync(vec![record]).await;
21219 let scope = || Some(rig_selector("s", Some(1)));
21220
21221 rig.bound(Some(OWNER), PLEXUS, scope()).await;
21224 rig.bound(Some(AFT), OTHER, scope()).await;
21225 rig.bound(Some(BROCA), PLEXUS, scope()).await;
21226
21227 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
21231 assert_eq!(
21232 rig.refused(opener, target, scope()).await,
21233 error_codes::SCOPE_NOT_CARRIER,
21234 "{opener:?} -> {target}"
21235 );
21236 }
21237 }
21238
21239 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
21240 ScopeSelector::new(
21241 Principal::Reserved {
21242 module_id: OWNER.to_string(),
21243 },
21244 scope_ref.to_string(),
21245 scope_epoch,
21246 )
21247 }
21248
21249 #[tokio::test]
21250 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
21251 let mut rig = rig().await;
21252 rig.sync(vec![session(1)]).await;
21253 for opener in [OWNER, AFT] {
21254 assert_eq!(
21255 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
21256 .await,
21257 error_codes::SCOPE_EPOCH_REQUIRED,
21258 "{opener}"
21259 );
21260 }
21261 }
21262
21263 #[tokio::test]
21264 async fn admission_separates_not_synced_not_live_and_ended() {
21265 let mut rig = rig().await;
21266 let code = rig
21268 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21269 .await;
21270 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
21271 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
21272
21273 let ghost = ScopeSelector::new(
21275 Principal::Reserved {
21276 module_id: "ghost".to_string(),
21277 },
21278 "s".to_string(),
21279 Some(1),
21280 );
21281 assert_eq!(
21282 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
21283 error_codes::SCOPE_NOT_LIVE
21284 );
21285
21286 rig.sync(vec![session(2)]).await;
21287 assert_eq!(
21288 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
21289 .await,
21290 error_codes::SCOPE_NOT_LIVE
21291 );
21292 for epoch in [1, 3] {
21293 assert_eq!(
21294 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
21295 .await,
21296 error_codes::SCOPE_ENDED,
21297 "epoch {epoch}"
21298 );
21299 }
21300 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
21302 .await;
21303 }
21304
21305 #[tokio::test]
21306 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
21307 let mut rig = rig().await;
21308 rig.sync(vec![session(1)]).await;
21309 let route = rig
21310 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21311 .await;
21312 let stamp = route.stamp().expect("a scoped bind carries the stamp");
21313 assert_eq!(stamp.scope_ref, "s");
21314 assert_eq!(stamp.scope_epoch, 1);
21315 assert_eq!(stamp.kind, ScopeKind::Head);
21316 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
21317 assert!(stamp.attributes.delegates);
21318 assert!(stamp.owner_authorized);
21319 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21320 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
21321
21322 let (broca, mut broca_rx) = wide_ctx(50);
21325 hello_via_sink(
21326 &rig.handler,
21327 &broca,
21328 &mut broca_rx,
21329 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
21330 )
21331 .await;
21332 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
21333 .await
21334 .unwrap();
21335 let own = ScopeSelector::new(
21336 Principal::Reserved {
21337 module_id: BROCA.to_string(),
21338 },
21339 "b".to_string(),
21340 Some(1),
21341 );
21342 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
21343 assert!(!route.stamp().unwrap().owner_authorized);
21344 }
21345
21346 #[tokio::test]
21347 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
21348 let mut rig = rig().await;
21349 let mut record = head("s", 1);
21350 record.carriers = vec![carrier(AFT, None)];
21351 let flow_id = "Flow:run-7/step_2!~";
21352 record.attributes.flow_id = Some(flow_id.to_string());
21353 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
21354 .await
21355 .unwrap();
21356 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
21357 panic!("not a sync reply");
21358 };
21359 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
21360 let route = rig
21361 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21362 .await;
21363 assert!(rig.live(&route), "the stamped bind committed");
21364 let stamp = route.stamp().expect("a flow scope carries a stamp");
21365 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
21366 assert_eq!(stamp.attributes.agent_id, None);
21367 assert!(!stamp.attributes.delegates);
21368 assert!(stamp.owner_authorized);
21369 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21370 assert_eq!(unscoped.stamp(), None);
21371 }
21372
21373 #[tokio::test]
21374 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
21375 let mut rig = rig_with_flow_support(false).await;
21376 let mut record = session(1);
21377 record.attributes.flow_id = Some("flow:7".to_string());
21378 rig.sync(vec![record]).await;
21379 for opener in [OWNER, AFT] {
21380 let body = rig
21381 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21382 .await;
21383 assert_eq!(body["code"], "target_flow_unsupported");
21384 let message = body["message"].as_str().unwrap();
21385 for required in [PLEXUS, "flow-scopes/v1"] {
21386 assert!(message.contains(required), "{message}");
21387 }
21388 }
21389 }
21390
21391 #[tokio::test]
21392 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
21393 let mut rig = rig_with_flow_support(true).await;
21394 let mut record = session(1);
21395 record.attributes.flow_id = Some("flow:7".to_string());
21396 rig.sync(vec![record]).await;
21397 for opener in [OWNER, AFT] {
21398 let route = rig
21399 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21400 .await;
21401 assert!(rig.live(&route));
21402 assert_eq!(
21403 route.stamp().unwrap().attributes.flow_id.as_deref(),
21404 Some("flow:7")
21405 );
21406 }
21407 }
21408
21409 #[tokio::test]
21410 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
21411 use std::future::Future;
21412
21413 let mut rig = rig_with_flow_support(true).await;
21414 let mut record = session(1);
21415 record.attributes.flow_id = Some("flow:7".to_string());
21416 rig.sync(vec![record]).await;
21417 let (client, mut client_rx, frame) =
21418 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
21419 for _ in 0..64 {
21422 client.egress.try_send(route_bind_ack(1)).unwrap();
21423 }
21424 let handler = rig.handler.clone();
21425 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21426 std::future::poll_fn(|cx| {
21427 assert!(open.as_mut().poll(cx).is_pending());
21428 std::task::Poll::Ready(())
21429 })
21430 .await;
21431 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21432
21433 let old_connection = rig.modules[PLEXUS].0.connection_id;
21434 rig.handler.cleanup_connection(old_connection).unwrap();
21435 let (replacement, mut replacement_rx) = wide_ctx(200);
21436 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
21437 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
21438 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
21439 let hello = Frame::build(
21440 FrameType::Hello,
21441 control_flags(),
21442 0,
21443 0,
21444 200,
21445 serde_json::to_vec(&body).unwrap(),
21446 )
21447 .unwrap();
21448 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
21449
21450 client_rx.try_recv().unwrap();
21451 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21452 .await
21453 .expect("the replacement is refused without waiting for a bind ack")
21454 .unwrap();
21455 assert_eq!(replies.len(), 1);
21456 let body = parse_error(&replies[0]);
21457 assert_eq!(body["code"], "target_flow_unsupported");
21458 for required in [PLEXUS, "flow-scopes/v1"] {
21459 assert!(body["message"].as_str().unwrap().contains(required));
21460 }
21461 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
21462 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21463 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21464 assert!(rig
21465 .handler
21466 .registry
21467 .get_module_by_connection(replacement.connection_id)
21468 .unwrap()
21469 .is_some());
21470 }
21471
21472 #[tokio::test]
21473 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
21474 use std::future::Future;
21475 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
21476 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21477 const NEXT_HASH: &str =
21478 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21479 let mut rig = rig().await;
21480 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
21481 record.carriers.push(ScopeCarrier::new(Principal::Direct));
21482 rig.sync(vec![record.clone()]).await;
21483 let (client, mut rx, frame) = rig.open_frame(
21484 None,
21485 PLEXUS,
21486 Some(
21487 rig.selector("s", Some(1))
21488 .with_bind_proof(Some(PROOF.into())),
21489 ),
21490 );
21491 for _ in 0..64 {
21494 client.egress.try_send(route_bind_ack(1)).unwrap();
21495 }
21496 let handler = rig.handler.clone();
21497 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21498 std::future::poll_fn(|cx| {
21499 assert!(open.as_mut().poll(cx).is_pending());
21500 std::task::Poll::Ready(())
21501 })
21502 .await;
21503 record.bind_proof_sha256 = Some(NEXT_HASH.into());
21504 rig.sync(vec![record]).await;
21505 rx.try_recv().unwrap();
21506 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21507 .await
21508 .unwrap()
21509 .unwrap();
21510 assert_eq!(
21511 parse_error(&replies[0])["code"],
21512 error_codes::SCOPE_BIND_PROOF_MISMATCH
21513 );
21514 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
21515 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21516 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21517 }
21518
21519 #[tokio::test]
21520 async fn scope_bind_proof_reserved_openers_ignore_proof() {
21521 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21522 let mut rig = rig().await;
21523 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21524 .await;
21525 for opener in [OWNER, AFT] {
21526 for proof in [None, Some("malformed+proof=")] {
21527 let route = rig
21528 .bound(
21529 Some(opener),
21530 PLEXUS,
21531 Some(
21532 rig.selector("s", Some(1))
21533 .with_bind_proof(proof.map(str::to_string)),
21534 ),
21535 )
21536 .await;
21537 assert!(rig.live(&route));
21538 }
21539 }
21540 }
21541
21542 #[tokio::test]
21543 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
21544 use std::future::Future;
21545 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21546 const NEXT_HASH: &str =
21547 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21548 let mut rig = rig().await;
21549 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21550 .await;
21551 let (client, mut rx, frame) =
21552 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
21553 for _ in 0..64 {
21554 client.egress.try_send(route_bind_ack(1)).unwrap();
21555 }
21556 let handler = rig.handler.clone();
21557 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21558 std::future::poll_fn(|cx| {
21559 assert!(open.as_mut().poll(cx).is_pending());
21560 std::task::Poll::Ready(())
21561 })
21562 .await;
21563 rig.sync(vec![
21564 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
21565 ])
21566 .await;
21567 rx.try_recv().unwrap();
21568 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21569 let bind = std::future::poll_fn(|cx| {
21570 assert!(open.as_mut().poll(cx).is_pending());
21571 module_rx.poll_recv(cx)
21572 })
21573 .await
21574 .unwrap()
21575 .frame;
21576 handler
21577 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
21578 .await
21579 .unwrap();
21580 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21581 .await
21582 .unwrap()
21583 .unwrap();
21584 assert_eq!(replies.len(), 1);
21585 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
21588 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21589 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
21590 let route = rig
21591 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
21592 .await;
21593 assert!(rig.live(&route));
21594 }
21595
21596 #[tokio::test]
21597 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
21598 {
21599 let mut rig = rig_with_flow_support(false).await;
21600 rig.sync(vec![session(1)]).await;
21601 let route = rig
21602 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21603 .await;
21604 assert!(rig.live(&route));
21605 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
21606 let mut record = session(1);
21607 record.attributes.flow_id = Some("flow:7".to_string());
21608 rig.sync(vec![record]).await;
21609 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
21610 assert!(rig.live(&unscoped));
21611 assert_eq!(unscoped.stamp(), None);
21612 }
21613
21614 #[tokio::test]
21615 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
21616 let mut rig = rig().await;
21617 let mut record = session(1);
21618 record.attributes.flow_id = Some("flow:7".to_string());
21619 rig.sync(vec![record.clone()]).await;
21620 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21621 let mut owner_route = rig
21622 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21623 .await;
21624 let mut carrier_route = rig
21625 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
21626 .await;
21627 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21628 rig.sync(vec![record.clone()]).await;
21629 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21630 assert!(rig.live(&owner_route) && owner_route.untouched());
21631 assert!(rig.live(&carrier_route) && carrier_route.untouched());
21632
21633 record.attributes.flow_id = Some("flow:8".to_string());
21634 rig.sync(vec![record]).await;
21635 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
21636 let before = before.unwrap();
21637 assert_eq!(after.scope_epoch, before.scope_epoch);
21638 assert!(after.version > before.version);
21639 for route in [&mut owner_route, &mut carrier_route] {
21640 assert!(!rig.live(route));
21641 assert_eq!(
21642 route.closed_reason(),
21643 RouteCloseReason::ScopeDelegationChanged
21644 );
21645 }
21646 assert!(rig.live(&unscoped) && unscoped.untouched());
21647 for target in [PLEXUS, OTHER] {
21650 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
21651 let goodbye = module_rx
21652 .try_recv()
21653 .expect("the provider sees the drain")
21654 .frame;
21655 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21656 assert!(module_rx.try_recv().is_err());
21657 }
21658 let rebound = rig
21659 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21660 .await;
21661 assert_eq!(
21662 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
21663 Some("flow:8")
21664 );
21665 }
21666
21667 #[tokio::test]
21672 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
21673 let mut rig = rig().await;
21674 rig.sync(vec![session(1)]).await;
21675 let mut cotenant = rig
21676 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21677 .await;
21678
21679 let mut changed = session(1);
21680 changed.child_owners.push(Principal::Reserved {
21681 module_id: MAGIC.to_string(),
21682 });
21683 let mut ended = None;
21684 for (code, next) in [
21685 (error_codes::SCOPE_CHANGED, vec![changed]),
21686 (error_codes::SCOPE_ENDED, Vec::new()),
21687 ] {
21688 let relayed = rig
21689 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21690 .await;
21691 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
21692 ended = Some(next.is_empty());
21693 rig.sync(next).await;
21694 rig.ack(&relayed).await;
21695 let replies = relayed.task.await.unwrap();
21696 assert_eq!(replies.len(), 1, "{replies:?}");
21697 assert_eq!(parse_error(&replies[0])["code"], code);
21698 assert_eq!(
21699 subc_protocol::error_codes::is_retryable_route_open(code),
21700 code == error_codes::SCOPE_CHANGED
21701 );
21702 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21703 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21705 let mut goodbyes = Vec::new();
21708 while let Ok(outbound) = plexus_rx.try_recv() {
21709 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
21710 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
21711 }
21712 assert!(
21713 goodbyes.contains(&(bind_channel, bind_epoch)),
21714 "{goodbyes:?}"
21715 );
21716 assert!(rig
21717 .handler
21718 .registry
21719 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
21720 .unwrap()
21721 .is_some());
21722 }
21723 assert_eq!(ended, Some(true));
21724 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
21727 }
21728
21729 #[tokio::test]
21732 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
21733 struct Case {
21734 name: &'static str,
21735 change: fn(&mut ScopeRecord),
21736 closed: [Option<RouteCloseReason>; 4],
21739 }
21740 use RouteCloseReason::*;
21741 let cases = [
21742 Case {
21743 name: "a carrier entry removed",
21744 change: |r| {
21745 r.carriers.retain(|c| {
21746 c.principal
21747 != Principal::Reserved {
21748 module_id: AFT.to_string(),
21749 }
21750 })
21751 },
21752 closed: [None, Some(ScopeCarrierRemoved), None, None],
21753 },
21754 Case {
21755 name: "a target removed from a carrier",
21756 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
21757 closed: [None, None, None, Some(ScopeCarrierRemoved)],
21758 },
21759 Case {
21760 name: "a bare carrier narrowed to targets",
21761 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
21762 closed: [None, Some(ScopeCarrierRemoved), None, None],
21763 },
21764 Case {
21765 name: "delegates turned off",
21766 change: |r| r.attributes.delegates = false,
21767 closed: [Some(ScopeDelegationChanged); 4],
21768 },
21769 Case {
21770 name: "agent_id changed",
21771 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
21772 closed: [Some(ScopeDelegationChanged); 4],
21773 },
21774 Case {
21775 name: "a carrier added, child owners changed, the record re-sent",
21776 change: |r| {
21777 r.carriers.push(carrier(MAGIC, None));
21778 r.child_owners.push(Principal::Reserved {
21779 module_id: MAGIC.to_string(),
21780 });
21781 },
21782 closed: [None; 4],
21783 },
21784 Case {
21785 name: "a target added",
21786 change: |r| {
21787 r.carriers[1]
21788 .targets
21789 .as_mut()
21790 .unwrap()
21791 .push("third".to_string())
21792 },
21793 closed: [None; 4],
21794 },
21795 Case {
21796 name: "delegates turned on",
21797 change: |r| r.attributes.delegates = true,
21798 closed: [None; 4],
21799 },
21800 ];
21801 for case in cases {
21802 let mut rig = rig().await;
21803 rig.sync(vec![session(1)]).await;
21804 let scope = || Some(rig_selector("s", Some(1)));
21805 let mut routes = [
21806 rig.bound(Some(OWNER), PLEXUS, scope()).await,
21807 rig.bound(Some(AFT), PLEXUS, scope()).await,
21808 rig.bound(Some(BROCA), PLEXUS, scope()).await,
21809 rig.bound(Some(BROCA), OTHER, scope()).await,
21810 ];
21811 let mut record = session(1);
21812 (case.change)(&mut record);
21813 rig.sync(vec![record]).await;
21814 for (index, expected) in case.closed.iter().enumerate() {
21815 let route = &mut routes[index];
21816 match expected {
21817 Some(reason) => {
21818 assert!(!rig.live(route), "{}: route {index} still live", case.name);
21819 assert_eq!(
21820 route.closed_reason(),
21821 *reason,
21822 "{}: route {index}",
21823 case.name
21824 );
21825 }
21826 None => {
21827 assert!(rig.live(route), "{}: route {index} closed", case.name);
21828 assert!(
21829 route.untouched(),
21830 "{}: route {index} was told something",
21831 case.name
21832 );
21833 }
21834 }
21835 }
21836 }
21837 }
21838
21839 #[tokio::test]
21840 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
21841 for next in [Vec::new(), vec![session(2)]] {
21843 let mut rig = rig().await;
21844 rig.sync(vec![session(1)]).await;
21845 let mut route = rig
21846 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21847 .await;
21848 rig.sync(next).await;
21849 assert!(!rig.live(&route));
21850 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
21851 }
21852
21853 let mut rig = rig().await;
21856 let mut child = session(1);
21857 child.scope_ref = "child".to_string();
21858 child.kind = ScopeKind::Worker;
21859 child.parent = Some(ScopeParent::new(
21860 Principal::Reserved {
21861 module_id: OWNER.to_string(),
21862 },
21863 "s".to_string(),
21864 1,
21865 ));
21866 rig.sync(vec![session(1), child.clone()]).await;
21867 let mut child_route = rig
21868 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
21869 .await;
21870 assert_eq!(
21871 child_route.stamp().unwrap().parent_state,
21872 Some(ParentState::Linked)
21873 );
21874 rig.sync(vec![child]).await;
21875 assert!(!rig.live(&child_route));
21876 assert_eq!(
21877 child_route.closed_reason(),
21878 RouteCloseReason::ScopeParentEnded
21879 );
21880 }
21881
21882 #[tokio::test]
21883 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
21884 ) {
21885 let mut rig = rig().await;
21886 rig.sync(vec![session(1)]).await;
21887 let mut route = rig
21888 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21889 .await;
21890 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21891 rig.sync(vec![session(1)]).await;
21892 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21893 assert!(rig.live(&route) && route.untouched());
21894
21895 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
21899 .forwarding
21900 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21901 .unwrap()
21902 else {
21903 panic!("the route is bound");
21904 };
21905 binding.flow.acquire_tagged(9, false).await.unwrap();
21906 let mut widened = session(1);
21907 widened.carriers.push(carrier(MAGIC, None));
21908 rig.sync(vec![widened]).await;
21909 assert!(rig.live(&route) && route.untouched());
21910 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21911 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
21912 assert_eq!(binding.flow.in_flight(), 1);
21915 binding
21916 .flow
21917 .acquire_tagged(10, false)
21918 .await
21919 .expect("the flow is still open");
21920 }
21921
21922 #[tokio::test]
21925 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
21926 let mut rig = rig().await;
21927 rig.sync(vec![session(1)]).await;
21928 let mut on_incumbent = rig
21929 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21930 .await;
21931
21932 let (candidate, _candidate_rx) = wide_ctx(9);
21935 let registration = rig
21936 .handler
21937 .registry
21938 .register_candidate_with_control_ops(
21939 manifest(PLEXUS, PROTOCOL_VERSION),
21940 PROTOCOL_VERSION,
21941 candidate.connection_id,
21942 module_baseline_control_ops(),
21943 )
21944 .unwrap();
21945 rig.forwarding
21946 .register_candidate_module_connection(
21947 candidate.connection_id,
21948 PLEXUS.to_string(),
21949 PROTOCOL_VERSION,
21950 manifest_concurrency(®istration.manifest),
21951 candidate.egress.clone(),
21952 )
21953 .unwrap();
21954 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
21955 rig.handler
21956 .registry
21957 .promote_candidate(PLEXUS)
21958 .unwrap()
21959 .unwrap();
21960 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
21961
21962 rig.sync(Vec::new()).await;
21963 assert!(!rig.live(&on_incumbent));
21964 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
21965 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21966 let goodbye = incumbent_rx
21967 .try_recv()
21968 .expect("the superseded endpoint is told")
21969 .frame;
21970 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21971 }
21972 }
21973}
21974
21975#[cfg(test)]
21976mod concurrency_default_exposure_tests {
21977 use super::*;
21978
21979 fn hello_body(role_json: &str) -> Vec<u8> {
21980 format!(
21981 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
21982 )
21983 .into_bytes()
21984 }
21985
21986 fn manifest_from(body: &[u8]) -> ModuleManifest {
21987 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
21988 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
21989 .expect("manifest parses")
21990 }
21991
21992 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
21993
21994 #[test]
21995 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
21996 let body = hello_body(&format!(
21997 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
21998 ));
21999 let manifest = manifest_from(&body);
22000 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
22003 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
22004 }
22005
22006 #[test]
22007 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
22008 let body = hello_body(&format!(
22009 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
22010 ));
22011 let manifest = manifest_from(&body);
22012 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
22013 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
22014 }
22015
22016 #[test]
22017 fn non_management_roles_are_never_reported() {
22018 let body = hello_body(
22019 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
22020 );
22021 let manifest = manifest_from(&body);
22022 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
22023 }
22024}