1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26 error_codes,
27 manifest::{
28 validate_hello_capability_grammar, validate_hello_confirm_operations,
29 validate_hello_display_name, validate_hello_event_declarations,
30 validate_hello_flow_operations, validate_hello_operation_deferral,
31 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32 validate_hello_self_signal_declarations, validate_role_declarations_value,
33 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34 ProviderRole,
35 },
36 peer::{
37 ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY,
38 PEER_BUILD_CAPABILITY,
39 },
40 provider_declarations::ProviderDeclarations,
41 scope::{
42 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
43 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
44 SCOPE_SYNC_OP,
45 },
46 session::{
47 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
48 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
49 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
50 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
51 },
52 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
53 Priority, RouteTarget, PROTOCOL_VERSION,
54};
55use tokio::time::{timeout_at, Instant};
56use tracing::{debug, info, instrument::WithSubscriber, warn};
57
58use crate::{
59 caller_identity::CallerIdentity,
60 capability_requirements::{
61 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
62 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
63 RequirementStatus, RuntimeModule,
64 },
65 child_roster::DaemonShutdownFlag,
66 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
67 forwarding::{
68 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
69 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
70 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
71 },
72 observability::{
73 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
74 },
75 privileged_prompt::{prompt_text, PrivilegedOperation},
76 provenance::{
77 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
78 },
79 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
80 router::{daemon_error_flags, RouteCtx, RouterError},
81 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
82 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
83 stderr_tail::{CaptureState, TailEntry},
84 supervise::{
85 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
86 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
87 },
88 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
89};
90
91pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
97
98const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
99const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
100const CAP_PING_PONG: &str = "ping_pong_v1";
101const CAP_SESSION_ATTACH: &str = "session_attach_v1";
102const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
103
104const SUBC_CONTROL_OPS: &[&str] = &[
105 "claim.mint",
106 "claim.present",
107 ops::DAEMON_SHUTDOWN_INTENT,
108 ops::ACTIVITY_RECORD,
109 ops::ACTIVITY_QUERY,
110 ops::ACTIVITY_SUBSCRIBE,
111 ops::SERVER_DESCRIBE,
112 ops::CATALOG_LIST,
113 ops::CATALOG_WATCH,
114 ops::ROUTE_OPEN,
115 ops::ROUTE_POLL,
116 ops::ROUTE_CLOSING,
117 ops::ROUTE_CLOSED,
118 ops::SUPERVISOR_LIST,
119 ops::SUPERVISOR_RESTART,
120 ops::SUPERVISOR_SWAP,
121 ops::SUPERVISOR_RELOAD,
122 ops::SUPERVISOR_RESCAN,
123 ops::SUPERVISOR_RELEASE_RESERVED,
124 ops::SUPERVISOR_SET_ENABLED,
125 ops::SUPERVISOR_HEALTH_PROBE,
126 ops::SUPERVISOR_HEALTH,
127 ops::SUPERVISOR_STDERR_TAIL,
128 ops::SUPERVISOR_TERMINALS,
129 ops::SUPERVISOR_ROUTES,
130 ops::SUPERVISOR_PROVENANCE,
131 ops::SUPERVISOR_SPAWN_SNAPSHOT,
132 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
133];
134
135const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[
142 ops::SUPERVISOR_RESCAN_CHECKED,
143 "config.status",
144 "config.plan",
145 "config.apply",
146 "config.stamp",
147 "config.unstamp",
148 "config.history",
149 "config.rollback",
150];
151
152mod caller_attestation;
153mod channel_operations;
154#[cfg(all(test, unix, feature = "test-support"))]
155#[path = "control/channel_operations_tests.rs"]
156mod channel_operations_tests;
157pub(crate) mod claims;
158#[cfg(test)]
159#[path = "control/config_advertisement_tests.rs"]
160mod config_advertisement_tests;
161mod config_gate;
162use caller_attestation::{caller_for_target, scope_acts_for_another};
163#[cfg(all(test, unix, feature = "test-support"))]
164#[path = "control/caller_attestation_tests.rs"]
165mod caller_attestation_tests;
166mod scope_expiry;
167mod scope_watch;
168
169mod agent_commands;
170pub(crate) mod catalog;
171mod rescan_checked;
172use rescan_checked::{RescanMode, RescanPlan};
173
174const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
175 "claim.mint",
176 "claim.present",
177 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
178 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
179 "supervisor.live_roots",
180 "supervisor.agent_commands",
181 SCOPE_SYNC_OP,
182 SCOPE_APPLY_OP,
183 SCOPE_DESCRIBE_OP,
184 ops::SCOPE_WATCH,
185 subc_protocol::scope::ROUTE_AUTHORIZE_OP,
186 "operator.confirm",
187];
188
189const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
193
194const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
195
196pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
204
205pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
220
221pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
239
240const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
241const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
242
243fn reload_verdict(
244 configured: &Path,
245 spawned_from: Option<&Path>,
246 image: subc_control::RunningImageAgreement,
247) -> PendingReloadVerdict {
248 let path = match spawned_from {
249 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
250 ReloadPathAgreement::Match
251 }
252 Some(spawned_from) => ReloadPathAgreement::Mismatch {
253 configured: configured.to_path_buf(),
254 spawned_from: spawned_from.to_path_buf(),
255 },
256 None => ReloadPathAgreement::Unavailable {
257 reason: if matches!(
258 image,
259 subc_control::RunningImageAgreement::Unavailable {
260 reason: subc_control::RunningImageUnavailableReason::NotRunning
261 }
262 ) {
263 ReloadPathUnavailableReason::NotRunning
264 } else {
265 ReloadPathUnavailableReason::SpawnedPathUnavailable
266 },
267 },
268 };
269 PendingReloadVerdict::new(path, image)
270}
271
272fn launch_spec_agreement(
273 configured: &crate::supervise::ModuleSpec,
274 running: bool,
275 spawned: Option<&crate::supervise::LaunchSpecDigest>,
276) -> subc_control::LaunchSpecAgreement {
277 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
278 if !running {
279 return Agreement::Unavailable {
280 reason: Reason::NotRunning,
281 };
282 }
283 match spawned {
284 None => Agreement::Unavailable {
285 reason: Reason::SpawnedSpecUnavailable,
286 },
287 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
288 Agreement::Match
289 }
290 Some(_) => Agreement::Mismatch,
291 }
292}
293
294fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
295 if configured == spawned_from {
296 return true;
297 }
298 #[cfg(windows)]
299 {
300 match (
306 std::fs::canonicalize(configured),
307 std::fs::canonicalize(spawned_from),
308 ) {
309 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
310 _ => false,
311 }
312 }
313 #[cfg(not(windows))]
314 {
315 false
316 }
317}
318
319#[derive(Clone)]
320struct DaemonProvenanceFacts {
321 build: DaemonBuildProvenance,
322 pid: Option<u32>,
323 started_at_ms: Option<u64>,
324 start_clock: Option<crate::clock::StartClock>,
325 executable_path: Option<PathBuf>,
326 executable_identity: Option<SpawnedFileIdentity>,
327 process_start_time: Option<u64>,
328 probe: ExecutableIdentityProbe,
329}
330
331impl Default for DaemonProvenanceFacts {
332 fn default() -> Self {
333 Self {
334 build: DaemonBuildProvenance {
335 build_git_sha: None,
336 build_lock_digest: None,
337 },
338 pid: None,
339 started_at_ms: None,
340 start_clock: None,
341 executable_path: None,
342 executable_identity: None,
343 process_start_time: None,
344 probe: ExecutableIdentityProbe::default(),
345 }
346 }
347}
348
349#[derive(Debug, Clone)]
350struct SupervisorRescanContext {
351 supervisor: Supervisor,
352 config_path: PathBuf,
353 configured_port: Option<u16>,
354 storage_config: Option<crate::daemon_config::StorageConfig>,
355 admission_facts_carrier_module_id: Option<String>,
356 admission_facts_targets: Option<Vec<String>>,
357 scope_authority_owners: Vec<String>,
358 federated_scope_owner: String,
359 privileged_operator_prompt: PrivilegedOperatorPrompt,
360 generation: Arc<std::sync::atomic::AtomicU64>,
366 module_entries: Arc<Mutex<rescan_checked::ModuleEntries>>,
367}
368
369const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
390 "reloading",
391 "supervisor_not_live",
392 "registration_not_active",
393 "no_forwarding_connection",
394 "relay_send_failed",
395];
396
397#[cfg(test)]
399struct ActivityRequestPause {
400 connection: ConnectionId,
401 started: tokio::sync::Notify,
402 cancelled: tokio::sync::Notify,
403 cleanup_started: std::sync::atomic::AtomicBool,
404 cancelled_after_cleanup: std::sync::atomic::AtomicBool,
405}
406
407#[cfg(test)]
408struct ActivityRequestCancellation(Arc<ActivityRequestPause>);
409
410#[cfg(test)]
411impl Drop for ActivityRequestCancellation {
412 fn drop(&mut self) {
413 self.0.cancelled_after_cleanup.store(
414 self.0
415 .cleanup_started
416 .load(std::sync::atomic::Ordering::Acquire),
417 std::sync::atomic::Ordering::Release,
418 );
419 self.0.cancelled.notify_one();
420 }
421}
422
423#[cfg(test)]
425type TestPause = (
426 tokio::sync::oneshot::Sender<()>,
427 std::sync::mpsc::Receiver<()>,
428);
429
430#[cfg(test)]
432type TestHook = Box<dyn FnOnce() + Send>;
433
434#[cfg(test)]
436type AsyncTestPause = (
437 tokio::sync::oneshot::Sender<()>,
438 tokio::sync::oneshot::Receiver<()>,
439);
440
441#[derive(Clone)]
443pub struct ControlHandler {
444 #[cfg(test)]
445 activity_request_pause: Arc<Mutex<Option<Arc<ActivityRequestPause>>>>,
446 #[cfg(test)]
450 cleanup_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
451 #[cfg(test)]
454 after_nonce_attestation: Arc<Mutex<Option<TestHook>>>,
455 #[cfg(test)]
458 hello_ingress_pause: Arc<Mutex<Option<AsyncTestPause>>>,
459 #[cfg(test)]
461 agent_commands_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
462 claims: Arc<claims::Claims>,
463 registry: Arc<Registry>,
464 config_runtime: Option<Arc<crate::config_gate::runtime::RuntimeOwner>>,
465 config_review_admission: Arc<dyn crate::config_gate::ReviewAdmission>,
466 config_review_admission_is_default: bool,
467 activity: Option<crate::activity::ActivityLedger>,
468 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
469 catalog_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
470 scope_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
471 forwarding: Arc<ForwardingTable>,
472 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
473 supervisor: SupervisorHandle,
474 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
475 agent_declarations: agent_commands::Queues,
476 list_admission: Arc<tokio::sync::Semaphore>,
477 subc_capabilities: Arc<[String]>,
478 route_bind_relay_timeout: Duration,
482 route_bind_relay_timeouts: BTreeMap<String, Duration>,
486 route_bind_breakers: RouteBindBreakers,
489 route_bind_concurrency: RouteBindConcurrency,
492 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
497 route_bind_breaker_threshold: u32,
499 route_bind_breaker_cooldown: Duration,
501 health_probe_timeout: Duration,
502 storage_config: Option<crate::daemon_config::StorageConfig>,
505 machine_id: Option<crate::machine_id::MachineId>,
509 admission_facts_carrier_module_id: Option<String>,
510 admission_facts_targets: Option<Vec<String>>,
511 scopes: Arc<RwLock<ScopeTable>>,
515 scope_deadline_added: Arc<tokio::sync::Notify>,
519 scope_authority_owners: Vec<String>,
522 federated_scope_owner: String,
523 privileged_operator_prompt: PrivilegedOperatorPrompt,
525 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
528 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
529 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
530 captured_peers: Arc<Mutex<HashMap<ConnectionId, subc_os::local_peer::CapturedPeer>>>,
531 config_desktop_stamps:
532 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
533 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
534 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
535 pub(crate) operator_apps:
536 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
537 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
542 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
548 #[cfg(test)]
550 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
551 #[cfg(test)]
552 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
553 #[cfg(test)]
554 recovery_exit_test_after_step: Option<fn(usize) -> Result<(), crate::config_gate::GateError>>,
555 #[cfg(test)]
556 recovery_exit_test_completed: Option<fn(u64)>,
557 #[cfg(test)]
558 apply_reconcile_pause: Arc<Mutex<Option<AsyncTestPause>>>,
559 #[cfg(test)]
560 recovery_exit_test_quiet_retry: bool,
561 #[cfg(test)]
562 recovery_exit_test_retry_pause: Option<fn(u64) -> Option<PathBuf>>,
563 #[cfg(test)]
564 config_delivery_test_wakes:
565 Option<tokio::sync::mpsc::UnboundedSender<(&'static str, tokio::time::Instant)>>,
566 #[cfg(all(test, unix))]
567 config_delivery_test_idle:
568 Option<tokio::sync::mpsc::UnboundedSender<tokio::sync::oneshot::Sender<()>>>,
569 #[allow(dead_code)]
573 #[cfg(windows)]
574 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
575 #[allow(dead_code)]
576 #[cfg(windows)]
577 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
578 #[allow(dead_code)]
582 #[cfg(windows)]
583 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
584 #[cfg(all(test, windows))]
587 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
588 #[cfg(test)]
589 caller_lookup: Option<TestCallerLookup>,
590 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
593 rescan: Option<SupervisorRescanContext>,
594 connected_clients: ConnectedClients,
595 counters: DaemonCounters,
596 local_endpoint_unavailable: Option<String>,
599 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
600 daemon_shutdown: DaemonShutdownFlag,
605 daemon_provenance: DaemonProvenanceFacts,
606 #[cfg(test)]
607 control_dispatch_delay: Option<Duration>,
608 #[cfg(test)]
609 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
610 #[cfg(test)]
614 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
615 #[cfg(test)]
621 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
622 #[cfg(test)]
623 hello_ack_to_publication_hold: Option<Arc<std::sync::Barrier>>,
624 #[cfg(test)]
625 hello_before_publication_hold: Option<Arc<std::sync::Barrier>>,
626 #[cfg(test)]
627 hello_after_launch_check_hold: Option<Arc<std::sync::Barrier>>,
628 #[cfg(test)]
629 hello_inside_publication_hold: Option<Arc<std::sync::Barrier>>,
630}
631
632impl fmt::Debug for ControlHandler {
633 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
634 f.debug_struct("ControlHandler")
635 .field("registry", &self.registry)
636 .field("forwarding", &self.forwarding)
637 .field("process_liveness", &self.process_liveness.is_some())
638 .field("supervisor", &self.supervisor)
639 .field("subc_capabilities", &self.subc_capabilities)
640 .finish()
641 }
642}
643
644struct RouteOpenRequest {
645 target: RouteTarget,
646 identity: BindIdentity,
647 consumer_identity: Option<ConsumerIdentity>,
648 consumer_capabilities: Option<Vec<String>>,
649 role_versions: Option<BTreeMap<String, String>>,
650 busy_refusals: Option<bool>,
651 admission_facts: Option<serde_json::Value>,
652 scope: Option<ScopeSelector>,
653 accepts_close_reasons: Option<Vec<String>>,
654}
655
656#[derive(Clone, Copy)]
657enum RouteOpenPhase {
658 Admission,
659 ProjectRoot,
660 EgressReservation,
661 BindSend,
662 BindAckWait,
663}
664
665struct RouteOpenTimings {
669 module_id: String,
670 principal: String,
671 admission: Duration,
672 project_root: Duration,
673 egress_reservation: Duration,
674 bind_send: Duration,
675 bind_ack_wait: Duration,
676 phase: Option<RouteOpenPhase>,
677 phase_started_at: Instant,
678}
679
680impl RouteOpenTimings {
681 fn new() -> Self {
682 Self {
683 module_id: String::new(),
684 principal: "unresolved".to_string(),
686 admission: Duration::ZERO,
687 project_root: Duration::ZERO,
688 egress_reservation: Duration::ZERO,
689 bind_send: Duration::ZERO,
690 bind_ack_wait: Duration::ZERO,
691 phase: Some(RouteOpenPhase::Admission),
692 phase_started_at: Instant::now(),
693 }
694 }
695
696 fn finish(&mut self) {
697 let elapsed = self.phase_started_at.elapsed();
698 match self.phase.take() {
699 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
700 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
701 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
702 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
703 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
704 None => {}
705 }
706 }
707
708 fn enter(&mut self, phase: RouteOpenPhase) {
709 self.finish();
710 self.phase = Some(phase);
711 self.phase_started_at = Instant::now();
712 }
713}
714
715struct RouteBindReservationGuard {
716 forwarding: Arc<ForwardingTable>,
717 endpoint: ModuleEndpointId,
718 relay_corr: u64,
719 armed: bool,
720}
721
722struct ModuleControlRpcGuard {
723 forwarding: Arc<ForwardingTable>,
724 endpoint: ModuleEndpointId,
725 corr: u64,
726 armed: bool,
727}
728
729impl ModuleControlRpcGuard {
730 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
731 Self {
732 forwarding,
733 endpoint,
734 corr,
735 armed: true,
736 }
737 }
738
739 fn disarm(&mut self) {
740 self.armed = false;
741 }
742}
743
744impl Drop for ModuleControlRpcGuard {
745 fn drop(&mut self) {
746 if self.armed {
747 let _ = self
748 .forwarding
749 .cancel_module_control_rpc(self.endpoint, self.corr);
750 }
751 }
752}
753
754impl RouteBindReservationGuard {
755 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
756 Self {
757 forwarding,
758 endpoint,
759 relay_corr,
760 armed: true,
761 }
762 }
763
764 fn release_and_disarm(&mut self) {
765 if !self.armed {
766 return;
767 }
768 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
769 self.endpoint,
770 self.relay_corr,
771 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
772 ) {
773 send_goodbye_target_best_effort(
774 &self.forwarding.counters(),
775 &target,
776 "canceled route.bind",
777 );
778 }
779 self.armed = false;
780 }
781
782 fn disarm(&mut self) {
783 self.armed = false;
784 }
785}
786
787impl Drop for RouteBindReservationGuard {
788 fn drop(&mut self) {
789 self.release_and_disarm();
790 }
791}
792
793#[derive(Debug, Clone, Default)]
823pub(crate) struct RouteBindBreakers {
824 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
825}
826
827#[derive(Debug, Clone, Default)]
828pub(crate) struct RouteBindConcurrency {
829 modules: Arc<Mutex<HashMap<String, usize>>>,
830}
831
832struct RouteBindConcurrencyGuard {
833 concurrency: RouteBindConcurrency,
834 module_id: String,
835}
836
837impl RouteBindConcurrency {
838 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
841 let mut modules = self
842 .modules
843 .lock()
844 .expect("route.bind concurrency mutex poisoned");
845 let in_flight = modules.entry(module_id.to_string()).or_default();
846 if *in_flight >= limit {
847 return Err(*in_flight);
848 }
849 *in_flight += 1;
850 Ok(RouteBindConcurrencyGuard {
851 concurrency: self.clone(),
852 module_id: module_id.to_string(),
853 })
854 }
855}
856
857impl Drop for RouteBindConcurrencyGuard {
858 fn drop(&mut self) {
859 let mut modules = self
860 .concurrency
861 .modules
862 .lock()
863 .expect("route.bind concurrency mutex poisoned");
864 let remove = {
865 let in_flight = modules
866 .get_mut(&self.module_id)
867 .expect("admitted route.bind has a concurrency entry");
868 *in_flight -= 1;
869 *in_flight == 0
870 };
871 if remove {
872 modules.remove(&self.module_id);
873 }
874 }
875}
876
877#[derive(Debug, Default)]
878struct ModuleBreakerState {
879 consecutive_timeouts: u32,
881 cooldown_until: Option<Instant>,
884 probe_in_flight: Option<Arc<()>>,
889}
890
891enum RouteBindAdmission<'a> {
893 Admitted {
894 guard: RouteBindBreakerGuard<'a>,
895 probe: bool,
898 },
899 Refused {
900 consecutive_timeouts: u32,
901 retry_in: Duration,
905 probe_in_flight: bool,
906 },
907}
908
909struct RouteBindBreakerGuard<'a> {
916 breakers: RouteBindBreakers,
917 module_id: &'a str,
918 probe_token: Option<Arc<()>>,
919 settled: bool,
920}
921
922impl RouteBindBreakerGuard<'_> {
923 fn record_accepted(&mut self) -> bool {
927 self.settled = true;
928 self.breakers.record_accepted(self.module_id)
929 }
930
931 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
934 self.settled = true;
935 self.breakers.record_timeout(
936 self.module_id,
937 self.probe_token.as_ref(),
938 threshold,
939 cooldown,
940 )
941 }
942
943 fn record_inconclusive(&mut self) {
952 self.settled = true;
953 self.breakers
954 .record_inconclusive(self.module_id, self.probe_token.as_ref());
955 }
956}
957
958impl Drop for RouteBindBreakerGuard<'_> {
959 fn drop(&mut self) {
960 if !self.settled {
961 self.breakers
962 .record_inconclusive(self.module_id, self.probe_token.as_ref());
963 }
964 }
965}
966
967struct BreakerOpened {
971 consecutive_timeouts: u32,
972 reopened_after_probe: bool,
975}
976
977impl RouteBindBreakers {
978 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
979 self.modules
980 .lock()
981 .expect("route.bind breaker mutex poisoned")
982 }
983
984 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
987 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
988 probe: probe_token.is_some(),
989 guard: RouteBindBreakerGuard {
990 breakers: self.clone(),
991 module_id,
992 probe_token,
993 settled: false,
994 },
995 };
996
997 let mut modules = self.lock();
998 let Some(state) = modules.get_mut(module_id) else {
999 return admitted(None);
1000 };
1001 let Some(cooldown_until) = state.cooldown_until else {
1002 return admitted(None);
1003 };
1004 if state.probe_in_flight.is_some() {
1005 return RouteBindAdmission::Refused {
1006 consecutive_timeouts: state.consecutive_timeouts,
1007 retry_in: Duration::ZERO,
1008 probe_in_flight: true,
1009 };
1010 }
1011 let now = Instant::now();
1012 if now < cooldown_until {
1013 return RouteBindAdmission::Refused {
1014 consecutive_timeouts: state.consecutive_timeouts,
1015 retry_in: cooldown_until - now,
1016 probe_in_flight: false,
1017 };
1018 }
1019 let token = Arc::new(());
1020 state.probe_in_flight = Some(Arc::clone(&token));
1021 admitted(Some(token))
1022 }
1023
1024 fn record_accepted(&self, module_id: &str) -> bool {
1025 self.lock()
1026 .remove(module_id)
1027 .is_some_and(|state| state.cooldown_until.is_some())
1028 }
1029
1030 fn record_timeout(
1031 &self,
1032 module_id: &str,
1033 probe_token: Option<&Arc<()>>,
1034 threshold: u32,
1035 cooldown: Duration,
1036 ) -> Option<BreakerOpened> {
1037 let mut modules = self.lock();
1038 let state = modules.entry(module_id.to_string()).or_default();
1039 let was_open = state.cooldown_until.is_some();
1040 let was_probe = Self::owns_probe(state, probe_token);
1041 if was_probe {
1042 state.probe_in_flight = None;
1043 }
1044 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
1045 if state.consecutive_timeouts < threshold {
1046 return None;
1047 }
1048 state.cooldown_until = Some(Instant::now() + cooldown);
1049 Some(BreakerOpened {
1050 consecutive_timeouts: state.consecutive_timeouts,
1051 reopened_after_probe: was_open && was_probe,
1052 })
1053 }
1054
1055 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
1056 state
1060 .probe_in_flight
1061 .as_ref()
1062 .zip(token)
1063 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
1064 }
1065
1066 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
1067 if let Some(state) = self.lock().get_mut(module_id) {
1068 if Self::owns_probe(state, probe_token) {
1069 state.probe_in_flight = None;
1070 }
1071 }
1072 }
1073
1074 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
1089 self.lock()
1090 .remove(module_id)
1091 .map(|state| state.consecutive_timeouts)
1092 .filter(|discarded| *discarded > 0)
1093 }
1094
1095 fn open_snapshot(&self) -> Option<serde_json::Value> {
1102 let now = Instant::now();
1103 let modules = self.lock();
1104 let open = modules
1105 .iter()
1106 .filter_map(|(module_id, state)| {
1107 let cooldown_until = state.cooldown_until?;
1108 Some((
1109 module_id.clone(),
1110 serde_json::json!({
1111 "consecutive_timeouts": state.consecutive_timeouts,
1112 "cooldown_remaining_ms":
1113 cooldown_until.saturating_duration_since(now).as_millis() as u64,
1114 "probe_in_flight": state.probe_in_flight.is_some(),
1115 }),
1116 ))
1117 })
1118 .collect::<serde_json::Map<String, serde_json::Value>>();
1119 (!open.is_empty()).then_some(serde_json::Value::Object(open))
1120 }
1121}
1122
1123impl ControlHandler {
1124 pub(crate) fn install_registration_service(
1125 &self,
1126 router: std::sync::Weak<crate::Router>,
1127 auth: crate::server::ServerAuth,
1128 ) {
1129 self.supervisor.install_registration_service(router, auth);
1130 }
1131 pub fn new(registry: Arc<Registry>) -> Self {
1132 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
1133 }
1134
1135 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
1136 let counters = forwarding.counters();
1137 let route_bind_breakers = forwarding.route_bind_breakers();
1141 let route_bind_concurrency = forwarding.route_bind_concurrency();
1142 let route_outages = forwarding.route_outages();
1143 Self {
1144 registry,
1145 #[cfg(test)]
1146 activity_request_pause: Arc::new(Mutex::new(None)),
1147 #[cfg(test)]
1148 cleanup_pause: Arc::new(Mutex::new(None)),
1149 #[cfg(test)]
1150 after_nonce_attestation: Arc::new(Mutex::new(None)),
1151 #[cfg(test)]
1152 hello_ingress_pause: Arc::new(Mutex::new(None)),
1153 #[cfg(test)]
1154 agent_commands_pause: Arc::new(Mutex::new(None)),
1155 claims: Arc::new(claims::Claims::default()),
1156 config_runtime: None,
1157 config_review_admission: Arc::new(crate::config_gate::MissingReviewAdmission),
1158 config_review_admission_is_default: true,
1159 activity: None,
1160 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1161 catalog_streams: Arc::new(Mutex::new(HashMap::new())),
1162 scope_streams: Arc::new(Mutex::new(HashMap::new())),
1163 forwarding,
1164 process_liveness: None,
1165 supervisor: SupervisorHandle::new(),
1166 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1167 env!("CARGO_PKG_VERSION").to_owned(),
1168 )),
1169 agent_declarations: agent_commands::Queues::default(),
1170 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1171 subc_capabilities: Arc::from([
1172 CAP_MANIFEST_REGISTRATION.to_string(),
1173 CAP_CHANNEL_LIFECYCLE.to_string(),
1174 CAP_PING_PONG.to_string(),
1175 CAP_SESSION_ATTACH.to_string(),
1176 CAP_ADMISSION_FACTS_RELAY.to_string(),
1177 CAP_SCOPES_V1.to_string(),
1178 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1179 subc_protocol::peer::CALLER_ATTESTATION_CAPABILITY.to_string(),
1180 PEER_BUILD_CAPABILITY.to_string(),
1181 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1182 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1183 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1184 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1185 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1186 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1187 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1188 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1189 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1190 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1191 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1192 subc_protocol::ROUTE_CLOSE_REASONS_CAPABILITY.to_string(),
1193 ]),
1194 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1195 route_bind_relay_timeouts: BTreeMap::new(),
1196 route_bind_breakers,
1197 route_bind_concurrency,
1198 route_outages,
1199 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1200 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1201 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1202 storage_config: None,
1203 machine_id: None,
1204 admission_facts_carrier_module_id: None,
1205 admission_facts_targets: None,
1206 scopes: Arc::new(RwLock::new(ScopeTable::new(
1207 crate::daemon_config::default_scope_authority_owners(),
1208 ))),
1209 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1210 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1211 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1212 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1213 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1214 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1215 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1216 captured_peers: Arc::new(Mutex::new(HashMap::new())),
1217 config_desktop_stamps: Arc::new(Mutex::new(HashMap::new())),
1218 local_connections: Arc::new(Mutex::new(HashSet::new())),
1219 operator_app_policy: Default::default(),
1220 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1221 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1222 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1223 #[cfg(test)]
1224 app_open_test_platform: None,
1225 #[cfg(test)]
1226 app_open_test_launch: None,
1227 #[cfg(test)]
1228 recovery_exit_test_after_step: None,
1229 #[cfg(test)]
1230 recovery_exit_test_completed: None,
1231 #[cfg(test)]
1232 apply_reconcile_pause: Arc::new(Mutex::new(None)),
1233 #[cfg(test)]
1234 recovery_exit_test_quiet_retry: false,
1235 #[cfg(test)]
1236 recovery_exit_test_retry_pause: None,
1237 #[cfg(test)]
1238 config_delivery_test_wakes: None,
1239 #[cfg(all(test, unix))]
1240 config_delivery_test_idle: None,
1241 #[cfg(windows)]
1242 windows_verify: None,
1243 #[cfg(windows)]
1244 windows_flash: None,
1245 #[cfg(windows)]
1246 windows_track: None,
1247 #[cfg(all(test, windows))]
1248 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1249 #[cfg(test)]
1250 caller_lookup: None,
1251 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1252 rescan: None,
1253 connected_clients: ConnectedClients::new(),
1254 local_endpoint_unavailable: None,
1255 counters,
1256 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1257 daemon_shutdown: DaemonShutdownFlag::default(),
1258 daemon_provenance: DaemonProvenanceFacts::default(),
1259 #[cfg(test)]
1260 control_dispatch_delay: None,
1261 #[cfg(test)]
1262 provenance_probe_override: None,
1263 #[cfg(test)]
1264 route_bind_relay_sent_hold: None,
1265 #[cfg(test)]
1266 hello_registry_to_forwarding_hold: None,
1267 #[cfg(test)]
1268 hello_ack_to_publication_hold: None,
1269 #[cfg(test)]
1270 hello_before_publication_hold: None,
1271 #[cfg(test)]
1272 hello_after_launch_check_hold: None,
1273 #[cfg(test)]
1274 hello_inside_publication_hold: None,
1275 }
1276 }
1277
1278 pub fn with_storage_config(
1281 mut self,
1282 storage_config: Option<crate::daemon_config::StorageConfig>,
1283 ) -> Self {
1284 self.storage_config = storage_config;
1285 self
1286 }
1287
1288 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1291 self.activity = Some(crate::activity::ActivityLedger::open(
1292 run_dir.join("activity.jsonl"),
1293 incarnation,
1294 ));
1295 self.supervisor.set_activity(self.activity.clone());
1296 self
1297 }
1298
1299 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1302 self.machine_id = machine_id;
1303 self
1304 }
1305
1306 pub fn with_admission_facts_config(
1310 mut self,
1311 carrier_module_id: Option<String>,
1312 targets: Option<Vec<String>>,
1313 ) -> Self {
1314 self.admission_facts_carrier_module_id = carrier_module_id;
1315 self.admission_facts_targets = targets;
1316 self
1317 }
1318
1319 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1323 self.scopes = Arc::new(RwLock::new(
1324 ScopeTable::new(owners.iter().cloned())
1325 .with_federated_owner(self.federated_scope_owner.clone()),
1326 ));
1327 self.scope_authority_owners = owners;
1328 self
1329 }
1330
1331 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1333 self.scopes = Arc::new(RwLock::new(
1334 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1335 .with_federated_owner(owner.clone()),
1336 ));
1337 self.federated_scope_owner = owner;
1338 self
1339 }
1340
1341 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1343 self.privileged_operator_prompt = mode;
1344 self
1345 }
1346
1347 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1350 self.route_bind_relay_timeout = timeout;
1351 self
1352 }
1353
1354 pub fn with_route_bind_relay_timeouts(
1360 mut self,
1361 timeouts: impl IntoIterator<Item = (String, Duration)>,
1362 ) -> Self {
1363 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1364 self
1365 }
1366
1367 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1373 self.route_bind_relay_timeouts
1374 .get(module_id)
1375 .copied()
1376 .unwrap_or(self.route_bind_relay_timeout)
1377 }
1378
1379 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1387 self.route_bind_breaker_threshold = threshold.max(1);
1388 self.route_bind_breaker_cooldown = cooldown;
1389 self
1390 }
1391
1392 #[cfg(test)]
1393 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1394 self.health_probe_timeout = timeout;
1395 self
1396 }
1397
1398 #[cfg(test)]
1399 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1400 self.control_dispatch_delay = Some(delay);
1401 self
1402 }
1403
1404 #[cfg(test)]
1408 pub(crate) fn with_route_bind_relay_sent_hold(
1409 mut self,
1410 hold: Arc<tokio::sync::Barrier>,
1411 ) -> Self {
1412 self.route_bind_relay_sent_hold = Some(hold);
1413 self
1414 }
1415
1416 #[cfg(test)]
1421 pub(crate) fn with_hello_registry_to_forwarding_hold(
1422 mut self,
1423 hold: Arc<std::sync::Barrier>,
1424 ) -> Self {
1425 self.hello_registry_to_forwarding_hold = Some(hold);
1426 self
1427 }
1428
1429 #[cfg(test)]
1430 fn hold_between_registry_and_forwarding(&self) {
1431 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1432 hold.wait();
1433 hold.wait();
1434 }
1435 }
1436
1437 #[cfg(test)]
1440 fn hold_between_ack_and_publication(&self) {
1441 if let Some(hold) = &self.hello_ack_to_publication_hold {
1442 hold.wait();
1443 hold.wait();
1444 }
1445 }
1446
1447 #[cfg(all(test, unix))]
1448 pub(crate) fn with_hello_before_publication_hold(
1449 mut self,
1450 hold: Arc<std::sync::Barrier>,
1451 ) -> Self {
1452 self.hello_before_publication_hold = Some(hold);
1453 self
1454 }
1455
1456 #[cfg(all(test, unix))]
1457 pub(crate) fn with_hello_after_launch_check_hold(
1458 mut self,
1459 hold: Arc<std::sync::Barrier>,
1460 ) -> Self {
1461 self.hello_after_launch_check_hold = Some(hold);
1462 self
1463 }
1464
1465 #[cfg(all(test, unix))]
1466 pub(crate) fn with_hello_inside_publication_hold(
1467 mut self,
1468 hold: Arc<std::sync::Barrier>,
1469 ) -> Self {
1470 self.hello_inside_publication_hold = Some(hold);
1471 self
1472 }
1473
1474 pub fn with_process_liveness(
1475 mut self,
1476 process_liveness: Arc<dyn ModuleProcessLiveness>,
1477 ) -> Self {
1478 self.process_liveness = Some(process_liveness);
1479 self
1480 }
1481
1482 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1483 self.supervisor = supervisor;
1484 if let Some(owner) = &self.config_runtime {
1485 self.supervisor.install_binding_retirement(owner);
1486 }
1487 self.supervisor.set_activity(self.activity.clone());
1488 self
1489 }
1490
1491 pub fn with_daemon_provenance(
1492 mut self,
1493 pid: u32,
1494 started_at_ms: u64,
1495 executable_path: Option<PathBuf>,
1496 build_git_sha: Option<String>,
1497 build_lock_digest: Option<String>,
1498 ) -> Self {
1499 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1500 let process_start_time = process_start_time(pid);
1501 self.daemon_provenance = DaemonProvenanceFacts {
1502 build: DaemonBuildProvenance {
1503 build_git_sha,
1504 build_lock_digest,
1505 },
1506 pid: Some(pid),
1507 started_at_ms: Some(started_at_ms),
1508 start_clock: None,
1509 executable_path,
1510 executable_identity,
1511 process_start_time,
1512 probe: ExecutableIdentityProbe::default(),
1513 };
1514 self
1515 }
1516
1517 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1518 self.daemon_provenance.start_clock = Some(clock);
1519 self
1520 }
1521
1522 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1524 self.scopes
1525 .write()
1526 .unwrap_or_else(|poisoned| poisoned.into_inner())
1527 .set_wall_clock(Arc::new(clock));
1528 self
1529 }
1530
1531 #[cfg(test)]
1532 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1533 self.provenance_probe_override = Some(result);
1534 self
1535 }
1536
1537 pub fn with_capability_config(
1541 self,
1542 modules: impl IntoIterator<Item = (String, bool)>,
1543 reserved_capabilities: BTreeMap<String, String>,
1544 ) -> Self {
1545 self.capability_evaluator
1546 .configure(modules, reserved_capabilities);
1547 self
1548 }
1549
1550 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1554 self.daemon_shutdown = flag;
1555 self
1556 }
1557
1558 pub fn with_supervisor_rescan(
1559 mut self,
1560 supervisor: Supervisor,
1561 config_path: impl Into<PathBuf>,
1562 configured_port: Option<u16>,
1563 ) -> Self {
1564 self.rescan = Some(SupervisorRescanContext {
1565 supervisor,
1566 config_path: config_path.into(),
1567 configured_port,
1568 storage_config: self.storage_config.clone(),
1569 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1570 admission_facts_targets: self.admission_facts_targets.clone(),
1571 scope_authority_owners: self.scope_authority_owners.clone(),
1572 federated_scope_owner: self.federated_scope_owner.clone(),
1573 privileged_operator_prompt: self.privileged_operator_prompt,
1574 generation: Arc::default(),
1575 module_entries: Arc::default(),
1576 });
1577 self
1578 }
1579
1580 pub(crate) fn with_supervisor_rescan_entries(
1581 self,
1582 modules: &[crate::daemon_config::ConfiguredModule],
1583 ) -> Self {
1584 if let Some(context) = &self.rescan {
1585 let mut entries = context
1586 .module_entries
1587 .lock()
1588 .unwrap_or_else(|e| e.into_inner());
1589 entries.seed(modules);
1590 }
1591 self
1592 }
1593
1594 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1595 self.connected_clients = connected_clients;
1596 self
1597 }
1598
1599 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1602 self.local_endpoint_unavailable = reason;
1603 self
1604 }
1605
1606 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1607 Arc::clone(&self.forwarding)
1608 }
1609
1610 pub(crate) fn counters(&self) -> DaemonCounters {
1611 self.counters.clone()
1612 }
1613
1614 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1617 let handler = Arc::downgrade(&self);
1618 let evaluator = self.capability_evaluator.clone();
1619 tokio::spawn(async move {
1620 loop {
1621 evaluator.wait_for_change_or_deadline().await;
1622 let Some(handler) = handler.upgrade() else {
1623 break;
1624 };
1625 handler.refresh_capability_requirements();
1626 }
1627 });
1628 }
1629
1630 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1636 let context = scope_expiry::ScopeExpiryContext::new(&self);
1637 let deadline_added = self.scope_deadline_added.clone();
1638 tokio::spawn(async move {
1639 loop {
1640 if !context.has_deadlines() {
1641 deadline_added.notified().await;
1642 continue;
1643 }
1644 tokio::time::sleep(Duration::from_secs(1)).await;
1645 if let Err(error) = context.sweep() {
1646 warn!(%error, "scope expiry sweep failed");
1647 }
1648 }
1649 });
1650 }
1651
1652 #[cfg(test)]
1653 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1654 scope_expiry::ScopeExpiryContext::new(self).sweep()
1655 }
1656
1657 fn publish_scope_changes(
1658 &self,
1659 changes: &[crate::scopes::ScopeTagChange],
1660 expired: &[crate::scopes::ScopeExpired],
1661 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1662 scope_expiry::ScopeExpiryContext::new(self).publish_scope_changes(changes, expired)
1663 }
1664
1665 fn runtime_capability_snapshot(
1666 &self,
1667 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1668 let runtime = self
1669 .supervisor
1670 .list()
1671 .into_iter()
1672 .map(|module| {
1673 let status = module.status().map_err(|err| {
1674 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1675 })?;
1676 Ok(RuntimeModule {
1677 module_id: status.module_id,
1678 state: status.state,
1679 enabled: status.enabled,
1680 })
1681 })
1682 .collect::<Result<Vec<_>, RouterError>>()?;
1683 let registrations = self.registry.list_registered_modules().map_err(|err| {
1689 RouterError::backend(
1690 0,
1691 0,
1692 format!("failed to list capability registrations: {err}"),
1693 )
1694 })?;
1695 let registrations = registrations
1696 .into_iter()
1697 .map(|registration| RegisteredModule {
1698 module_id: registration.manifest.module_id,
1699 module_version: registration.manifest.module_version,
1700 capabilities: registration.manifest.capabilities,
1701 })
1702 .collect();
1703 Ok((runtime, registrations))
1704 }
1705
1706 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1713 let cached_registration = RegisteredModule {
1714 module_id: registration.manifest.module_id.clone(),
1715 module_version: registration.manifest.module_version.clone(),
1716 capabilities: registration.manifest.capabilities.clone(),
1717 };
1718 let change = self.capability_evaluator.record_hello(&cached_registration);
1719 log_manifest_change(&cached_registration.module_id, &change);
1720 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1721 self.enforce_capability_denies();
1722 }
1723 self.refresh_capability_requirements();
1724 }
1725
1726 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1730 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1731 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1732 self.supervisor.set_swap_promotion_observer(observer);
1733 }
1734
1735 pub fn refresh_capability_requirements(&self) {
1736 match self.runtime_capability_snapshot() {
1737 Ok((runtime, registrations)) => {
1738 log_requirement_events(
1739 self.capability_evaluator
1740 .evaluate_now(&runtime, ®istrations),
1741 &self.daemon_shutdown,
1742 );
1743 }
1744 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1745 }
1746 }
1747
1748 fn enforce_capability_denies(&self) {
1752 let registrations = match self.registry.list_registered_modules() {
1753 Ok(snapshot) => snapshot,
1754 Err(err) => {
1755 warn!(error = %err, "failed to read registrations for capability deny census");
1756 return;
1757 }
1758 };
1759 let manifests = registrations
1760 .into_iter()
1761 .map(|registration| {
1762 (
1763 registration.manifest.module_id.clone(),
1764 registration.manifest,
1765 )
1766 })
1767 .collect::<BTreeMap<_, _>>();
1768 let census = match self.forwarding.route_census(None) {
1769 Ok(census) => census,
1770 Err(err) => {
1771 warn!(error = %err, "failed to read route census for capability deny enforcement");
1772 return;
1773 }
1774 };
1775
1776 for (target_module_id, routes) in census {
1777 let Some(target_manifest) = manifests.get(&target_module_id) else {
1778 continue;
1779 };
1780 let mut closed_routes = Vec::new();
1781 let mut goodbyes = Vec::new();
1782 for route in routes {
1783 let Principal::Reserved {
1784 module_id: opening_module_id,
1785 } = &route.principal
1786 else {
1787 continue;
1788 };
1789 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1790 continue;
1791 };
1792 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1793 continue;
1794 };
1795
1796 match self.forwarding.release_client_route(
1797 route.goodbye_target.connection_id,
1798 route.goodbye_target.channel,
1799 route.goodbye_target.epoch,
1800 ) {
1801 Ok(RouteRelease::Removed(module_goodbye)) => {
1802 warn!(
1803 opening_module_id,
1804 target_module_id,
1805 capability,
1806 "force-closing route because an attested capability deny edge now matches"
1807 );
1808 goodbyes.push(module_goodbye);
1809 goodbyes.push(route.goodbye_target.clone());
1810 closed_routes.push(route);
1811 }
1812 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1813 Err(err) => warn!(
1814 opening_module_id,
1815 target_module_id,
1816 capability,
1817 error = %err,
1818 "failed to force-close capability-denied route"
1819 ),
1820 }
1821 }
1822
1823 if closed_routes.is_empty() {
1824 continue;
1825 }
1826 send_route_control_pushes(
1827 &self.forwarding,
1828 closed_routes,
1829 ClientControlPush::RouteClosed {
1830 module_id: target_module_id,
1831 channels: Vec::new(),
1832 reason: RouteCloseReason::CapabilityDenied,
1833 drained: false,
1834 abandoned: 0,
1835 excluded_subscriptions: 0,
1836 terminal: Some(false),
1837 },
1838 );
1839 self.emit_route_goodbyes(goodbyes);
1842 }
1843 }
1844
1845 fn not_ready_reason(
1851 &self,
1852 registration: &crate::registry::ModuleRegistration,
1853 ) -> Option<NotReadyReason> {
1854 if !registration.ready {
1855 return Some(NotReadyReason {
1856 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1857 capability: None,
1858 });
1859 }
1860 self.first_unprovided_required_capability(registration)
1861 .map(|capability| NotReadyReason {
1862 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1863 capability: Some(capability),
1864 })
1865 }
1866
1867 fn first_unprovided_required_capability(
1885 &self,
1886 registration: &crate::registry::ModuleRegistration,
1887 ) -> Option<String> {
1888 let required = registration
1889 .manifest
1890 .capabilities
1891 .iter()
1892 .flat_map(|declarations| declarations.requires.iter())
1893 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1894 .map(|requirement| requirement.capability.as_str())
1895 .collect::<BTreeSet<_>>();
1896 if required.is_empty() {
1897 return None;
1898 }
1899 let module_id = registration.manifest.module_id.as_str();
1900 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1901 if required
1902 .iter()
1903 .any(|capability| verdict(capability).is_none())
1904 {
1905 self.refresh_capability_requirements();
1906 }
1907 required
1908 .into_iter()
1909 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1910 .map(str::to_string)
1911 }
1912
1913 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1914 self.capability_evaluator
1915 .statuses()
1916 .into_iter()
1917 .map(capability_requirement_status)
1918 .collect()
1919 }
1920
1921 fn deregister_connection(
1928 &self,
1929 connection_id: ConnectionId,
1930 reason: RegistrationEndReason,
1931 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1932 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
1933 spawn.end_primary_registration();
1934 }
1935 self.registry
1936 .deregister_connection_with_reason(connection_id, reason)
1937 }
1938
1939 fn rollback_unpublished_hello(
1944 &self,
1945 connection: ConnectionId,
1946 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1947 self.registry.deregister_connection_with_reason(
1948 connection,
1949 RegistrationEndReason::RegistrationFailed,
1950 )
1951 }
1952
1953 fn request_refusal_close(
1954 &self,
1955 connection: ConnectionId,
1956 sink: Option<&crate::FrameSink>,
1957 reason: CloseReason,
1958 ) {
1959 if let Some(sink) = sink {
1960 sink.terminate_ingress();
1961 }
1962 if self.claims.close_notifications_enabled() {
1963 self.forwarding.request_connection_close(connection, reason);
1964 }
1965 }
1966
1967 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1968 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1969 return None;
1970 }
1971 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1972 parse_client_control_request(&frame.body)
1973 else {
1974 return None;
1975 };
1976 Some(target_module_id(&target).to_string())
1977 }
1978
1979 pub(crate) fn route_open_capacity_refusal(
1980 &self,
1981 ctx: &RouteCtx,
1982 frame: &Frame,
1983 target_module_id: &str,
1984 in_flight: usize,
1985 limit: usize,
1986 ) -> Result<Frame, RouterError> {
1987 let caller = self.route_open_caller_from_frame(ctx, frame);
1990 self.route_open_admission_refusal_frame(
1991 ctx,
1992 &caller,
1993 frame,
1994 target_module_id,
1995 "open_admission_full",
1996 (in_flight, limit),
1997 format!(
1998 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1999 ),
2000 )
2001 }
2002
2003 fn route_open_target_capacity_refusal(
2004 &self,
2005 ctx: &RouteCtx,
2006 caller: &RouteOpenCaller,
2007 frame: &Frame,
2008 target_module_id: &str,
2009 in_flight: usize,
2010 ) -> Result<Frame, RouterError> {
2011 self.route_open_admission_refusal_frame(
2012 ctx,
2013 caller,
2014 frame,
2015 target_module_id,
2016 "target_binds_full",
2017 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
2018 format!(
2019 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2020 ),
2021 )
2022 }
2023
2024 #[allow(clippy::too_many_arguments)]
2032 fn route_open_admission_refusal_frame(
2033 &self,
2034 ctx: &RouteCtx,
2035 caller: &RouteOpenCaller,
2036 frame: &Frame,
2037 target_module_id: &str,
2038 reason: &'static str,
2039 (in_flight, limit): (usize, usize),
2040 message: impl Into<String>,
2041 ) -> Result<Frame, RouterError> {
2042 let code = error_codes::TARGET_UNAVAILABLE;
2043 self.counters.increment_route_open_refused(code);
2044 info!(
2045 target: "control",
2046 code,
2047 reason,
2048 module_id = ?target_module_id,
2049 connection_id = ctx.connection_id.get(),
2050 principal = %caller.principal,
2051 scope = %caller.scope,
2052 harness = %caller.harness,
2053 in_flight,
2054 limit,
2055 "route.open refused"
2056 );
2057 control_error_frame(frame, code, message.into())
2058 }
2059
2060 #[cfg(test)]
2066 pub fn handle_control(
2067 &self,
2068 connection_id: ConnectionId,
2069 frame: Frame,
2070 ) -> Result<Vec<Frame>, RouterError> {
2071 match frame.header.ty {
2072 FrameType::Ping => Ok(vec![pong(&frame)?]),
2073 FrameType::Hello => self.handle_hello(connection_id, None, frame),
2074 FrameType::Goodbye => self.handle_goodbye(connection_id),
2075 ty => Ok(vec![control_error_frame(
2076 &frame,
2077 "unsupported_control_frame",
2078 format!("unsupported channel-0 frame {ty:?}"),
2079 )?]),
2080 }
2081 }
2082
2083 pub async fn handle_control_frame(
2084 &self,
2085 ctx: &RouteCtx,
2086 frame: Frame,
2087 ) -> Result<Vec<Frame>, RouterError> {
2088 self.handle_control_frame_timed(ctx, frame, None).await
2089 }
2090
2091 #[cfg(any(test, feature = "test-support"))]
2092 pub fn with_test_operator_app_policy(
2093 mut self,
2094 policy: crate::operator_apps::TestOperatorAppPolicy,
2095 ) -> Self {
2096 self.operator_app_policy = policy.into();
2097 self
2098 }
2099
2100 #[cfg(all(windows, any(test, feature = "test-support")))]
2104 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
2105 self.windows_verify = Some(verify);
2106 self
2107 }
2108
2109 #[cfg(all(windows, any(test, feature = "test-support")))]
2111 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
2112 self.windows_flash = Some(flash);
2113 self
2114 }
2115
2116 #[cfg(all(windows, any(test, feature = "test-support")))]
2120 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
2121 self.windows_track = Some(track);
2122 self
2123 }
2124
2125 pub(crate) fn record_captured_peer(
2126 &self,
2127 connection: ConnectionId,
2128 peer: subc_os::local_peer::CapturedPeer,
2129 ) {
2130 self.record_observed_peer(connection, peer.observation().clone());
2131 let identity = self.operator_app_policy.config_identity(peer.clone());
2132 #[cfg(windows)]
2133 let identity = identity.or_else(|| self.config_windows_launch_identity(&peer));
2134 if let Some(identity) = identity {
2135 self.config_desktop_stamps
2138 .lock()
2139 .unwrap_or_else(|p| p.into_inner())
2140 .insert(connection, identity);
2141 }
2142 self.captured_peers
2143 .lock()
2144 .unwrap_or_else(|p| p.into_inner())
2145 .insert(connection, peer.clone());
2146 #[cfg(windows)]
2147 if let Some(stamp) = self.stamp_windows_launch(&peer) {
2148 self.operator_apps
2149 .lock()
2150 .unwrap_or_else(|p| p.into_inner())
2151 .insert(connection, stamp);
2152 }
2153 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
2154 self.operator_apps
2155 .lock()
2156 .unwrap_or_else(|p| p.into_inner())
2157 .insert(connection, stamp);
2158 }
2159 }
2160
2161 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
2162 self.local_connections
2163 .lock()
2164 .unwrap_or_else(|p| p.into_inner())
2165 .insert(connection);
2166 }
2167
2168 fn connection_operator_app(
2171 &self,
2172 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
2173 ) -> Option<subc_protocol::peer::OperatorApp> {
2174 stamp
2175 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
2176 .map(|stamp| stamp.app.clone())
2177 }
2178
2179 fn handle_connection_describe(
2180 &self,
2181 ctx: &RouteCtx,
2182 frame: Frame,
2183 ) -> Result<Vec<Frame>, RouterError> {
2184 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
2185 return Ok(vec![control_error_frame(
2186 &frame,
2187 "invalid_control_body",
2188 "malformed connection.describe body",
2189 )?]);
2190 }
2191 let local = self
2192 .local_connections
2193 .lock()
2194 .unwrap_or_else(|p| p.into_inner())
2195 .contains(&ctx.connection_id);
2196 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
2197 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
2198 Ok(vec![control_response_body_frame(
2199 &frame,
2200 &subc_control::ConnectionDescribeReply::new(local, operator_app),
2201 ops::CONNECTION_DESCRIBE,
2202 )?])
2203 }
2204
2205 pub(crate) fn record_observed_peer(
2206 &self,
2207 connection: ConnectionId,
2208 peer: subc_os::local_peer::ObservedPeer,
2209 ) {
2210 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
2211 let peer = match peer {
2212 OsPeer::Unobserved => ObservedPeer::unobserved(),
2213 OsPeer::Observed(peer) => {
2214 let observed = match peer.user {
2215 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
2216 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
2217 };
2218 observed
2219 .with_exe(
2220 peer.exe
2221 .and_then(|path| path.into_os_string().into_string().ok()),
2222 )
2223 .with_signing_id(peer.signing_id)
2224 }
2225 };
2226 self.observed_peers
2227 .lock()
2228 .unwrap_or_else(|p| p.into_inner())
2229 .insert(connection, peer);
2230 }
2231
2232 pub(crate) fn record_caller_endpoints(
2233 &self,
2234 connection: ConnectionId,
2235 endpoints: Option<(SocketAddr, SocketAddr)>,
2236 ) {
2237 let mut callers = self
2238 .caller_endpoints
2239 .lock()
2240 .unwrap_or_else(|p| p.into_inner());
2241 callers.remove(&connection);
2242 if let Some(endpoints) = endpoints {
2243 callers.insert(connection, endpoints);
2244 }
2245 }
2246
2247 #[cfg(test)]
2248 pub(crate) fn with_caller_lookup_for_test(
2249 mut self,
2250 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2251 ) -> Self {
2252 self.caller_lookup = Some(Arc::new(lookup));
2253 self
2254 }
2255
2256 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2257 let endpoints = self
2258 .caller_endpoints
2259 .lock()
2260 .unwrap_or_else(|p| p.into_inner())
2261 .get(&connection)
2262 .copied();
2263 #[cfg(test)]
2264 let lookup = self.caller_lookup.clone();
2265 tokio::task::spawn_blocking(move || {
2266 #[cfg(test)]
2267 if let Some(lookup) = lookup {
2268 return lookup(endpoints);
2269 }
2270 endpoints
2271 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2272 .unwrap_or_default()
2273 })
2274 .await
2275 .unwrap_or_default()
2276 }
2277
2278 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2279 frame.header.channel == 0
2280 && frame.header.ty == FrameType::Request
2281 && self
2282 .forwarding
2283 .module_endpoint_for_connection(ctx.connection_id)
2284 .is_ok_and(|endpoint| endpoint.is_none())
2285 && parse_client_control_request(&frame.body)
2286 .ok()
2287 .is_some_and(|request| privileged_operation(&request).is_some())
2288 }
2289
2290 pub(crate) async fn handle_control_frame_timed(
2291 &self,
2292 ctx: &RouteCtx,
2293 frame: Frame,
2294 dispatch_started_at: Option<StdInstant>,
2295 ) -> Result<Vec<Frame>, RouterError> {
2296 let first = ctx.egress.first_ingress_frame();
2297 self.handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
2298 .await
2299 }
2300
2301 pub(crate) async fn handle_control_frame_ingress(
2302 &self,
2303 ctx: &RouteCtx,
2304 frame: Frame,
2305 dispatch_started_at: Option<StdInstant>,
2306 first: bool,
2307 ) -> Result<Vec<Frame>, RouterError> {
2308 if ctx.egress.ingress_terminal() {
2309 return Ok(Vec::new());
2310 }
2311 #[cfg(test)]
2312 self.claims.record_dispatch(frame.header.corr);
2313 let claim_binding = self.claims.binding(ctx.connection_id);
2314 let mut prefix_attempt = (frame.header.ty == FrameType::Hello
2315 && claim_binding
2316 .as_ref()
2317 .is_some_and(|binding| binding.prefix_module_id().is_some()))
2318 .then(|| self.prefix_hello_attempt(ctx));
2319 if claim_binding
2320 .as_ref()
2321 .is_some_and(|binding| binding.prefix_module_id().is_some())
2322 {
2323 let registered = self
2324 .forwarding
2325 .module_endpoint_for_connection(ctx.connection_id)
2326 .map_err(RouterError::Forwarding)?
2327 .is_some();
2328 if !registered && frame.header.ty != FrameType::Hello {
2329 return self
2330 .refuse_claim_connection(ctx, &frame, "claim_hello_required")
2331 .await;
2332 }
2333 if registered && frame.header.ty == FrameType::Goodbye {
2334 ctx.egress.terminate_ingress();
2335 let result = self.handle_goodbye(ctx.connection_id);
2336 self.claims.connection_closed(ctx.connection_id);
2337 self.request_refusal_close(
2338 ctx.connection_id,
2339 Some(&ctx.egress),
2340 CloseReason::new("claim_registration_ended", "prefix registration ended"),
2341 );
2342 return result;
2343 }
2344 if registered && frame.header.ty == FrameType::Request {
2345 let allowed =
2346 serde_json::from_slice::<ControlOpProbe>(&frame.body).is_ok_and(|probe| {
2347 matches!(
2348 probe.op.as_str(),
2349 "catalog.update"
2350 | "catalog.update_role_declarations"
2351 | "scope.describe"
2352 | "scope.watch"
2353 | "supervisor.live_roots"
2354 | "activity.record"
2355 )
2356 });
2357 if !allowed {
2358 return Ok(vec![control_error_frame(
2359 &frame,
2360 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2361 "prefix registrations have only brokered module operations",
2362 )?]);
2363 }
2364 }
2365 if registered
2366 && frame.header.ty == FrameType::Push
2367 && !serde_json::from_slice::<ControlOpProbe>(&frame.body)
2368 .is_ok_and(|probe| probe.op == "route.status")
2369 {
2370 return Ok(vec![control_error_frame(
2371 &frame,
2372 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2373 "prefix registrations may push only route.status for routes they serve",
2374 )?]);
2375 }
2376 }
2377 match frame.header.ty {
2378 FrameType::Ping => Ok(vec![pong(&frame)?]),
2379 FrameType::Hello => {
2380 if claim_binding
2381 .as_ref()
2382 .is_some_and(|binding| binding.prefix_module_id().is_none())
2383 {
2384 return self
2385 .refuse_claim_connection(ctx, &frame, "claim_hello_not_allowed")
2386 .await;
2387 }
2388 let channel_spawn = self.supervisor.registration_spawn(ctx.connection_id);
2392 if let Some(spawn) = &channel_spawn {
2393 spawn.wait_launch().await;
2394 if !spawn.launch_ready() {
2397 ctx.egress.terminate_ingress();
2398 return Ok(Vec::new());
2399 }
2400 #[cfg(test)]
2401 self.pause_hello_ingress_for_test().await;
2402 } else if claim_binding.is_none() {
2403 if let Ok(hello) = serde_json::from_slice::<serde_json::Value>(&frame.body) {
2404 if let (Some(module), Some(nonce)) = (
2405 hello
2406 .get("manifest")
2407 .and_then(|value| value.get("module_id"))
2408 .and_then(|value| value.as_str()),
2409 hello.get("launch_nonce").and_then(|value| value.as_str()),
2410 ) {
2411 if !self
2412 .config_runtime
2413 .as_ref()
2414 .is_some_and(|owner| owner.module_is_gated(module))
2415 && !self.supervisor.wait_program_binding(module, nonce).await
2416 {
2417 self.request_refusal_close(
2418 ctx.connection_id,
2419 Some(&ctx.egress),
2420 CloseReason::new(
2421 "program_refused",
2422 "the spawned program binding failed",
2423 ),
2424 );
2425 return Ok(vec![control_error_frame(
2426 &frame,
2427 if self
2428 .config_runtime
2429 .as_ref()
2430 .is_some_and(|owner| owner.is_recovering())
2431 {
2432 "config_gated"
2433 } else {
2434 "program_refused"
2435 },
2436 "the spawned program binding failed",
2437 )?]);
2438 }
2439 }
2440 }
2441 }
2442 let handler = self.clone();
2443 let connection = ctx.connection_id;
2444 let egress = ctx.egress.clone();
2445 let correlation = frame.header.corr;
2446 let dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
2447 let span = tracing::Span::current();
2448 let hello_binding = claim_binding.clone();
2449 let replies = tokio::task::spawn_blocking(move || {
2450 tracing::dispatcher::with_default(&dispatcher, || {
2451 span.in_scope(|| {
2452 handler.handle_hello_checked(
2453 connection,
2454 Some(egress),
2455 frame,
2456 true,
2457 hello_binding,
2458 channel_spawn,
2459 )
2460 })
2461 })
2462 })
2463 .await
2464 .map_err(|error| RouterError::backend(0, correlation, error.to_string()))??;
2465 #[cfg(test)]
2466 if prefix_attempt.is_some() {
2467 self.claims.pause_prefix_commit_for_test(false).await;
2468 }
2469 if let Some(attempt) = &mut prefix_attempt {
2470 let accepted = !replies
2471 .iter()
2472 .any(|reply| reply.header.ty == FrameType::Error)
2473 && self
2474 .registry
2475 .get_module_by_connection(ctx.connection_id)
2476 .map_err(|err| RouterError::backend(0, correlation, err.to_string()))?
2477 .is_some();
2478 if accepted {
2479 attempt.accepted = true;
2480 } else {
2481 ctx.egress.terminate_ingress();
2482 for reply in replies {
2483 ctx.egress.send_flushed(reply).await?;
2484 }
2485 return Ok(Vec::new());
2486 }
2487 }
2488 Ok(replies)
2489 }
2490 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2491 FrameType::Cancel => {
2492 if self
2498 .agent_declarations
2499 .cancel(ctx.connection_id, frame.header.corr)
2500 || self
2501 .forwarding
2502 .operator_confirms()
2503 .cancel(ctx.connection_id, frame.header.corr)
2504 || self.cancel_config_approval(ctx.connection_id, frame.header.corr)
2505 || self
2506 .supervisor
2507 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2508 || self.cancel_activity_subscription(ctx, &frame)?
2509 || self.cancel_catalog_subscription(ctx.connection_id, frame.header.corr)
2510 || self.cancel_scope_subscription(ctx.connection_id, frame.header.corr)
2511 {
2512 Ok(Vec::new())
2513 } else {
2514 Ok(vec![control_error_frame(
2515 &frame,
2516 "unknown_subscription",
2517 "no subscription has this correlation id",
2518 )?])
2519 }
2520 }
2521 FrameType::Request => {
2522 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2525 let claim_present = op.as_ref().is_some_and(|probe| probe.op == "claim.present");
2526 if let Some(refusal) = self
2527 .recovery_request_refusal(
2528 ctx,
2529 &frame,
2530 op.as_ref().map(|probe| probe.op.as_str()),
2531 )
2532 .await?
2533 {
2534 if claim_present {
2535 self.reclaim_presented_expiry(&frame);
2536 return self
2537 .refuse_claim_connection(ctx, &frame, "config_gated")
2538 .await;
2539 }
2540 return Ok(vec![refusal]);
2541 }
2542 if claim_present {
2543 return self.handle_claim_present(ctx, &frame, first).await;
2544 }
2545 if op.as_ref().is_some_and(|probe| probe.op == "claim.mint") {
2546 return self.handle_claim_mint(ctx, &frame).await;
2547 }
2548 if let Some(refusal) = self.registration_operation_refusal(
2549 ctx,
2550 &frame,
2551 op.as_ref().map(|probe| probe.op.as_str()),
2552 )? {
2553 return Ok(vec![refusal]);
2554 }
2555 if op
2556 .as_ref()
2557 .is_some_and(|probe| probe.op == ops::CATALOG_WATCH)
2558 {
2559 return self.handle_catalog_watch(ctx, frame);
2560 }
2561 if op
2562 .as_ref()
2563 .is_some_and(|probe| probe.op == ops::SCOPE_WATCH)
2564 {
2565 return self.handle_scope_watch(ctx, frame);
2566 }
2567 if op
2568 .as_ref()
2569 .is_some_and(|probe| probe.op == MODULE_TO_SUBC_OP_CATALOG_UPDATE)
2570 && subc_protocol::provider_declarations::catalog_update_has_provider_declarations(&frame.body)
2571 .unwrap_or(false)
2572 {
2573 return Ok(vec![control_error_frame(
2574 &frame,
2575 error_codes::CATALOG_UPDATE_FROZEN_DECLARATIONS,
2576 "provider_declarations are fixed at HELLO; restart the module to change them",
2577 )?]);
2578 }
2579 if let Some(probe) = op
2580 .as_ref()
2581 .filter(|probe| config_gate::is_module_config_op(&probe.op))
2582 {
2583 return self
2584 .handle_module_config_request(ctx, frame, &probe.op)
2585 .await;
2586 }
2587 if op
2588 .as_ref()
2589 .is_some_and(|probe| config_gate::is_config_op(&probe.op))
2590 {
2591 return self.handle_config_gate_request(ctx, frame).await;
2592 }
2593 if op
2594 .as_ref()
2595 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2596 {
2597 return self.handle_activity_record(ctx, frame).await;
2598 }
2599 if op.as_ref().is_some_and(|probe| {
2600 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2601 }) {
2602 return self.handle_activity_read(ctx, frame).await;
2603 }
2604 if op
2605 .as_ref()
2606 .is_some_and(|probe| probe.op == "operator.confirm")
2607 {
2608 return self.handle_operator_confirm(ctx, frame);
2609 }
2610 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2611 return self
2612 .observe_control_wait(
2613 ctx,
2614 None,
2615 ops::APP_OPEN,
2616 self.handle_app_open(ctx, frame),
2617 )
2618 .await;
2619 }
2620 if op
2621 .as_ref()
2622 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2623 {
2624 return self.handle_connection_describe(ctx, frame);
2625 }
2626 if op
2627 .as_ref()
2628 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2629 {
2630 return self.handle_scope_served_by(ctx.connection_id, frame);
2631 }
2632 if self
2633 .forwarding
2634 .module_endpoint_for_connection(ctx.connection_id)
2635 .map_err(RouterError::Forwarding)?
2636 .is_some()
2637 {
2638 if !is_known_module_request_op(&frame.body) {
2639 return Ok(vec![control_error_frame(
2640 &frame,
2641 "unsupported_control_frame",
2642 "module-originated channel-0 REQUEST is not supported",
2643 )?]);
2644 }
2645 let request = match parse_module_control_request_from_module(&frame.body) {
2646 Ok(request) => request,
2647 Err((err, ControlRequestBodyError::UnknownOp)) => {
2648 return Ok(vec![control_error_frame(
2649 &frame,
2650 "unsupported_control_frame",
2651 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2652 )?])
2653 }
2654 Err((err, ControlRequestBodyError::InvalidBody)) => {
2655 return Ok(vec![control_error_frame(
2656 &frame,
2657 "invalid_control_body",
2658 format!("malformed module control body: {err}"),
2659 )?])
2660 }
2661 };
2662 let op = module_control_request_op(&request);
2663 let corr = frame.header.corr;
2664 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2665 if let ModuleControlRequestFromModule::AgentCommands {
2666 started,
2667 finished,
2668 known,
2669 } = request
2670 {
2671 return self
2672 .agent_declarations
2673 .enqueue(
2674 self,
2675 ctx,
2676 frame,
2677 crate::resource_totals::Declaration {
2678 started,
2679 finished,
2680 known,
2681 },
2682 )
2683 .await;
2684 }
2685 let result =
2686 self.handle_module_control_request(ctx.connection_id, frame, request);
2687 log_slow_control_dispatch(
2688 dispatch_started_at,
2689 op,
2690 ctx.connection_id,
2691 corr,
2692 None,
2693 );
2694 return result;
2695 }
2696
2697 if is_known_module_request_op(&frame.body) {
2698 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2699 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2700 let code = if is_agents {
2701 "not_supervised"
2702 } else if op
2703 .as_ref()
2704 .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2705 {
2706 error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2707 } else {
2708 "not_registered"
2709 };
2710 return Ok(vec![control_error_frame(
2711 &frame,
2712 code,
2713 "module-originated control requests require the module's own registered connection",
2714 )?]);
2715 }
2716
2717 let request = match parse_client_control_request(&frame.body) {
2718 Ok(request) => request,
2719 Err((err, ControlRequestBodyError::UnknownOp)) => {
2720 return Ok(vec![control_error_frame(
2721 &frame,
2722 "unknown_control_op",
2723 format!("unknown client control op: {err}"),
2724 )?])
2725 }
2726 Err((err, ControlRequestBodyError::InvalidBody)) => {
2727 return Ok(vec![control_error_frame(
2728 &frame,
2729 "invalid_control_body",
2730 format!("malformed client control body: {err}"),
2731 )?])
2732 }
2733 };
2734 let op = client_control_request_op(&request);
2735 let corr = frame.header.corr;
2736 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2737 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2738 #[cfg(test)]
2739 if let Some(delay) = self.control_dispatch_delay {
2740 tokio::time::sleep(delay).await;
2741 }
2742 let result = self
2743 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2744 .await;
2745 if let Some(timings) = route_open_timings.as_mut() {
2746 timings.finish();
2747 }
2748 log_slow_control_dispatch(
2749 dispatch_started_at,
2750 op,
2751 ctx.connection_id,
2752 corr,
2753 route_open_timings.as_ref(),
2754 );
2755 result
2756 }
2757 FrameType::Push => {
2758 let Some(endpoint) = self
2759 .forwarding
2760 .module_endpoint_for_connection(ctx.connection_id)
2761 .map_err(RouterError::Forwarding)?
2762 else {
2763 return Ok(vec![control_error_frame(
2764 &frame,
2765 "unsupported_control_frame",
2766 "client-originated channel-0 PUSH is not supported",
2767 )?]);
2768 };
2769 self.handle_status_update(endpoint, frame)
2770 }
2771 FrameType::Response | FrameType::Error
2772 if self
2773 .forwarding
2774 .module_endpoint_for_connection(ctx.connection_id)
2775 .map_err(RouterError::Forwarding)?
2776 .is_some() =>
2777 {
2778 self.handle_module_relay_response(ctx.connection_id, frame)
2779 }
2780 ty => Ok(vec![control_error_frame(
2781 &frame,
2782 "unsupported_control_frame",
2783 format!("unsupported channel-0 frame {ty:?}"),
2784 )?]),
2785 }
2786 }
2787
2788 #[cfg(test)]
2789 fn pause_cleanup_for_test(&self, stage: u8) {
2790 let mut slot = self.cleanup_pause.lock().unwrap_or_else(|p| p.into_inner());
2791 if slot.as_ref().is_some_and(|(at, _)| *at == stage) {
2792 let (_, (reached, release)) = slot.take().unwrap();
2793 drop(slot);
2794 let _ = reached.send(());
2795 let _ = release.recv_timeout(Duration::from_secs(5));
2796 }
2797 }
2798
2799 #[cfg(all(test, unix, feature = "test-support"))]
2802 pub(crate) fn after_nonce_attestation_for_test(&self, hook: impl FnOnce() + Send + 'static) {
2803 *self
2804 .after_nonce_attestation
2805 .lock()
2806 .unwrap_or_else(|p| p.into_inner()) = Some(Box::new(hook));
2807 }
2808
2809 #[cfg(test)]
2810 pub(super) fn pause_after_nonce_attestation_for_test(&self) {
2811 let hook = self
2812 .after_nonce_attestation
2813 .lock()
2814 .unwrap_or_else(|p| p.into_inner())
2815 .take();
2816 if let Some(hook) = hook {
2817 hook();
2818 }
2819 }
2820
2821 #[cfg(test)]
2822 async fn pause_hello_ingress_for_test(&self) {
2823 let pause = self
2824 .hello_ingress_pause
2825 .lock()
2826 .unwrap_or_else(|p| p.into_inner())
2827 .take();
2828 if let Some((reached, release)) = pause {
2829 let _ = reached.send(());
2830 let _ = release.await;
2831 }
2832 }
2833
2834 pub fn cleanup_connection(
2835 &self,
2836 connection_id: ConnectionId,
2837 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2838 self.agent_declarations.close(connection_id);
2852 self.close_catalog_subscriptions(connection_id);
2853 self.close_scope_subscriptions(connection_id);
2854 #[cfg(test)]
2855 self.pause_cleanup_for_test(0);
2856 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
2857 spawn.retire_registration_channel();
2858 }
2859 self.supervisor
2860 .forget_registration_connection(connection_id);
2861 #[cfg(test)]
2862 self.pause_cleanup_for_test(1);
2863 self.local_connections
2864 .lock()
2865 .unwrap_or_else(|p| p.into_inner())
2866 .remove(&connection_id);
2867 self.caller_endpoints
2868 .lock()
2869 .unwrap_or_else(|p| p.into_inner())
2870 .remove(&connection_id);
2871 self.build_class_warned
2872 .lock()
2873 .unwrap_or_else(|p| p.into_inner())
2874 .remove(&connection_id);
2875 self.observed_peers
2876 .lock()
2877 .unwrap_or_else(|p| p.into_inner())
2878 .remove(&connection_id);
2879 self.captured_peers
2880 .lock()
2881 .unwrap_or_else(|p| p.into_inner())
2882 .remove(&connection_id);
2883 self.config_desktop_stamps
2884 .lock()
2885 .unwrap_or_else(|p| p.into_inner())
2886 .remove(&connection_id);
2887 self.operator_apps
2888 .lock()
2889 .unwrap_or_else(|p| p.into_inner())
2890 .remove(&connection_id);
2891 self.forwarding
2892 .operator_confirms()
2893 .connection_closed(connection_id);
2894 let cleanup = self.cleanup_connection_with_end_reason(
2895 connection_id,
2896 RegistrationEndReason::ConnectionClosed,
2897 );
2898 self.claims.connection_closed(connection_id);
2901 cleanup
2902 }
2903
2904 fn cleanup_connection_with_end_reason(
2905 &self,
2906 connection_id: ConnectionId,
2907 requested_reason: RegistrationEndReason,
2908 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2909 #[cfg(test)]
2910 if let Some(pause) = self
2911 .activity_request_pause
2912 .lock()
2913 .unwrap()
2914 .as_ref()
2915 .filter(|pause| pause.connection == connection_id)
2916 {
2917 pause
2918 .cleanup_started
2919 .store(true, std::sync::atomic::Ordering::Release);
2920 }
2921 if let Some(runtime) = &self.config_runtime {
2922 runtime.disconnected(connection_id);
2923 }
2924 self.agent_declarations.close(connection_id);
2925 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2926 self.registry
2927 .get_module_by_connection(connection_id)?
2928 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2929 .and_then(|module| module.registration_end_reason().ok().flatten())
2930 .unwrap_or(requested_reason)
2931 } else {
2932 requested_reason
2933 };
2934 let crash_closed = self
2935 .registry
2936 .get_module_by_connection(connection_id)?
2937 .and_then(|registration| {
2938 self.forwarding
2939 .module_endpoint_for_connection(connection_id)
2940 .ok()
2941 .flatten()
2942 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2943 .map(|routes| (registration.manifest.module_id, routes))
2944 });
2945 let crash_closed = crash_closed.map(|(module_id, routes)| {
2946 let terminal = match self.supervisor.get(&module_id) {
2947 None => false,
2948 Some(module) => match module.will_recover_after_connection_loss() {
2949 Ok(will_recover) => !will_recover,
2950 Err(err) => {
2951 warn!(
2952 %module_id,
2953 error = %err,
2954 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2955 );
2956 false
2957 }
2958 },
2959 };
2960 let (reason, drained) = match self
2964 .forwarding
2965 .daemon_drain_result_for_connection(connection_id)
2966 {
2967 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2968 Ok(None) => (RouteCloseReason::Crash, false),
2969 Err(err) => {
2970 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2971 (RouteCloseReason::Crash, false)
2972 }
2973 };
2974 (module_id, routes, reason, drained, terminal)
2975 });
2976 let registrations = self.deregister_connection(connection_id, end_reason);
2977 let ended_authority = self
2980 .forwarding
2981 .target_authority_for_connection(connection_id)
2982 .ok()
2983 .flatten();
2984 let cleanup = if crash_closed.is_some() {
2985 self.forwarding.cleanup_connection_counted(connection_id)
2986 } else {
2987 self.forwarding
2993 .cleanup_connection(connection_id)
2994 .map(|released| crate::forwarding::ConnectionCleanup {
2995 released,
2996 abandoned_relays: 0,
2997 })
2998 };
2999 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
3004 let abandoned = cleanup
3005 .as_ref()
3006 .map_or(0, |cleanup| cleanup.abandoned_relays);
3007 send_route_control_pushes(
3008 &self.forwarding,
3009 routes,
3010 ClientControlPush::RouteClosed {
3011 module_id,
3012 channels: Vec::new(),
3013 reason,
3014 drained,
3015 abandoned,
3016 excluded_subscriptions: 0,
3017 terminal: Some(terminal),
3018 },
3019 );
3020 }
3021 if let Ok(cleanup) = cleanup {
3022 self.emit_route_goodbyes(cleanup.released);
3023 }
3024 if let Some(record) = ended_authority {
3025 self.change_spawn_authority(&record);
3026 }
3027 if matches!(®istrations, Ok(r) if !r.is_empty()) {
3031 crate::supervise::notify_registration_release();
3032 self.capability_evaluator.wake_deadline_loop();
3033 self.refresh_capability_requirements();
3034 }
3035 self.supervisor.remove_spawn_subscribers(connection_id);
3036 self.activity_streams
3037 .lock()
3038 .unwrap_or_else(|p| p.into_inner())
3039 .retain(|(id, _), task| {
3040 if *id == connection_id {
3041 task.abort();
3042 false
3043 } else {
3044 true
3045 }
3046 });
3047 self.hello_launch_nonces
3050 .lock()
3051 .unwrap_or_else(|poisoned| poisoned.into_inner())
3052 .forget(connection_id);
3053 self.scopes
3054 .write()
3055 .unwrap_or_else(|poisoned| poisoned.into_inner())
3056 .release_connection(connection_id);
3057 registrations
3058 }
3059
3060 pub(crate) fn handle_route_goodbye(
3061 &self,
3062 connection_id: ConnectionId,
3063 route_channel: u16,
3064 route_epoch: u32,
3065 ) -> Result<bool, RouterError> {
3066 debug!(
3067 connection_id = connection_id.get(),
3068 route_channel, route_epoch, "handling route GOODBYE"
3069 );
3070 let RouteRelease::Removed(released_route) = self
3071 .forwarding
3072 .release_client_route(connection_id, route_channel, route_epoch)
3073 .map_err(RouterError::Forwarding)?
3074 else {
3075 return Ok(false);
3076 };
3077 self.emit_route_goodbyes(vec![released_route]);
3078 Ok(true)
3079 }
3080
3081 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
3082 scope_expiry::ScopeExpiryContext::new(self).emit_route_goodbyes(released_routes);
3083 }
3084
3085 fn send_abandoned_route_bind_goodbye(
3097 &self,
3098 module_sink: &crate::FrameSink,
3099 negotiated_ver: u8,
3100 module_channel: u16,
3101 module_epoch: u32,
3102 ) {
3103 let frame = match Frame::build_with_version(
3104 negotiated_ver,
3105 FrameType::Goodbye,
3106 control_flags(),
3107 module_channel,
3108 module_epoch,
3109 0,
3110 Vec::new(),
3111 ) {
3112 Ok(frame) => frame,
3113 Err(err) => {
3114 warn!(
3115 route_channel = module_channel,
3116 error = %err,
3117 "failed to build GOODBYE for abandoned route.bind"
3118 );
3119 return;
3120 }
3121 };
3122 crate::forwarding::send_module_route_goodbye(
3123 &self.counters,
3124 module_sink,
3125 frame,
3126 None,
3127 "abandoned route.bind",
3128 );
3129 }
3130
3131 #[cfg(test)]
3132 fn handle_hello(
3133 &self,
3134 connection_id: ConnectionId,
3135 egress: Option<crate::router::FrameSink>,
3136 frame: Frame,
3137 ) -> Result<Vec<Frame>, RouterError> {
3138 let binding = self.claims.binding(connection_id);
3139 let channel_spawn = self.supervisor.registration_spawn(connection_id);
3140 self.handle_hello_checked(connection_id, egress, frame, false, binding, channel_spawn)
3141 }
3142
3143 fn handle_hello_checked(
3144 &self,
3145 connection_id: ConnectionId,
3146 sink: Option<crate::FrameSink>,
3147 frame: Frame,
3148 wait_for_gate: bool,
3149 prefix_binding: Option<claims::ClaimBinding>,
3150 channel_spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3151 ) -> Result<Vec<Frame>, RouterError> {
3152 let primary_sink = channel_spawn.as_ref().and_then(|_| sink.clone());
3153 #[cfg(test)]
3154 if prefix_binding.is_some() {
3155 self.claims.pause_prefix_hello_for_test(5);
3156 match self.claims.take_prefix_hello_failure_for_test() {
3157 Some("error") => {
3158 return Err(RouterError::backend(
3159 0,
3160 frame.header.corr,
3161 "injected HELLO failure",
3162 ))
3163 }
3164 Some("panic") => panic!("injected HELLO task panic"),
3165 _ => {}
3166 }
3167 }
3168 if let Some(binding) = &prefix_binding {
3169 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3170 let module = value
3171 .as_ref()
3172 .and_then(|value| value["manifest"]["module_id"].as_str());
3173 if !binding.admit_prefix_hello(module)
3174 || !self.prefix_binding_live(binding, connection_id)
3175 || !module.is_some_and(|module| {
3176 self.supervisor
3177 .prefix_claim_authorized(&binding.registration.module_id, module)
3178 })
3179 || binding.registration.check().is_err()
3180 {
3181 return Ok(vec![control_error_frame(
3182 &frame,
3183 "claim_prefix_refused",
3184 "HELLO must name the prefix claim's exact module id once",
3185 )?]);
3186 }
3187 }
3188 struct RefuseUnregisteredChannel {
3189 spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3190 sink: Option<crate::FrameSink>,
3191 }
3192 impl Drop for RefuseUnregisteredChannel {
3193 fn drop(&mut self) {
3194 if let Some(spawn) = &self.spawn {
3195 if !spawn.registered() {
3196 if let Some(sink) = &self.sink {
3197 sink.terminate_ingress();
3198 }
3199 spawn.retire_registration_channel();
3200 }
3201 }
3202 }
3203 }
3204 let _channel_attempt = RefuseUnregisteredChannel {
3205 spawn: channel_spawn.clone(),
3206 sink: sink.clone(),
3207 };
3208 if let Some(spawn) = &channel_spawn {
3209 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3210 if let Err(code) = spawn.admit_hello(
3211 value
3212 .as_ref()
3213 .and_then(|value| value["manifest"]["module_id"].as_str()),
3214 ) {
3215 return Ok(vec![control_error_frame(
3216 &frame,
3217 code,
3218 "registration channel HELLO refused",
3219 )?]);
3220 }
3221 }
3222 debug!(
3223 connection_id = connection_id.get(),
3224 corr = frame.header.corr,
3225 "handling HELLO"
3226 );
3227 if self
3231 .registry
3232 .get_module_by_connection(connection_id)
3233 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3234 .is_some()
3235 {
3236 return Ok(vec![control_error_frame(
3237 &frame,
3238 "invalid_hello",
3239 "connection is already registered as a module",
3240 )?]);
3241 }
3242 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
3243 Ok(value) => value,
3244 Err(err) => {
3245 return Ok(vec![control_error_frame(
3246 &frame,
3247 "invalid_hello",
3248 format!("malformed HELLO body: {err}"),
3249 )?])
3250 }
3251 };
3252 if let Err(detail) = catalog::validate_hello_declarations(&hello_value) {
3253 return Ok(vec![control_error_frame(
3254 &frame,
3255 error_codes::INVALID_PROVIDER_DECLARATIONS,
3256 detail,
3257 )?]);
3258 }
3259 if let Some(module) = hello_value
3260 .get("manifest")
3261 .and_then(|value| value.get("module_id"))
3262 .and_then(|value| value.as_str())
3263 {
3264 let recovering = self
3265 .config_runtime
3266 .as_ref()
3267 .is_some_and(|owner| owner.check_recovery().unwrap_or(true));
3268 if recovering
3272 && channel_spawn.as_ref().is_none_or(|spawn| {
3273 !self.supervisor.has_live_verified_channel_binding(spawn)
3274 || spawn.check().is_err()
3275 })
3276 && !(channel_spawn.is_none()
3277 && hello_value
3278 .get("launch_nonce")
3279 .and_then(|value| value.as_str())
3280 .is_some_and(|nonce| {
3281 self.supervisor.spawned_consumer_authorized(module, nonce)
3282 && self
3283 .supervisor
3284 .has_live_verified_program_binding(module, nonce)
3285 }))
3286 {
3287 return Ok(vec![control_error_frame(
3288 &frame,
3289 "config_gated",
3290 "recovery admits only previously bound supervised launches",
3291 )?]);
3292 }
3293 if self
3294 .config_runtime
3295 .as_ref()
3296 .is_some_and(|owner| owner.module_is_gated(module))
3297 && (channel_spawn
3298 .as_ref()
3299 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3300 || self.config_runtime.as_ref().is_some_and(|owner| {
3301 !recovering && owner.check_module_hold(module).is_err()
3302 }))
3303 {
3304 return Ok(vec![control_error_frame(
3305 &frame,
3306 "reserved_module",
3307 "a gated identity belongs only to its supervised child",
3308 )?]);
3309 }
3310 }
3311 if channel_spawn.is_none() {
3312 if let (Some(module), Some(nonce)) = (
3313 hello_value
3314 .get("manifest")
3315 .and_then(|value| value.get("module_id"))
3316 .and_then(|value| value.as_str()),
3317 hello_value
3318 .get("launch_nonce")
3319 .and_then(|value| value.as_str()),
3320 ) {
3321 let binding = self.supervisor.has_program_binding_record(module, nonce);
3322 let required = self
3323 .config_runtime
3324 .as_ref()
3325 .is_some_and(|owner| owner.module_is_gated(module));
3326 if let Some(error) = (binding || required)
3327 .then(|| {
3328 self.supervisor
3329 .check_current_program_binding(module, nonce)
3330 .err()
3331 })
3332 .flatten()
3333 {
3334 let code = if error.code == "program_refused" {
3335 "program_refused"
3336 } else {
3337 "config_delivery_deferred"
3338 };
3339 self.request_refusal_close(
3340 connection_id,
3341 sink.as_ref(),
3342 CloseReason::new(code, error.detail.clone()),
3343 );
3344 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3345 }
3346 }
3347 }
3348 if let Err(err) = validate_hello_display_name(&hello_value) {
3349 return Ok(vec![control_error_frame(
3350 &frame,
3351 "invalid_manifest",
3352 err.to_string(),
3353 )?]);
3354 }
3355 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
3356 return Ok(vec![control_error_frame(
3357 &frame,
3358 "invalid_capability_grammar",
3359 err.to_string(),
3360 )?]);
3361 }
3362 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
3363 return Ok(vec![control_error_frame(
3364 &frame,
3365 "invalid_manifest",
3366 err.to_string(),
3367 )?]);
3368 }
3369 if let Err(err) = validate_hello_event_declarations(&hello_value) {
3370 return Ok(vec![control_error_frame(
3371 &frame,
3372 "invalid_event_declaration",
3373 err.to_string(),
3374 )?]);
3375 }
3376 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
3377 return Ok(vec![control_error_frame(
3378 &frame,
3379 "invalid_manifest",
3380 err.to_string(),
3381 )?]);
3382 }
3383 if let Some(provenance) = hello_value
3384 .get("manifest")
3385 .and_then(|manifest| manifest.get("provenance"))
3386 {
3387 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
3388 return Ok(vec![control_error_frame(
3389 &frame,
3390 "invalid_manifest",
3391 format!("malformed manifest provenance: {err}"),
3392 )?]);
3393 }
3394 }
3395 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
3396 let module_id = hello_value["manifest"]["module_id"]
3397 .as_str()
3398 .unwrap_or("<unknown>");
3399 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
3400 }
3401 if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
3402 let module_id = hello_value
3403 .get("manifest")
3404 .and_then(|m| m.get("module_id"))
3405 .and_then(serde_json::Value::as_str)
3406 .unwrap_or("unknown");
3407 warn!(module_id, %reason, "dropping operation_deferral from HELLO");
3408 }
3409 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
3410 let module_id = hello_value["manifest"]["module_id"]
3411 .as_str()
3412 .unwrap_or("<unknown>");
3413 warn!(module_id, %reason, "dropping flow_operations from HELLO");
3414 }
3415 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
3416 let module_id = hello_value["manifest"]["module_id"]
3417 .as_str()
3418 .unwrap_or("<unknown>");
3419 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
3420 }
3421 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
3422 Ok(hello) => hello,
3423 Err(err) => {
3424 return Ok(vec![control_error_frame(
3425 &frame,
3426 "invalid_hello",
3427 format!("malformed HELLO body: {err}"),
3428 )?])
3429 }
3430 };
3431
3432 if hello.protocol_ver != hello.manifest.protocol_ver {
3433 return Ok(vec![control_error_frame(
3434 &frame,
3435 "invalid_manifest",
3436 format!(
3437 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
3438 hello.protocol_ver, hello.manifest.protocol_ver
3439 ),
3440 )?]);
3441 }
3442
3443 if hello.manifest.module_id.trim().is_empty() {
3444 return Ok(vec![control_error_frame(
3445 &frame,
3446 "invalid_manifest",
3447 "manifest module_id must not be empty",
3448 )?]);
3449 }
3450
3451 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
3452 Ok(negotiated_ver) => negotiated_ver,
3453 Err(message) => {
3454 return Ok(vec![control_error_frame(
3455 &frame,
3456 "version_unsupported",
3457 message,
3458 )?])
3459 }
3460 };
3461
3462 let _publication_gate = if let Some(owner) = &self.config_runtime {
3468 let gate = owner
3469 .module_gate(&hello.manifest.module_id)
3470 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3471 Some(if wait_for_gate {
3472 gate.blocking_lock_owned()
3473 } else {
3474 match gate.try_lock_owned() {
3475 Ok(guard) => guard,
3476 Err(_) => {
3477 return Ok(vec![control_error_frame(
3478 &frame,
3479 "config_delivery_deferred",
3480 "module admission is busy",
3481 )?])
3482 }
3483 }
3484 })
3485 } else {
3486 None
3487 };
3488 #[cfg(test)]
3489 if let Some(hold) = &self.hello_before_publication_hold {
3490 hold.wait();
3491 hold.wait();
3492 }
3493 let module = &hello.manifest.module_id;
3494 let required = self
3495 .config_runtime
3496 .as_ref()
3497 .is_some_and(|owner| owner.module_is_gated(module));
3498 let recovering = self
3499 .config_runtime
3500 .as_ref()
3501 .is_some_and(|owner| owner.is_recovering());
3502 if recovering
3510 && channel_spawn.as_ref().is_none_or(|spawn| {
3511 !self.supervisor.has_live_verified_channel_binding(spawn) || spawn.check().is_err()
3512 })
3513 && !(channel_spawn.is_none()
3514 && hello.launch_nonce.as_deref().is_some_and(|nonce| {
3515 self.supervisor.spawned_consumer_authorized(module, nonce)
3516 && self
3517 .supervisor
3518 .has_live_verified_program_binding(module, nonce)
3519 }))
3520 {
3521 return Ok(vec![control_error_frame(
3522 &frame,
3523 "config_gated",
3524 "recovery admits only previously bound supervised launches",
3525 )?]);
3526 }
3527 if required
3528 && (channel_spawn
3529 .as_ref()
3530 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3531 || self
3532 .config_runtime
3533 .as_ref()
3534 .is_some_and(|owner| !recovering && owner.check_module_hold(module).is_err()))
3535 {
3536 return Ok(vec![control_error_frame(
3537 &frame,
3538 "reserved_module",
3539 "gated admission requires the current supervised child's registration channel and verified image binding",
3540 )?]);
3541 }
3542 if let Some(spawn) = &channel_spawn {
3543 if let Err(code) = spawn.check() {
3544 return Ok(vec![control_error_frame(
3545 &frame,
3546 code,
3547 "registration launch is no longer live",
3548 )?]);
3549 }
3550 } else if let Some(nonce) = hello.launch_nonce.as_deref() {
3551 if required || self.supervisor.has_program_binding_record(module, nonce) {
3552 if let Err(error) = self.supervisor.check_current_program_binding(module, nonce) {
3553 let code = if error.code == "program_refused" {
3554 "program_refused"
3555 } else {
3556 "config_delivery_deferred"
3557 };
3558 self.request_refusal_close(
3559 connection_id,
3560 sink.as_ref(),
3561 CloseReason::new(code, error.detail.clone()),
3562 );
3563 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3564 }
3565 }
3566 }
3567
3568 let swap_admission = match &channel_spawn {
3576 Some(spawn) if spawn.is_candidate() => SwapHelloAdmission::Candidate,
3577 Some(_) => SwapHelloAdmission::NotSwapping,
3578 None if prefix_binding.is_some() => SwapHelloAdmission::NotSwapping,
3579 None => self
3580 .supervisor
3581 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref()),
3582 };
3583 if swap_admission == SwapHelloAdmission::ProtocolNone {
3584 warn!(
3585 module_id = %hello.manifest.module_id,
3586 connection_id = connection_id.get(),
3587 "HELLO refused: the module_id belongs to a protocol:none module, which never registers"
3588 );
3589 return Ok(vec![control_error_frame(
3592 &frame,
3593 "swap_token_invalid",
3594 format!(
3595 "module_id '{}' is a protocol:none module; a protocol:none module may not register",
3596 hello.manifest.module_id
3597 ),
3598 )?]);
3599 }
3600 if swap_admission == SwapHelloAdmission::ChannelRequired {
3601 warn!(
3602 module_id = %hello.manifest.module_id,
3603 connection_id = connection_id.get(),
3604 "HELLO refused: the module_id is gated and the HELLO did not arrive over its registration channel"
3605 );
3606 return Ok(vec![control_error_frame(
3609 &frame,
3610 "swap_token_invalid",
3611 format!(
3612 "module_id '{}' is gated; it must register over its supervised registration channel",
3613 hello.manifest.module_id
3614 ),
3615 )?]);
3616 }
3617 if swap_admission == SwapHelloAdmission::Refused {
3618 warn!(
3619 module_id = %hello.manifest.module_id,
3620 connection_id = connection_id.get(),
3621 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
3622 );
3623 return Ok(vec![control_error_frame(
3624 &frame,
3625 "swap_token_invalid",
3626 format!(
3627 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
3628 hello.manifest.module_id
3629 ),
3630 )?]);
3631 }
3632 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
3633
3634 let verification = if swap_candidate && channel_spawn.is_none() {
3639 ReservedHelloVerification {
3640 nonce_verified: true,
3641 prefix_owner: None,
3642 }
3643 } else {
3644 let verification = if let Some(binding) = &prefix_binding {
3645 if !self.supervisor.prefix_claim_authorized(
3646 &binding.registration.module_id,
3647 &hello.manifest.module_id,
3648 ) {
3649 return Ok(vec![control_error_frame(
3650 &frame,
3651 "claim_prefix_refused",
3652 "prefix ownership changed",
3653 )?]);
3654 }
3655 Ok(ReservedHelloVerification {
3656 nonce_verified: true,
3657 prefix_owner: Some(binding.registration.module_id.clone()),
3658 })
3659 } else if channel_spawn.is_some() {
3660 self.supervisor
3661 .channel_hello_verification(&hello.manifest.module_id)
3662 } else {
3663 self.supervisor.reserved_hello_verification(
3664 &hello.manifest.module_id,
3665 hello.launch_nonce.as_deref(),
3666 )
3667 };
3668 match verification {
3669 Ok(verified) => verified,
3670 Err(rejection) => {
3671 let message = match rejection {
3672 ReservedHelloRejection::Exact { module_id } => format!(
3673 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
3674 ),
3675 ReservedHelloRejection::Prefix {
3676 prefix,
3677 owner_module_id,
3678 } => format!(
3679 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
3680 hello.manifest.module_id
3681 ),
3682 };
3683 return Ok(vec![control_error_frame(
3684 &frame,
3685 "reserved_module",
3686 message,
3687 )?]);
3688 }
3689 }
3690 };
3691
3692 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
3693 &hello.manifest.module_id,
3694 hello.manifest.capabilities.as_ref(),
3695 );
3696 if let Some(refusal) = reserved_capability_refusals.first() {
3697 let capability = refusal.capability.clone();
3698 let bound_module = refusal.claimants[0].clone();
3699 log_duplicate_claim_events(reserved_capability_refusals);
3700 return Ok(vec![control_error_frame(
3701 &frame,
3702 "reserved_capability",
3703 format!(
3704 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3705 capability, bound_module, hello.manifest.module_id
3706 ),
3707 )?]);
3708 }
3709
3710 if self
3713 .forwarding
3714 .connection_has_client_routes(connection_id)
3715 .map_err(RouterError::Forwarding)?
3716 {
3717 return Ok(vec![control_error_frame(
3718 &frame,
3719 "invalid_hello",
3720 "connection has open client routes and cannot also register as a module",
3721 )?]);
3722 }
3723
3724 self.hello_launch_nonces
3732 .lock()
3733 .unwrap_or_else(|poisoned| poisoned.into_inner())
3734 .record(
3735 connection_id,
3736 if prefix_binding.is_some() || channel_spawn.is_some() {
3737 None
3738 } else {
3739 hello.launch_nonce.as_deref()
3740 },
3741 );
3742 let control_ops = effective_module_control_ops(hello.control_ops);
3743 let prefix_submodule = verification.prefix_owner.is_some();
3744 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
3747 #[cfg(test)]
3748 struct PublicationFinished(Arc<std::sync::Barrier>);
3749 #[cfg(test)]
3750 impl Drop for PublicationFinished {
3751 fn drop(&mut self) {
3752 self.0.wait();
3753 }
3754 }
3755 #[cfg(test)]
3756 let _publication_finished = self.hello_after_launch_check_hold.as_ref().map(|hold| {
3757 hold.wait();
3758 hold.wait();
3759 PublicationFinished(hold.clone())
3760 });
3761 let publication_spawn = channel_spawn
3762 .as_ref()
3763 .or_else(|| prefix_binding.as_ref().map(|binding| &binding.registration));
3764 #[cfg(test)]
3765 if prefix_binding.is_some() {
3766 self.claims.pause_prefix_hello_for_test(7);
3767 }
3768 let _registration_publication = match publication_spawn {
3769 Some(spawn) => Some(match spawn.begin_publication() {
3770 Ok(guard) => guard,
3771 Err(code) => {
3772 return Ok(vec![control_error_frame(
3773 &frame,
3774 code,
3775 "registration ended before HELLO publication",
3776 )?])
3777 }
3778 }),
3779 None => None,
3780 };
3781 let (_prefix_live, _prefix_ownership) = if let Some(binding) = &prefix_binding {
3782 let live = self.claims.lock_prefix_binding(binding, connection_id);
3783 let ownership = self.supervisor.prefix_claim_publication(
3784 &binding.registration.module_id,
3785 &hello.manifest.module_id,
3786 );
3787 if live.is_none()
3788 || ownership.is_none()
3789 || !self.claim_primary_registered(binding.spawn, &binding.registration)
3790 || binding.prefix_module_id() != Some(hello.manifest.module_id.as_str())
3791 {
3792 return Ok(vec![control_error_frame(
3793 &frame,
3794 "claim_prefix_refused",
3795 "prefix authority ended before publication",
3796 )?]);
3797 }
3798 (live, ownership)
3799 } else {
3800 (None, None)
3801 };
3802 if let Some(spawn) = publication_spawn {
3803 self.forwarding
3804 .bind_spawn(connection_id, spawn)
3805 .map_err(RouterError::Forwarding)?;
3806 }
3807 let hello_authority = match (&channel_spawn, &prefix_binding) {
3817 (Some(spawn), _) => Some(spawn.authority.clone()),
3818 (None, None) => hello.launch_nonce.as_deref().and_then(|nonce| {
3819 self.supervisor
3820 .spawned_consumer_record(&hello.manifest.module_id, nonce)
3821 }),
3822 (None, Some(_)) => None,
3823 }
3824 .map(|record| HelloAuthority {
3825 record,
3826 requested: hello
3827 .manifest
3828 .provider_declarations
3829 .clone()
3830 .unwrap_or_default(),
3831 froze: Default::default(),
3832 });
3833 #[cfg(test)]
3834 if prefix_binding.is_some() {
3835 self.claims.pause_prefix_hello_for_test(8);
3836 }
3837 if swap_candidate {
3838 let module_id = hello.manifest.module_id.clone();
3839 let mut differing_declarations = Default::default();
3840 let reply = self.register_swap_candidate(
3841 connection_id,
3842 sink,
3843 &frame,
3844 hello.manifest,
3845 negotiated_ver,
3846 control_ops,
3847 hello_ack,
3848 hello_authority.as_ref(),
3849 &mut differing_declarations,
3850 #[cfg(windows)]
3851 hello.launch_nonce.as_deref(),
3852 )?;
3853 if let Some(spawn) = &channel_spawn {
3854 if self
3855 .registry
3856 .get_module_by_connection(connection_id)
3857 .ok()
3858 .flatten()
3859 .is_some()
3860 {
3861 if let Some(sink) = primary_sink {
3862 spawn.bind_primary_sink(sink);
3863 }
3864 spawn.hello_accepted();
3865 }
3866 } else if self
3867 .registry
3868 .get_module_by_connection(connection_id)
3869 .ok()
3870 .flatten()
3871 .is_some()
3872 {
3873 self.supervisor
3874 .retire_unused_registration_channel(&module_id, true);
3875 }
3876 if self
3877 .registry
3878 .get_module_by_connection(connection_id)
3879 .ok()
3880 .flatten()
3881 .is_some()
3882 {
3883 self.file_development_widening(
3884 hello_authority.map(|authority| authority.record),
3885 differing_declarations,
3886 );
3887 }
3888 return Ok(reply);
3889 }
3890 let registration = match self
3897 .registry
3898 .register_unpublished_verified_with_control_ops(
3899 hello.manifest,
3900 negotiated_ver,
3901 connection_id,
3902 control_ops,
3903 verification.nonce_verified,
3904 ) {
3905 Ok(registration) => registration,
3906 Err(RegistryError::DuplicateModuleId { module_id }) => {
3907 return Ok(vec![control_error_frame(
3908 &frame,
3909 "duplicate_module_id",
3910 format!(
3911 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
3912 ),
3913 )?])
3914 }
3915 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3916 return Ok(vec![control_error_frame(
3917 &frame,
3918 "invalid_module_id",
3919 err.to_string(),
3920 )?])
3921 }
3922 Err(err) => {
3923 return Ok(vec![control_error_frame(
3924 &frame,
3925 "registry_error",
3926 err.to_string(),
3927 )?])
3928 }
3929 };
3930 let differing_declarations =
3931 match self.admit_hello_authority(connection_id, hello_authority.as_ref()) {
3932 Ok(differing) => differing,
3933 Err(err) => {
3934 if matches!(
3935 self.rollback_unpublished_hello(connection_id),
3936 Ok(r) if !r.is_empty()
3937 ) {
3938 crate::supervise::notify_registration_release();
3939 }
3940 return Err(RouterError::Forwarding(err));
3941 }
3942 };
3943 #[cfg(test)]
3944 self.hold_between_registry_and_forwarding();
3945
3946 let mut reply = Vec::new();
3947 let publication = self
3950 .registry
3951 .publish_registration_after(connection_id, || {
3952 if let Some(sink) = sink {
3953 let concurrency = manifest_concurrency(®istration.manifest);
3969 self.forwarding.register_module_connection_acked(
3970 connection_id,
3971 registration.manifest.module_id.clone(),
3972 negotiated_ver,
3973 concurrency,
3974 sink,
3975 crate::forwarding::ModuleHelloAck {
3976 frame: hello_ack,
3977 verified_prefix_owner: verification.prefix_owner,
3978 },
3979 )?;
3980 } else {
3981 reply.push(hello_ack);
3984 }
3985 #[cfg(test)]
3986 self.hold_between_ack_and_publication();
3987 Ok(())
3988 })
3989 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3990 let published = match publication {
3991 Ok(published) => published,
3992 Err(err) => {
3993 if matches!(
4000 self.rollback_unpublished_hello(connection_id),
4001 Ok(r) if !r.is_empty()
4002 ) {
4003 crate::supervise::notify_registration_release();
4004 }
4005 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
4006 return Ok(vec![control_error_frame(
4007 &frame,
4008 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
4009 "invalid_hello"
4010 } else {
4011 forwarding_error_code(&err)
4012 },
4013 err.to_string(),
4014 )?]);
4015 }
4016 };
4017 if !published {
4018 debug!(
4019 module_id = %registration.manifest.module_id,
4020 connection_id = connection_id.get(),
4021 "registration was removed before it could be published"
4022 );
4023 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
4024 return Ok(reply);
4025 }
4026
4027 drop(_prefix_ownership);
4028 drop(_prefix_live);
4029
4030 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
4039 info!(
4040 module_id = %registration.manifest.module_id,
4041 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
4042 );
4043 }
4044 if registration.manifest.registration_channel == Some(false)
4045 && self
4046 .supervisor
4047 .module_is_gated(®istration.manifest.module_id)
4048 && self
4049 .config_runtime
4050 .as_ref()
4051 .and_then(|owner| owner.registration_declaration(®istration.manifest.module_id))
4052 .as_deref()
4053 == Some("declared")
4054 {
4055 warn!(module_id = %registration.manifest.module_id,
4056 "HELLO registration_channel is false despite the approved channel declaration; registration remains admitted");
4057 }
4058
4059 self.apply_registration_capabilities(®istration);
4060
4061 self.supervisor
4062 .record_activity_registration(®istration.manifest.module_id);
4063
4064 info!(
4065 module_id = %registration.manifest.module_id,
4066 module_version = %registration.manifest.module_version,
4067 negotiated_ver,
4068 routable_provider = manifest_provides_routable_role(®istration.manifest),
4069 connection_id = connection_id.get(),
4070 "module registered"
4071 );
4072
4073 if let Some(spawn) = &channel_spawn {
4074 if let Some(sink) = primary_sink {
4075 spawn.bind_primary_sink(sink);
4076 }
4077 spawn.hello_accepted();
4078 } else if !prefix_submodule {
4079 self.supervisor
4080 .retire_unused_registration_channel(®istration.manifest.module_id, false);
4081 }
4082 self.file_development_widening(
4083 hello_authority.map(|authority| authority.record),
4084 differing_declarations,
4085 );
4086 #[cfg(windows)]
4087 if let Some(nonce) = hello.launch_nonce.as_deref() {
4088 self.supervisor
4089 .program_hello_accepted(®istration.manifest.module_id, nonce);
4090 }
4091
4092 #[cfg(test)]
4093 if let Some(hold) = &self.hello_inside_publication_hold {
4094 hold.wait();
4095 hold.wait();
4096 }
4097
4098 Ok(reply)
4099 }
4100
4101 #[allow(clippy::too_many_arguments)]
4118 fn register_swap_candidate(
4119 &self,
4120 connection_id: ConnectionId,
4121 sink: Option<crate::FrameSink>,
4122 frame: &Frame,
4123 manifest: ModuleManifest,
4124 negotiated_ver: u8,
4125 control_ops: Vec<String>,
4126 hello_ack: Frame,
4127 hello_authority: Option<&HelloAuthority>,
4128 differing_declarations: &mut ProviderDeclarations,
4129 #[cfg(windows)] launch_nonce: Option<&str>,
4130 ) -> Result<Vec<Frame>, RouterError> {
4131 let module_id = manifest.module_id.clone();
4132 self.supervisor
4135 .with_swap_candidate_admission(&module_id, || {
4136 let reply = self.publish_swap_candidate(
4137 connection_id,
4138 sink,
4139 frame,
4140 manifest,
4141 negotiated_ver,
4142 control_ops,
4143 hello_ack,
4144 hello_authority,
4145 differing_declarations,
4146 #[cfg(windows)]
4147 launch_nonce,
4148 );
4149 let admitted = self
4150 .registry
4151 .get_candidate(&module_id)
4152 .ok()
4153 .flatten()
4154 .is_some_and(|candidate| candidate.connection_id == connection_id);
4155 (reply, admitted)
4156 })
4157 .unwrap_or_else(|| {
4158 Ok(vec![control_error_frame(
4159 frame,
4160 "swap_token_invalid",
4161 "swap ended before candidate HELLO publication",
4162 )?])
4163 })
4164 }
4165
4166 #[allow(clippy::too_many_arguments)]
4167 fn publish_swap_candidate(
4168 &self,
4169 connection_id: ConnectionId,
4170 sink: Option<crate::FrameSink>,
4171 frame: &Frame,
4172 manifest: ModuleManifest,
4173 negotiated_ver: u8,
4174 control_ops: Vec<String>,
4175 hello_ack: Frame,
4176 hello_authority: Option<&HelloAuthority>,
4177 differing_declarations: &mut ProviderDeclarations,
4178 #[cfg(windows)] launch_nonce: Option<&str>,
4179 ) -> Result<Vec<Frame>, RouterError> {
4180 let module_id = manifest.module_id.clone();
4181 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
4182 manifest,
4183 negotiated_ver,
4184 connection_id,
4185 control_ops,
4186 ) {
4187 Ok(registration) => registration,
4188 Err(RegistryError::DuplicateModuleId { module_id }) => {
4189 return Ok(vec![control_error_frame(
4190 frame,
4191 "duplicate_module_id",
4192 format!(
4193 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
4194 ),
4195 )?])
4196 }
4197 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
4198 return Ok(vec![control_error_frame(
4199 frame,
4200 "invalid_module_id",
4201 err.to_string(),
4202 )?])
4203 }
4204 Err(err) => {
4205 return Ok(vec![control_error_frame(
4206 frame,
4207 "registry_error",
4208 err.to_string(),
4209 )?])
4210 }
4211 };
4212 *differing_declarations = match self.admit_hello_authority(connection_id, hello_authority) {
4213 Ok(differing) => differing,
4214 Err(err) => {
4215 if matches!(
4216 self.rollback_unpublished_hello(connection_id),
4217 Ok(r) if !r.is_empty()
4218 ) {
4219 crate::supervise::notify_registration_release();
4220 }
4221 return Err(RouterError::Forwarding(err));
4222 }
4223 };
4224 #[cfg(test)]
4225 self.hold_between_registry_and_forwarding();
4226 let mut reply = Vec::new();
4227 let publication = self
4228 .registry
4229 .publish_registration_after(connection_id, || {
4230 if let Some(sink) = sink {
4231 let concurrency = manifest_concurrency(®istration.manifest);
4235 self.forwarding.register_candidate_module_connection_acked(
4236 connection_id,
4237 module_id.clone(),
4238 negotiated_ver,
4239 concurrency,
4240 sink,
4241 hello_ack,
4242 )?;
4243 } else {
4244 reply.push(hello_ack);
4245 }
4246 #[cfg(test)]
4247 self.hold_between_ack_and_publication();
4248 Ok(())
4249 })
4250 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4251 let published = match publication {
4252 Ok(published) => published,
4253 Err(err) => {
4254 if matches!(
4255 self.rollback_unpublished_hello(connection_id),
4256 Ok(r) if !r.is_empty()
4257 ) {
4258 crate::supervise::notify_registration_release();
4259 }
4260 self.refuse_hello_authority(connection_id, hello_authority);
4261 return Ok(vec![control_error_frame(
4262 frame,
4263 forwarding_error_code(&err),
4264 err.to_string(),
4265 )?]);
4266 }
4267 };
4268 if !published {
4269 debug!(
4270 module_id = %module_id,
4271 connection_id = connection_id.get(),
4272 "swap candidate registration was removed before it could be published"
4273 );
4274 self.refuse_hello_authority(connection_id, hello_authority);
4275 return Ok(reply);
4276 }
4277 info!(
4278 module_id = %module_id,
4279 module_version = %registration.manifest.module_version,
4280 negotiated_ver,
4281 ready = registration.ready,
4282 connection_id = connection_id.get(),
4283 "swap candidate registered; not routable until cutover"
4284 );
4285 #[cfg(windows)]
4286 if let Some(nonce) = launch_nonce {
4287 self.supervisor.program_hello_accepted(&module_id, nonce);
4288 }
4289 Ok(reply)
4290 }
4291
4292 fn admit_hello_authority(
4298 &self,
4299 connection: ConnectionId,
4300 authority: Option<&HelloAuthority>,
4301 ) -> Result<ProviderDeclarations, ForwardingError> {
4302 let Some(authority) = authority else {
4303 return Ok(ProviderDeclarations::new());
4304 };
4305 let froze = authority
4306 .record
4307 .freeze_declarations(authority.requested.clone());
4308 authority
4309 .froze
4310 .store(froze.is_some(), std::sync::atomic::Ordering::Relaxed);
4311 self.forwarding
4312 .bind_target_authority(connection, authority.record.clone())
4313 .inspect_err(|_| self.refuse_hello_authority(connection, Some(authority)))?;
4314 Ok(froze.unwrap_or_default())
4315 }
4316
4317 fn refuse_hello_authority(&self, connection: ConnectionId, authority: Option<&HelloAuthority>) {
4322 let Some(authority) = authority else {
4323 return;
4324 };
4325 if let Err(err) = self.forwarding.unbind_target_authority(connection) {
4326 warn!(error = %err, "failed to unbind a refused HELLO from its spawn");
4327 }
4328 if authority
4329 .froze
4330 .swap(false, std::sync::atomic::Ordering::Relaxed)
4331 {
4332 authority.record.unfreeze_refused_declarations();
4333 }
4334 }
4335
4336 fn build_hello_ack(
4337 &self,
4338 frame: &Frame,
4339 negotiated_ver: u8,
4340 module_id: &str,
4341 ) -> Result<Frame, RouterError> {
4342 let ack = ModuleHelloAckBody {
4343 negotiated_ver,
4344 subc_ops: module_subc_ops(),
4345 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
4346 storage: self
4347 .storage_config
4348 .as_ref()
4349 .map(|cfg| cfg.descriptor_for(module_id)),
4350 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4351 };
4352 let body = serde_json::to_vec(&ack).map_err(|err| {
4353 RouterError::backend(
4354 0,
4355 frame.header.corr,
4356 format!("failed to encode HELLO_ACK: {err}"),
4357 )
4358 })?;
4359
4360 Frame::build_with_version(
4361 negotiated_ver,
4362 FrameType::HelloAck,
4363 control_flags(),
4364 0,
4365 0,
4366 frame.header.corr,
4367 body,
4368 )
4369 .map_err(RouterError::FrameBuild)
4370 }
4371
4372 async fn handle_client_control_request(
4373 &self,
4374 ctx: &RouteCtx,
4375 frame: Frame,
4376 request: ClientControlRequest,
4377 route_open_timings: &mut Option<RouteOpenTimings>,
4378 ) -> Result<Vec<Frame>, RouterError> {
4379 if let Some(owner) = &self.config_runtime {
4380 let module = match &request {
4381 ClientControlRequest::SupervisorRestart { module_id, .. }
4382 | ClientControlRequest::SupervisorReload { module_id }
4383 | ClientControlRequest::SupervisorSwap { module_id, .. }
4384 | ClientControlRequest::SupervisorSetEnabled {
4385 module_id,
4386 enabled: true,
4387 } => Some(module_id.as_str()),
4388 _ => None,
4389 };
4390 let rescan = matches!(
4391 &request,
4392 ClientControlRequest::SupervisorRescan { preview: false }
4393 | ClientControlRequest::SupervisorRescanChecked { preview: false, .. }
4394 );
4395 let held = if module.is_some() || rescan {
4396 let module = module.map(str::to_owned);
4397 owner
4398 .on_blocking(move |owner| owner.checked_admission_hold(module.as_deref()))
4399 .await
4400 } else {
4401 Ok(())
4402 };
4403 if let Err(error) = held {
4404 if error.code != "config_delivery_deferred" {
4405 let code = if error.code == "store_tampered" {
4406 "store_tampered"
4407 } else {
4408 "config_gated"
4409 };
4410 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
4411 }
4412 }
4413 }
4414 if let Some(operation) = privileged_operation(&request) {
4415 let caller = self.resolve_caller(ctx.connection_id).await;
4416 let op = client_control_request_op(&request);
4417 let text = prompt_text(operation, &caller);
4418 match self.privileged_operator_prompt {
4419 PrivilegedOperatorPrompt::Log => {
4420 info!(target: "subc_daemon::privileged_prompt", operation = op,
4421 target = ?operation, caller = %caller.description(), summary = %text,
4422 "privileged request would prompt the operator");
4423 }
4424 PrivilegedOperatorPrompt::Enforce => {
4425 let key =
4426 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
4427 let stamp = self
4428 .operator_apps
4429 .lock()
4430 .unwrap_or_else(|p| p.into_inner())
4431 .get(&ctx.connection_id)
4432 .cloned();
4433 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
4434 &self.operator_app_policy,
4435 &self.launch_table,
4436 stamp.as_ref(),
4437 )
4438 .then(|| self.shortcut_context(stamp, None, None));
4439 let outcome = self
4440 .forwarding
4441 .operator_confirms()
4442 .approve_privileged_with_shortcut(
4443 ctx,
4444 &frame,
4445 op,
4446 text,
4447 caller.description(),
4448 key,
4449 shortcut,
4450 )
4451 .await;
4452 if outcome.code != "confirmed" {
4453 return Ok(vec![outcome.frame(&frame)]);
4454 }
4455 }
4456 }
4457 }
4458 match request {
4459 ClientControlRequest::DaemonShutdownIntent { intent } => {
4460 if intent.as_str() != Some("restart") {
4461 return Ok(vec![control_error_frame(
4462 &frame,
4463 "invalid_params",
4464 "intent must be 'restart'",
4465 )?]);
4466 }
4467 if let Some(owner) = &self.config_runtime {
4472 match owner
4473 .on_blocking(|owner| owner.assign_daemon_restart())
4474 .await
4475 {
4476 Ok(Some(generation)) => {
4477 info!(
4478 generation,
4479 "daemon restart will warm up approved generation"
4480 )
4481 }
4482 Ok(None) => {}
4483 Err(error) => warn!(
4484 %error,
4485 "daemon restart intent not recorded; the next start boots the last active generation"
4486 ),
4487 }
4488 }
4489 let response = ClientControlResponse::DaemonShutdownIntent {
4490 intent: "restart".to_owned(),
4491 expires_at_ms: self.supervisor.announce_daemon_restart(),
4492 };
4493 Ok(vec![control_response_body_frame(
4494 &frame,
4495 &response,
4496 "ClientControlResponse::DaemonShutdownIntent",
4497 )?])
4498 }
4499 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
4500 ClientControlRequest::CatalogList { module_id } => {
4501 self.handle_catalog_list(frame, module_id)
4502 }
4503 ClientControlRequest::RouteOpen {
4504 target,
4505 identity,
4506 consumer_identity,
4507 consumer_capabilities,
4508 role_versions,
4509 busy_refusals,
4510 admission_facts,
4511 scope,
4512 ..
4513 } => {
4514 let extensions = match serde_json::from_slice::<subc_control::RouteOpenRequestExt>(
4515 &frame.body,
4516 ) {
4517 Ok(extensions) => extensions,
4518 Err(err) => {
4519 return Ok(vec![control_error_frame(
4520 &frame,
4521 "invalid_control_body",
4522 format!("malformed route.open extensions: {err}"),
4523 )?]);
4524 }
4525 };
4526 self.handle_route_open(
4527 ctx,
4528 frame,
4529 RouteOpenRequest {
4530 target,
4531 identity,
4532 consumer_identity,
4533 consumer_capabilities,
4534 role_versions,
4535 busy_refusals,
4536 admission_facts,
4537 scope,
4538 accepts_close_reasons: extensions.accepts_close_reasons,
4539 },
4540 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
4541 )
4542 .await
4543 }
4544 ClientControlRequest::RoutePoll {
4545 route_channel,
4546 route_epoch,
4547 kind,
4548 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
4549 ClientControlRequest::SupervisorList {} => {
4550 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
4551 self.agent_declarations.refuse(
4552 self,
4553 ctx,
4554 control_error_frame(
4555 &frame,
4556 "busy",
4557 "supervisor list work is at capacity; retry",
4558 )?,
4559 );
4560 return Ok(Vec::new());
4561 };
4562 let handler = self.clone();
4564 let sink = ctx.egress.clone();
4565 tokio::spawn(
4566 async move {
4567 let _permit = permit;
4568 let result = handler.handle_supervisor_list(frame.clone()).await;
4569 let replies = match result {
4570 Ok(replies) => replies,
4571 Err(error) => {
4572 control_error_frame(&frame, "backend_error", error.to_string())
4573 .into_iter()
4574 .collect()
4575 }
4576 };
4577 for reply in replies {
4578 if !matches!(
4579 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
4580 .await,
4581 Ok(Ok(()))
4582 ) {
4583 break;
4584 }
4585 }
4586 }
4587 .with_current_subscriber(),
4588 );
4589 Ok(Vec::new())
4590 }
4591 ClientControlRequest::SupervisorSpawnSnapshot {} => {
4592 self.handle_supervisor_spawn_snapshot(frame)
4593 }
4594 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
4595 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
4596 }
4597 ClientControlRequest::SupervisorRestart {
4598 module_id,
4599 drain_timeout_ms,
4600 } => {
4601 self.log_supervisor_request_received(
4602 ctx,
4603 frame.header.corr,
4604 ops::SUPERVISOR_RESTART,
4605 Some(&module_id),
4606 None,
4607 )?;
4608 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4609 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
4610 .await
4611 }
4612 ClientControlRequest::SupervisorSwap {
4613 module_id,
4614 ready_timeout_ms,
4615 } => {
4616 self.log_supervisor_request_received(
4617 ctx,
4618 frame.header.corr,
4619 ops::SUPERVISOR_SWAP,
4620 Some(&module_id),
4621 None,
4622 )?;
4623 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4624 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
4625 .await
4626 }
4627 ClientControlRequest::SupervisorReload { module_id } => {
4628 self.log_supervisor_request_received(
4629 ctx,
4630 frame.header.corr,
4631 ops::SUPERVISOR_RELOAD,
4632 Some(&module_id),
4633 None,
4634 )?;
4635 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4636 self.handle_supervisor_reload(frame, module_id, requester)
4637 .await
4638 }
4639 ClientControlRequest::SupervisorRescan { preview } => {
4640 if !preview {
4641 self.log_supervisor_request_received(
4642 ctx,
4643 frame.header.corr,
4644 ops::SUPERVISOR_RESCAN,
4645 None,
4646 None,
4647 )?;
4648 }
4649 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4650 let operation = self.handle_supervisor_rescan(
4651 frame,
4652 if preview {
4653 RescanMode::Preview
4654 } else {
4655 RescanMode::Execute
4656 },
4657 requester,
4658 );
4659 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
4660 .await
4661 }
4662 ClientControlRequest::SupervisorRescanChecked {
4663 preview,
4664 candidate,
4665 expect,
4666 } => {
4667 self.observe_control_wait(
4668 ctx,
4669 None,
4670 ops::SUPERVISOR_RESCAN,
4671 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
4672 )
4673 .await
4674 }
4675 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
4676 self.log_supervisor_request_received(
4677 ctx,
4678 frame.header.corr,
4679 ops::SUPERVISOR_RELEASE_RESERVED,
4680 Some(&module_id),
4681 None,
4682 )?;
4683 self.handle_supervisor_release_reserved(frame, module_id)
4684 .await
4685 }
4686 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
4687 self.log_supervisor_request_received(
4688 ctx,
4689 frame.header.corr,
4690 ops::SUPERVISOR_SET_ENABLED,
4691 Some(&module_id),
4692 Some(enabled),
4693 )?;
4694 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4695 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
4696 .await
4697 }
4698 ClientControlRequest::SupervisorHealthProbe { module_id } => {
4699 self.handle_supervisor_health_probe(ctx, frame, module_id)
4700 .await
4701 }
4702 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
4703 ClientControlRequest::SupervisorRoutes { module_id } => {
4704 self.handle_supervisor_routes(frame, module_id)
4705 }
4706 ClientControlRequest::SupervisorProvenance { module_id } => {
4707 self.handle_supervisor_provenance(ctx, frame, module_id)
4708 .await
4709 }
4710 ClientControlRequest::SupervisorStderrTail {
4711 module_id,
4712 max_lines,
4713 max_bytes,
4714 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
4715 ClientControlRequest::SupervisorTerminals { module_id } => {
4716 self.handle_supervisor_terminals(ctx, frame, module_id)
4717 .await
4718 }
4719 }
4720 }
4721
4722 fn log_supervisor_request_received(
4723 &self,
4724 ctx: &RouteCtx,
4725 corr: u64,
4726 op: &'static str,
4727 module_id: Option<&str>,
4728 enabled: Option<bool>,
4729 ) -> Result<(), RouterError> {
4730 let caller = self.supervisor_requester(ctx, corr)?;
4731 let caller = principal_label(&caller);
4732 info!(
4735 target: "control",
4736 op,
4737 module_id,
4738 enabled,
4739 connection_id = ctx.connection_id.get(),
4740 caller = %caller,
4741 "supervisor request received"
4742 );
4743 Ok(())
4744 }
4745
4746 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
4747 Ok(self
4748 .registry
4749 .get_module_by_connection(ctx.connection_id)
4750 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4751 .map(|registration| Principal::Reserved {
4752 module_id: registration.manifest.module_id,
4753 })
4754 .unwrap_or(Principal::Direct))
4755 }
4756
4757 fn handle_module_control_request(
4758 &self,
4759 connection_id: ConnectionId,
4760 frame: Frame,
4761 request: ModuleControlRequestFromModule,
4762 ) -> Result<Vec<Frame>, RouterError> {
4763 match request {
4764 ModuleControlRequestFromModule::RouteAuthorize {
4765 principal,
4766 scope,
4767 target,
4768 expected,
4769 } => self.handle_route_authorize(
4770 connection_id,
4771 frame,
4772 principal,
4773 scope,
4774 target,
4775 expected,
4776 ),
4777 ModuleControlRequestFromModule::CatalogUpdate {
4778 provides,
4779 capabilities,
4780 ready,
4781 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
4782 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
4783 self.handle_catalog_update_role_declarations(
4784 connection_id,
4785 frame,
4786 role_declarations,
4787 )
4788 }
4789 ModuleControlRequestFromModule::LiveRoots {} => {
4790 let registered = self
4791 .registry
4792 .get_module_by_connection(connection_id)
4793 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4794 let Some(registration) = registered else {
4795 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
4796 };
4797 let response = self
4798 .forwarding
4799 .live_roots(®istration.manifest.module_id)
4800 .map_err(RouterError::Forwarding)?;
4801 Ok(vec![control_response_body_frame(
4802 &frame,
4803 &response,
4804 "ModuleControlResponseToModule::LiveRoots",
4805 )?])
4806 }
4807 ModuleControlRequestFromModule::AgentCommands {
4808 started,
4809 finished,
4810 known,
4811 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
4812 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
4813 self.handle_scope_sync(connection_id, frame, generation, scopes)
4814 }
4815 ModuleControlRequestFromModule::ScopeApply {
4816 generation,
4817 upsert,
4818 end,
4819 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
4820 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
4821 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
4822 }
4823 _ => Ok(vec![control_error_frame(
4824 &frame,
4825 "unsupported_control_frame",
4826 "unsupported module control operation",
4827 )?]),
4828 }
4829 }
4830
4831 fn handle_scope_served_by(
4832 &self,
4833 connection_id: ConnectionId,
4834 frame: Frame,
4835 ) -> Result<Vec<Frame>, RouterError> {
4836 let registered = self
4837 .registry
4838 .get_module_by_connection(connection_id)
4839 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4840 let Some(registered) = registered else {
4841 return Ok(vec![control_error_frame(
4842 &frame,
4843 "not_registered",
4844 "scope.served_by requires an active module registration owned by this connection",
4845 )?]);
4846 };
4847 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
4848 {
4849 Ok(request) => request,
4850 Err(err) => {
4851 return Ok(vec![control_error_frame(
4852 &frame,
4853 "invalid_control_body",
4854 format!("malformed scope.served_by body: {err}"),
4855 )?])
4856 }
4857 };
4858 let answer = self
4859 .scopes
4860 .read()
4861 .unwrap_or_else(|poisoned| poisoned.into_inner())
4862 .served_by(
4863 ®istered.manifest.module_id,
4864 &request.scope_ref,
4865 request.scope_epoch,
4866 &request.module_id,
4867 );
4868 let reply = subc_control::ScopeServedByReply::new(
4869 answer,
4870 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4871 );
4872 Ok(vec![control_response_body_frame(
4873 &frame,
4874 &reply,
4875 "ScopeServedByReply",
4876 )?])
4877 }
4878
4879 fn handle_operator_confirm(
4880 &self,
4881 ctx: &RouteCtx,
4882 frame: Frame,
4883 ) -> Result<Vec<Frame>, RouterError> {
4884 use crate::operator_confirm::{audit, Outcome};
4885 let registration = self
4886 .registry
4887 .get_module_by_connection(ctx.connection_id)
4888 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4889 let Some(registration) = registration else {
4890 let outcome = Outcome::refusal("not_registered");
4891 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4892 return Ok(vec![outcome.frame(&frame)]);
4893 };
4894 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
4895 Ok(request) => request,
4896 Err(_) => {
4897 let outcome = Outcome::refusal("invalid_control_body");
4898 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4899 return Ok(vec![outcome.frame(&frame)]);
4900 }
4901 };
4902 let declared = matches!(
4903 request.kind,
4904 Some(subc_protocol::session::OperatorConfirmKind::Operation)
4905 ) && request.operation.as_ref().is_some_and(|operation| {
4906 registration
4907 .manifest
4908 .confirm_operations
4909 .as_ref()
4910 .is_some_and(|operations| operations.contains(operation))
4911 });
4912 let module_id = registration.manifest.module_id;
4913 let nonce = self
4917 .hello_launch_nonces
4918 .lock()
4919 .unwrap_or_else(|p| p.into_inner())
4920 .nonce(ctx.connection_id)
4921 .map(str::to_owned);
4922 let nonce_proven = self
4925 .confirm_channel_proven(ctx.connection_id, &module_id)
4926 .unwrap_or_else(|| {
4927 nonce.as_deref().is_some_and(|nonce| {
4928 self.supervisor
4929 .spawned_consumer_authorized(&module_id, nonce)
4930 })
4931 });
4932 let confirms = self.forwarding.operator_confirms();
4933 self.forwarding
4934 .with_operator_route(
4935 ctx.connection_id,
4936 request.route_channel,
4937 request.route_epoch,
4938 |binding| {
4939 let shortcut = binding
4940 .filter(|binding| {
4941 nonce_proven
4942 && crate::operator_confirm::valid_summary(&request.summary)
4943 && declared
4944 && crate::operator_confirm::ShortcutContext::route_eligible(
4945 &self.operator_app_policy,
4946 &self.launch_table,
4947 binding,
4948 )
4949 })
4950 .map(|binding| {
4951 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
4952 &self.operator_app_policy,
4953 binding,
4954 );
4955 let stamp = if phone.is_some() {
4956 None
4957 } else {
4958 binding.operator_app.clone()
4959 };
4960 self.shortcut_context(
4961 stamp,
4962 phone,
4963 Some((request.route_channel, request.route_epoch)),
4964 )
4965 });
4966 confirms.admit_with_shortcut(
4967 ctx,
4968 frame,
4969 module_id,
4970 nonce_proven,
4971 request,
4972 binding,
4973 shortcut,
4974 )
4975 },
4976 )
4977 .map_err(RouterError::Forwarding)
4978 }
4979
4980 fn shortcut_context(
4981 &self,
4982 stamp: Option<crate::operator_apps::OperatorAppStamp>,
4983 phone_fingerprint: Option<String>,
4984 route: Option<(u16, u32)>,
4985 ) -> crate::operator_confirm::ShortcutContext {
4986 crate::operator_confirm::ShortcutContext {
4987 ledger: self.activity.clone(),
4988 forwarding: self.forwarding.clone(),
4989 launches: self.launch_table.clone(),
4990 stamp,
4991 phone_fingerprint,
4992 route,
4993 connections: self.operator_apps.clone(),
4994 #[cfg(test)]
4995 hooks: Default::default(),
4996 }
4997 }
4998
4999 #[cfg(any(test, feature = "test-support"))]
5001 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
5002 self.operator_app_policy.phone_enabled = enabled;
5003 self
5004 }
5005
5006 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
5009 let mut scopes = ScopeRecords::from(scopes);
5010 for (index, record) in scopes.records.iter_mut().enumerate() {
5011 if let Some(refusal) = Self::canonicalize_workdir(record) {
5012 scopes.refusals.insert(index, refusal);
5013 }
5014 }
5015 scopes
5016 }
5017
5018 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
5019 let workdir = record.workdir.as_mut()?;
5020 let path = std::path::Path::new(workdir);
5021 if !path.is_absolute() {
5022 return Some((
5023 error_codes::INVALID_WORKDIR,
5024 "workdir must be absolute".to_string(),
5025 ));
5026 }
5027 match ProjectRootId::from_path_allowing_missing(path) {
5028 Ok(root) => match root.as_path().to_str() {
5029 Some(canonical) => {
5030 *workdir = canonical.to_string();
5031 None
5032 }
5033 None => Some((
5034 error_codes::INVALID_WORKDIR,
5035 "canonical workdir is not UTF-8".to_string(),
5036 )),
5037 },
5038 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
5039 }
5040 }
5041
5042 fn handle_scope_sync(
5046 &self,
5047 connection_id: ConnectionId,
5048 frame: Frame,
5049 generation: u64,
5050 scopes: Vec<ScopeRecord>,
5051 ) -> Result<Vec<Frame>, RouterError> {
5052 self.handle_scope_change(connection_id, frame, generation, scopes, None)
5053 }
5054
5055 fn handle_scope_change(
5056 &self,
5057 connection_id: ConnectionId,
5058 frame: Frame,
5059 generation: u64,
5060 scopes: Vec<ScopeRecord>,
5061 end: Option<Vec<ScopeEnd>>,
5062 ) -> Result<Vec<Frame>, RouterError> {
5063 let is_apply = end.is_some();
5064 let Some(registration) = self
5065 .registry
5066 .get_module_by_connection(connection_id)
5067 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5068 else {
5069 return Ok(vec![control_error_frame(
5070 &frame,
5071 "not_registered",
5072 "scope.sync and scope.apply require an active module registration owned by this connection",
5073 )?]);
5074 };
5075 let owner = registration.manifest.module_id;
5076 let scopes = Self::prepare_scope_records(scopes);
5077 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
5078 let is_current_launch = |connection: ConnectionId| {
5079 self.connection_is_current_launch(&owner, connection, current_nonce.as_deref())
5080 };
5081 let mut table = self
5086 .scopes
5087 .write()
5088 .unwrap_or_else(|poisoned| poisoned.into_inner());
5089 let outcome = match end {
5090 Some(end) => table.apply(
5091 &owner,
5092 connection_id,
5093 is_current_launch,
5094 generation,
5095 scopes,
5096 end,
5097 ),
5098 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
5099 };
5100 let drained = match &outcome {
5101 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
5102 Err(_) => Vec::new(),
5103 };
5104 let has_deadlines = outcome.is_ok() && table.has_deadlines();
5105 drop(table);
5106 if has_deadlines {
5107 self.scope_deadline_added.notify_one();
5108 }
5109 match outcome {
5110 Ok(applied) => {
5111 let counts = ScopeOutcomeCounts::of(&applied.results);
5112 info!(
5113 owner = %owner,
5114 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5115 generation,
5116 records = applied.results.len(),
5117 created = counts.created,
5118 replaced = counts.replaced,
5119 updated = counts.updated,
5120 unchanged = counts.unchanged,
5121 refused = counts.refused,
5122 ended = applied.ended.len(),
5123 tag_changes = applied.tag_changes.len(),
5124 routes_closed = drained.len(),
5125 "scope change accepted"
5126 );
5127 for refused in applied
5133 .results
5134 .iter()
5135 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
5136 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
5137 {
5138 warn!(
5139 owner = %owner,
5140 generation,
5141 scope_ref = %refused.scope_ref,
5142 scope_epoch = refused.scope_epoch,
5143 code = refused.code.as_deref().unwrap_or(""),
5144 "scope record refused"
5145 );
5146 }
5147 self.close_scope_drained_routes(drained);
5148 let response = if is_apply {
5149 ModuleControlResponseToModule::ScopeApply {
5150 generation,
5151 results: applied.results,
5152 end_results: applied.end_results,
5153 ended: applied.ended,
5154 }
5155 } else {
5156 ModuleControlResponseToModule::ScopeSync {
5157 generation,
5158 results: applied.results,
5159 ended: applied.ended,
5160 }
5161 };
5162 Ok(vec![control_response_body_frame(
5163 &frame,
5164 &response,
5165 "scope change response",
5166 )?])
5167 }
5168 Err(refusal) => {
5169 info!(
5170 owner = %owner,
5171 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5172 generation,
5173 code = refusal.code,
5174 "scope change refused"
5175 );
5176 Ok(vec![control_error_frame(
5177 &frame,
5178 refusal.code,
5179 refusal.message,
5180 )?])
5181 }
5182 }
5183 }
5184
5185 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
5187 scope_expiry::ScopeExpiryContext::new(self).close_scope_drained_routes(drained);
5188 }
5189
5190 fn handle_route_authorize(
5196 &self,
5197 connection_id: ConnectionId,
5198 frame: Frame,
5199 principal: Principal,
5200 scope: subc_protocol::scope::RouteAuthorizeScope,
5201 target: String,
5202 expected: subc_protocol::scope::RouteAuthorizeExpected,
5203 ) -> Result<Vec<Frame>, RouterError> {
5204 use crate::scopes::admission::{admit_route, AdmissionMode};
5205 use subc_protocol::scope::{
5206 RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
5207 };
5208 let registry = self
5209 .registry
5210 .route_admission_snapshot()
5211 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5212 let owner_configured = match &scope.owner {
5213 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5214 _ => false,
5215 };
5216 let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
5217 let answer = {
5218 let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
5219 let permitted = registry
5220 .iter()
5221 .find(|r| r.connection_id == connection_id)
5222 .is_some_and(|r| {
5223 scopes.is_federated_owner(&Principal::Reserved {
5224 module_id: r.manifest.module_id.clone(),
5225 }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
5226 caps.provides
5227 .iter()
5228 .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
5229 })
5230 });
5231 if !permitted {
5232 drop(scopes);
5233 return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
5234 "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
5235 }
5236 admit_route(
5237 &scopes,
5238 ®istry,
5239 &principal,
5240 &target,
5241 Some(&selector),
5242 owner_configured,
5243 AdmissionMode::Authorize(&expected),
5244 )
5245 };
5246 let response = match answer {
5247 Ok(admission) => {
5248 let mut response = RouteAuthorizeResponse::new(
5249 RouteAuthorizeOutcome::Authorized,
5250 "route admission authorized",
5251 );
5252 response.content_version = admission.content_version;
5253 response
5254 }
5255 Err(refusal) => refusal.response(),
5256 };
5257 Ok(vec![control_response_body_frame(
5258 &frame,
5259 &response,
5260 "RouteAuthorizeResponse",
5261 )?])
5262 }
5263
5264 fn handle_scope_describe(
5267 &self,
5268 connection_id: ConnectionId,
5269 frame: Frame,
5270 owner: Principal,
5271 scope_ref: String,
5272 ) -> Result<Vec<Frame>, RouterError> {
5273 let registered = self
5274 .registry
5275 .get_module_by_connection(connection_id)
5276 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5277 let Some(registered) = registered else {
5278 return Ok(vec![control_error_frame(
5279 &frame,
5280 "not_registered",
5281 "scope.describe requires an active module registration owned by this connection",
5282 )?]);
5283 };
5284 let scopes = self
5285 .scopes
5286 .read()
5287 .unwrap_or_else(|poisoned| poisoned.into_inner());
5288 if !scopes.reader_permitted(&owner, registered.manifest.capabilities.as_ref()) {
5289 return Ok(vec![control_error_frame(
5290 &frame,
5291 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
5292 "scope.describe of the federated owner requires scope-federated/v1",
5293 )?]);
5294 }
5295 let mut description = scopes.describe(&owner, &scope_ref);
5296 drop(scopes);
5297 if owner
5298 != (Principal::Reserved {
5299 module_id: registered.manifest.module_id.clone(),
5300 })
5301 {
5302 if let Some(facts) = description
5303 .stamp
5304 .as_mut()
5305 .and_then(|stamp| stamp.attributes.federated.as_mut())
5306 {
5307 facts.stamp_required = false;
5308 }
5309 }
5310 if !registered
5313 .manifest
5314 .capabilities
5315 .as_ref()
5316 .is_some_and(|caps| {
5317 caps.provides
5318 .iter()
5319 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5320 })
5321 {
5322 if let Some(stamp) = description.stamp.as_mut() {
5323 stamp.attributes.on_behalf_of_agent = None;
5324 }
5325 }
5326 if !registered
5327 .manifest
5328 .capabilities
5329 .as_ref()
5330 .is_some_and(|caps| {
5331 caps.provides
5332 .iter()
5333 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5334 })
5335 {
5336 if let Some(stamp) = description.stamp.as_mut() {
5337 stamp.attributes.flow_version = None;
5338 stamp.attributes.flow_code_hash = None;
5339 stamp.attributes.flow_install_id = None;
5340 }
5341 }
5342 let owner_configured = match &owner {
5343 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5348 _ => false,
5349 };
5350 let response = ModuleControlResponseToModule::ScopeDescribe {
5351 status: description.status,
5352 scope_epoch: description.scope_epoch,
5353 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
5354 owner_synced: description.owner_synced,
5355 owner_configured,
5356 scope: description.stamp,
5357 };
5358 Ok(vec![control_response_body_frame(
5359 &frame,
5360 &response,
5361 "ModuleControlResponseToModule::ScopeDescribe",
5362 )?])
5363 }
5364
5365 fn handle_catalog_update_role_declarations(
5366 &self,
5367 connection_id: ConnectionId,
5368 frame: Frame,
5369 role_declarations: serde_json::Value,
5370 ) -> Result<Vec<Frame>, RouterError> {
5371 if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
5372 return Ok(vec![control_error_frame(
5373 &frame,
5374 "invalid_role_declarations",
5375 err.to_string(),
5376 )?]);
5377 }
5378 let declarations = serde_json::from_value(role_declarations)
5379 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5380 if self
5381 .registry
5382 .replace_role_declarations_for_connection(connection_id, declarations)
5383 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5384 .is_none()
5385 {
5386 return Ok(vec![control_error_frame(
5387 &frame,
5388 "not_registered",
5389 "catalog.update_role_declarations requires an active module registration owned by this connection",
5390 )?]);
5391 }
5392 control_response_body_frame(
5393 &frame,
5394 &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
5395 "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
5396 )
5397 .map(|frame| vec![frame])
5398 }
5399
5400 fn handle_catalog_update(
5401 &self,
5402 connection_id: ConnectionId,
5403 frame: Frame,
5404 provides: Vec<ProviderRole>,
5405 capabilities: Option<CapabilityDeclarations>,
5406 ready: Option<bool>,
5407 ) -> Result<Vec<Frame>, RouterError> {
5408 self.refresh_capability_requirements();
5409 let Some(registration) = self
5410 .registry
5411 .get_module_by_connection(connection_id)
5412 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5413 else {
5414 return Ok(vec![control_error_frame(
5415 &frame,
5416 "not_registered",
5417 "catalog.update requires an active module registration owned by this connection",
5418 )?]);
5419 };
5420
5421 if let Some(message) =
5422 catalog_update_frozen_field_message(®istration.manifest, &provides)
5423 {
5424 return Ok(vec![control_error_frame(
5425 &frame,
5426 "catalog_update_frozen_field",
5427 message,
5428 )?]);
5429 }
5430
5431 let mut candidate = registration.manifest.clone();
5432 candidate.provides = provides.clone();
5433 candidate.capabilities = capabilities
5434 .clone()
5435 .or_else(|| registration.manifest.capabilities.clone());
5436 if let Some(rich_ui) = &candidate.rich_ui {
5437 if let Err(err) = rich_ui.validate() {
5438 return Ok(vec![control_error_frame(
5439 &frame,
5440 "invalid_manifest",
5441 err.to_string(),
5442 )?]);
5443 }
5444 }
5445 if let Err(err) = candidate.validate_capability_grammar() {
5446 return Ok(vec![control_error_frame(
5447 &frame,
5448 "invalid_capability_grammar",
5449 err.to_string(),
5450 )?]);
5451 }
5452
5453 let mut conflicts = self
5456 .capability_evaluator
5457 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
5458 if let Some(conflict) = conflicts.first() {
5459 let message = format!(
5460 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
5461 conflict.capability, conflict.claimants[0], candidate.module_id
5462 );
5463 for conflict in &mut conflicts {
5464 conflict.source = DuplicateClaimSource::CatalogUpdate;
5465 }
5466 log_duplicate_claim_events(conflicts);
5467 return Ok(vec![control_error_frame(
5468 &frame,
5469 "reserved_capability",
5470 message,
5471 )?]);
5472 }
5473
5474 let updated = match self.registry.replace_catalog_for_connection(
5475 connection_id,
5476 provides,
5477 capabilities,
5478 ready,
5479 ) {
5480 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => {
5481 return Ok(vec![control_error_frame(
5482 &frame,
5483 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5484 "scope-federated/v1 is fixed at HELLO; restart the module to change it",
5485 )?])
5486 }
5487 Err(crate::registry::RegistryError::CallerAttestationCapabilityChanged) => {
5488 return Ok(vec![control_error_frame(
5489 &frame,
5490 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5491 "caller-attestation/v1 is fixed at HELLO; restart the module to change it",
5492 )?])
5493 }
5494 result => {
5495 result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5496 }
5497 };
5498 if updated.is_none() {
5499 return Ok(vec![control_error_frame(
5500 &frame,
5501 "not_registered",
5502 "catalog.update requires an active module registration owned by this connection",
5503 )?]);
5504 }
5505 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
5506 log_duplicate_claim_events(
5507 self.capability_evaluator
5508 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
5509 );
5510 }
5511 if capability_census_trigger(
5512 registration.manifest.capabilities.as_ref(),
5513 updated
5514 .as_ref()
5515 .and_then(|entry| entry.manifest.capabilities.as_ref()),
5516 ) {
5517 self.enforce_capability_denies();
5518 }
5519 self.refresh_capability_requirements();
5520
5521 let response = ModuleControlResponseToModule::CatalogUpdate {};
5522 control_response_body_frame(
5523 &frame,
5524 &response,
5525 "ModuleControlResponseToModule::CatalogUpdate",
5526 )
5527 .map(|frame| vec![frame])
5528 }
5529
5530 pub(crate) fn with_activity_ledger_config(
5531 mut self,
5532 run_dir: &Path,
5533 incarnation: String,
5534 config: crate::activity::ActivityConfig,
5535 ) -> Self {
5536 self.activity = Some(crate::activity::ActivityLedger::open_config(
5537 run_dir.join("activity.jsonl"),
5538 incarnation,
5539 config,
5540 ));
5541 self.supervisor.set_activity(self.activity.clone());
5542 self
5543 }
5544
5545 pub(crate) async fn configure_activity(
5546 &self,
5547 config: crate::activity::ActivityConfig,
5548 ) -> Result<(), &'static str> {
5549 match &self.activity {
5550 Some(ledger) => ledger.configure(config).await,
5551 None => Ok(()),
5552 }
5553 }
5554
5555 fn cancel_activity_subscription(
5556 &self,
5557 ctx: &RouteCtx,
5558 frame: &Frame,
5559 ) -> Result<bool, RouterError> {
5560 let task = self
5561 .activity_streams
5562 .lock()
5563 .unwrap_or_else(|p| p.into_inner())
5564 .remove(&(ctx.connection_id, frame.header.corr));
5565 let Some(task) = task else {
5566 return Ok(false);
5567 };
5568 task.abort();
5569 let end = Frame::build_with_version(
5570 response_version(frame),
5571 FrameType::StreamEnd,
5572 control_flags(),
5573 0,
5574 0,
5575 frame.header.corr,
5576 Vec::new(),
5577 )
5578 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
5579 let sink = ctx.egress.clone();
5580 tokio::spawn(async move {
5581 let _ = sink.send(end).await;
5582 });
5583 Ok(true)
5584 }
5585
5586 async fn handle_activity_read(
5587 &self,
5588 ctx: &RouteCtx,
5589 frame: Frame,
5590 ) -> Result<Vec<Frame>, RouterError> {
5591 #[cfg(test)]
5592 {
5593 let pause = self
5594 .activity_request_pause
5595 .lock()
5596 .unwrap()
5597 .clone()
5598 .filter(|pause| pause.connection == ctx.connection_id);
5599 if let Some(pause) = pause {
5600 let _cancelled = ActivityRequestCancellation(pause.clone());
5601 pause.started.notify_one();
5602 std::future::pending::<()>().await;
5603 }
5604 }
5605 use subc_control::{
5606 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
5607 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
5608 };
5609 #[derive(Deserialize)]
5610 struct Identity {
5611 #[serde(default)]
5612 consumer_identity: Option<ConsumerIdentity>,
5613 }
5614 let mut bound_spawn = None;
5618 let legacy = match self.activity_reader(ctx.connection_id) {
5619 channel_operations::ActivityReader::Module { module_id, spawn } => {
5620 bound_spawn = Some(spawn);
5621 Some(Ok(Principal::Reserved { module_id }))
5622 }
5623 channel_operations::ActivityReader::Refused => {
5625 return Ok(vec![control_error_frame(
5626 &frame,
5627 ACTIVITY_READ_FORBIDDEN,
5628 "helper, relay and retired connections cannot read activity",
5629 )?]);
5630 }
5631 channel_operations::ActivityReader::Legacy => None,
5632 };
5633 let principal = match legacy {
5637 Some(principal) => principal,
5638 None => match self
5639 .forwarding
5640 .module_id_for_connection(ctx.connection_id)
5641 .map_err(RouterError::Forwarding)?
5642 {
5643 Some(module_id) => Ok(Principal::Reserved { module_id }),
5644 None => {
5645 let identity: Identity = match serde_json::from_slice(&frame.body) {
5646 Ok(identity) => identity,
5647 Err(_) => {
5648 return Ok(vec![control_error_frame(
5649 &frame,
5650 ACTIVITY_MALFORMED,
5651 "malformed activity read",
5652 )?])
5653 }
5654 };
5655 self.route_open_principal(
5656 ctx.connection_id,
5657 &frame,
5658 identity.consumer_identity,
5659 )?
5660 }
5661 },
5662 };
5663 let admitted = matches!(principal, Ok(Principal::Direct))
5664 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
5665 if !admitted {
5666 return Ok(vec![control_error_frame(
5667 &frame,
5668 ACTIVITY_READ_FORBIDDEN,
5669 "activity reads require an operator or trusted mirror principal",
5670 )?]);
5671 }
5672 let Some(ledger) = &self.activity else {
5673 return Ok(vec![control_error_frame(
5674 &frame,
5675 ACTIVITY_UNAVAILABLE,
5676 "activity ledger unavailable",
5677 )?]);
5678 };
5679 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
5680 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
5681 {
5682 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
5683 Ok(request) => ledger.query(request).await,
5684 Err(_) => Err(ACTIVITY_MALFORMED),
5685 };
5686 return match result {
5687 Ok(reply) => Ok(vec![control_response_body_frame(
5688 &frame,
5689 &reply,
5690 ops::ACTIVITY_QUERY,
5691 )?]),
5692 Err(code) => Ok(vec![control_error_frame(
5693 &frame,
5694 code,
5695 "activity query refused",
5696 )?]),
5697 };
5698 }
5699 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
5700 Ok(request) => request,
5701 Err(_) => {
5702 return Ok(vec![control_error_frame(
5703 &frame,
5704 ACTIVITY_MALFORMED,
5705 "malformed activity subscription",
5706 )?])
5707 }
5708 };
5709 let mut subscription = match ledger.subscribe(request.since).await {
5710 Ok(subscription) => subscription,
5711 Err(code) => {
5712 return Ok(vec![control_error_frame(
5713 &frame,
5714 code,
5715 "activity subscription refused",
5716 )?])
5717 }
5718 };
5719 let key = (ctx.connection_id, frame.header.corr);
5720 let sink = ctx.egress.clone();
5721 let mut streams = self
5722 .activity_streams
5723 .lock()
5724 .unwrap_or_else(|p| p.into_inner());
5725 streams.retain(|_, task| !task.is_finished());
5726 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
5727 return Ok(vec![control_error_frame(
5728 &frame,
5729 "duplicate_subscription",
5730 "activity subscription correlation id already in use",
5731 )?]);
5732 }
5733 let stream = async move {
5734 loop {
5735 let entry = if let Some(entry) = subscription.replay.pop_front() {
5736 Some(entry)
5737 } else {
5738 subscription.receiver.recv().await
5739 };
5740 let Some(entry) = entry else {
5741 break;
5742 };
5743 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
5744 return;
5745 };
5746 let Ok(next) = Frame::build_with_version(
5747 response_version(&frame),
5748 FrameType::StreamData,
5749 control_flags(),
5750 0,
5751 0,
5752 frame.header.corr,
5753 body,
5754 ) else {
5755 return;
5756 };
5757 if sink.send(next).await.is_err() {
5758 return;
5759 }
5760 }
5761 if let Some(code) = subscription.terminal_code() {
5764 if let Ok(end) =
5765 control_error_frame(&frame, code, "activity subscriber fell behind")
5766 {
5767 let _ = sink.send(end).await;
5768 }
5769 }
5770 };
5771 let task = tokio::spawn(async move {
5775 match bound_spawn {
5776 Some(spawn) => {
5777 tokio::select! {
5778 () = stream => {}
5779 () = spawn.wait_closed() => {}
5780 }
5781 }
5782 None => stream.await,
5783 }
5784 });
5785 streams.insert(key, task.abort_handle());
5786 Ok(Vec::new())
5787 }
5788
5789 async fn handle_activity_record(
5790 &self,
5791 ctx: &RouteCtx,
5792 frame: Frame,
5793 ) -> Result<Vec<Frame>, RouterError> {
5794 use subc_control::{
5795 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
5796 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
5797 };
5798 let result = async {
5799 let module = self
5800 .forwarding
5801 .module_id_for_connection(ctx.connection_id)
5802 .ok()
5803 .flatten()
5804 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
5805 let request: ActivityRecordRequest =
5806 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
5807 crate::activity::validate(&module, &request)?;
5808 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
5809 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
5810 let at_ms = std::time::SystemTime::now()
5811 .duration_since(std::time::UNIX_EPOCH)
5812 .unwrap_or_default()
5813 .as_millis() as u64;
5814 ledger.record(module, request, actor, at_ms).await
5815 }
5816 .await;
5817 match result {
5818 Ok(entry) => Ok(vec![control_response_body_frame(
5819 &frame,
5820 &ActivityRecordResponse { seq: entry.seq },
5821 "activity.record",
5822 )?]),
5823 Err(code) => Ok(vec![control_error_frame(
5824 &frame,
5825 code,
5826 "activity recording refused",
5827 )?]),
5828 }
5829 }
5830
5831 fn activity_actor(
5832 &self,
5833 connection: ConnectionId,
5834 module: &str,
5835 on: Option<&subc_control::ActivityOn>,
5836 ) -> Result<subc_control::ActivityActor, &'static str> {
5837 use crate::forwarding::{DataRoute, DataRouteState};
5838 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
5839 let principal = |id: &str| Principal::Reserved {
5840 module_id: id.to_owned(),
5841 };
5842 let mut actor = ActivityActor {
5843 principal: principal(module),
5844 scope_owner: None,
5845 agent_id: None,
5846 run_id: None,
5847 flow_id: None,
5848 project_root: None,
5849 workdir: None,
5850 project_id: None,
5851 };
5852 let scope = match on {
5853 None => return Ok(actor),
5854 Some(ActivityOn::Scope(scope)) => {
5855 let selector = subc_protocol::scope::ScopeSelector::new(
5856 principal(module),
5857 scope.scope_ref.clone(),
5858 Some(scope.epoch),
5859 );
5860 self.scopes
5861 .read()
5862 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5863 .attribute(&principal(module), module, &selector, true)
5864 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5865 }
5866 Some(ActivityOn::Route(route)) => {
5867 let DataRoute::Module(DataRouteState::Bound(binding)) = self
5868 .forwarding
5869 .lookup_data_route(connection, route.channel, route.epoch)
5870 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5871 else {
5872 return Err(ACTIVITY_ACTOR_UNKNOWN);
5873 };
5874 if binding.module_id != module {
5875 return Err(ACTIVITY_ACTOR_UNKNOWN);
5876 }
5877 actor.principal = binding.principal.clone();
5878 actor.project_root = binding
5879 .project_root
5880 .as_ref()
5881 .map(|root| root.as_path().to_path_buf());
5882 let Some(bound) = &binding.scope else {
5883 return Ok(actor);
5884 };
5885 let owner = principal(&bound.owner);
5886 let selector = subc_protocol::scope::ScopeSelector::new(
5887 owner.clone(),
5888 bound.scope_ref.clone(),
5889 Some(bound.tag.scope_epoch),
5890 );
5891 let admitted = self
5892 .scopes
5893 .read()
5894 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5895 .attribute(&owner, module, &selector, true)
5896 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
5897 if admitted.tag != bound.tag {
5898 return Err(ACTIVITY_ACTOR_UNKNOWN);
5899 }
5900 admitted
5901 }
5902 };
5903 actor.scope_owner = Some(scope.stamp.owner);
5904 actor.agent_id = scope.stamp.attributes.agent_id;
5905 actor.run_id = scope.stamp.attributes.run_id;
5906 actor.flow_id = scope.stamp.attributes.flow_id;
5907 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
5908 actor.project_id = scope.stamp.project_id;
5909 Ok(actor)
5910 }
5911
5912 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5913 self.refresh_capability_requirements();
5914 let mut counters = self.counters.snapshot();
5920 let losses = self.resource_tracker.loss_diagnostics();
5921 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
5922 if let Some(obj) = counters.as_object_mut() {
5923 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
5924 }
5925 }
5926 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
5927 obj.insert(
5928 "activity_unparseable_lines".into(),
5929 ledger.read_errors().into(),
5930 );
5931 obj.insert(
5932 "activity_lifecycle_append_failures".into(),
5933 ledger.lifecycle_failures().into(),
5934 );
5935 obj.insert(
5936 "activity_started_digest_skips".into(),
5937 ledger.lifecycle_digest_skips().into(),
5938 );
5939 }
5940 if let (Ok((connections_with_routes, max)), Some(obj)) = (
5941 self.forwarding.client_route_concentration(),
5942 counters.as_object_mut(),
5943 ) {
5944 obj.insert(
5945 "client_connections_with_routes".into(),
5946 connections_with_routes.into(),
5947 );
5948 obj.insert("max_routes_on_one_connection".into(), max.into());
5949 }
5950 if let (Some(open_breakers), Some(obj)) = (
5956 self.route_bind_breakers.open_snapshot(),
5957 counters.as_object_mut(),
5958 ) {
5959 obj.insert("route_bind_breakers_open".into(), open_breakers);
5960 }
5961 if let (Some(reason), Some(obj)) = (
5966 self.local_endpoint_unavailable.clone(),
5967 counters.as_object_mut(),
5968 ) {
5969 obj.insert("local_endpoint_unavailable".into(), reason.into());
5970 }
5971 let response = ClientControlResponse::ServerDescribe {
5972 protocol_ver: PROTOCOL_VERSION,
5973 subc_ops: subc_ops(),
5974 capabilities: self.describe_capabilities(),
5975 connected_clients: self.connected_clients.count(),
5976 counters: Some(counters),
5977 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
5978 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
5979 capability_requirements: self.capability_requirement_statuses(),
5980 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
5981 config_path: self
5982 .rescan
5983 .as_ref()
5984 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
5985 };
5986 Ok(vec![control_response_body_frame(
5987 &frame,
5988 &response,
5989 "ClientControlResponse::ServerDescribe",
5990 )?])
5991 }
5992
5993 fn handle_catalog_list(
5994 &self,
5995 frame: Frame,
5996 module_id: Option<String>,
5997 ) -> Result<Vec<Frame>, RouterError> {
5998 let (generation, modules) = self
5999 .registry
6000 .list_modules_with_nonce_verification()
6001 .map_err(|err| {
6002 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
6003 })?;
6004 let entries = modules
6005 .into_iter()
6006 .filter(|(registration, _)| {
6007 module_id
6008 .as_deref()
6009 .map(|wanted| registration.manifest.module_id == wanted)
6010 .unwrap_or(true)
6011 })
6012 .map(|(registration, nonce_verified)| {
6013 let entry = self
6014 .catalog_discovery_entry(®istration, module_id.is_some())
6015 .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
6016 let not_ready = self.not_ready_reason(®istration);
6017 let mut entry = entry;
6018 entry.roles = registration.manifest.provides;
6019 entry.control_ops = registration.control_ops;
6020 Ok(entry
6021 .with_ready(not_ready.is_none())
6022 .with_display_name(registration.manifest.display_name)
6023 .with_not_ready(not_ready)
6024 .with_module_version(Some(registration.manifest.module_version))
6025 .with_provenance(registration.manifest.provenance)
6026 .with_capabilities(registration.manifest.capabilities)
6027 .with_self_signals(registration.manifest.self_signals)
6028 .with_events(registration.manifest.events)
6029 .with_operation_delivery(registration.manifest.operation_delivery)
6030 .with_operation_payload(registration.manifest.operation_payload)
6031 .with_flow_operations(registration.manifest.flow_operations)
6032 .with_operation_deferral(registration.manifest.operation_deferral)
6033 .with_confirm_operations(registration.manifest.confirm_operations)
6034 .with_rich_ui(registration.manifest.rich_ui)
6035 .with_role_declarations(registration.manifest.role_declarations)
6036 .with_nonce_verified(nonce_verified))
6037 })
6038 .collect::<Result<Vec<_>, RouterError>>()?;
6039 let response = ClientControlResponse::CatalogList {
6040 generation,
6041 modules: entries,
6042 subc_ops: subc_ops(),
6043 };
6044 Ok(vec![control_response_body_frame(
6045 &frame,
6046 &response,
6047 "ClientControlResponse::CatalogList",
6048 )?])
6049 }
6050
6051 fn route_open_caller_from_frame(&self, ctx: &RouteCtx, frame: &Frame) -> RouteOpenCaller {
6055 let Ok(ClientControlRequest::RouteOpen {
6056 identity,
6057 consumer_identity,
6058 scope,
6059 ..
6060 }) = parse_client_control_request(&frame.body)
6061 else {
6062 return RouteOpenCaller::unknown();
6063 };
6064 let principal = self
6065 .attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref())
6066 .map(|attested| attested.principal);
6067 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
6068 }
6069
6070 fn route_open_principal(
6071 &self,
6072 connection_id: ConnectionId,
6073 frame: &Frame,
6074 consumer_identity: Option<ConsumerIdentity>,
6075 ) -> Result<Result<Principal, Frame>, RouterError> {
6076 let attested = self
6077 .attest_route_open_consumer(connection_id, consumer_identity.as_ref())
6078 .map(|attested| attested.principal);
6079 self.route_open_principal_attested(frame, consumer_identity, attested)
6080 }
6081
6082 fn route_open_principal_attested(
6086 &self,
6087 frame: &Frame,
6088 consumer_identity: Option<ConsumerIdentity>,
6089 attested: Option<Principal>,
6090 ) -> Result<Result<Principal, Frame>, RouterError> {
6091 if let Some(principal) = attested {
6092 return Ok(Ok(principal));
6093 }
6094 Ok(Err(control_error_frame(
6095 frame,
6096 "bad_consumer_identity",
6097 consumer_identity.map_or_else(
6098 || "connection claim is no longer authorized for routes".to_string(),
6099 |identity| format!(
6100 "consumer_identity for module_id '{}' conflicts with its connection claim or did not match a supervised launch nonce",
6101 identity.module_id
6102 ),
6103 ),
6104 )?))
6105 }
6106
6107 #[allow(clippy::too_many_arguments)]
6114 fn route_open_refusal_frame(
6115 &self,
6116 ctx: &RouteCtx,
6117 caller: &RouteOpenCaller,
6118 frame: &Frame,
6119 module_id: &str,
6120 reason: &'static str,
6121 code: &'static str,
6122 message: impl Into<String>,
6123 ) -> Result<Frame, RouterError> {
6124 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
6125 let detail = if matches!(
6126 reason,
6127 "registration_not_active"
6128 | "supervisor_not_live"
6129 | "no_forwarding_connection"
6130 | "reloading"
6131 ) {
6132 let expected_back = (reason == "reloading").then_some(true);
6135 self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
6136 } else {
6137 None
6138 };
6139 control_error_body_frame(
6140 frame,
6141 ErrorBody {
6142 code: code.to_string(),
6143 message: message.into(),
6144 detail,
6145 },
6146 )
6147 }
6148
6149 fn route_open_recovery_detail(
6150 &self,
6151 module_id: &str,
6152 corr: u64,
6153 detail: Option<serde_json::Value>,
6154 expected_back: Option<bool>,
6155 ) -> Result<Option<serde_json::Value>, RouterError> {
6156 let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
6157 return Ok(detail);
6158 };
6159 let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
6160 detail["state"] = serde_json::json!(status.state.to_string());
6161 if let Some(expected_back) = expected_back {
6162 detail["expected_back"] = serde_json::json!(expected_back);
6163 } else if let Some(module) = self.supervisor.get(module_id) {
6164 match module.will_recover_after_connection_loss() {
6165 Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
6166 Err(err) => warn!(
6167 %module_id,
6168 error = %err,
6169 "failed to read route.open recovery verdict; omitting expected_back"
6170 ),
6171 }
6172 }
6173 Ok(Some(detail))
6174 }
6175
6176 #[allow(clippy::too_many_arguments)]
6185 fn route_bind_rejection(
6186 &self,
6187 ctx: &RouteCtx,
6188 caller: &RouteOpenCaller,
6189 frame: &Frame,
6190 target_module_id: &str,
6191 body: ErrorBody,
6192 daemon_authored: bool,
6193 ) -> Result<Vec<Frame>, RouterError> {
6194 if daemon_authored {
6200 let scope_code = match body.code.as_str() {
6201 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
6202 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
6203 _ => None,
6204 };
6205 if let Some(code) = scope_code {
6206 self.observe_route_open_refusal(
6207 ctx,
6208 caller,
6209 target_module_id,
6210 "scope_changed_before_commit",
6211 code,
6212 );
6213 return Ok(vec![control_error_body_frame(frame, body)?]);
6214 }
6215 }
6216 self.counters
6217 .increment_route_open_refused("module_rejected");
6218 info!(
6219 target: "control",
6220 code = "module_rejected",
6221 module_code = ?body.code,
6222 module_id = ?target_module_id,
6223 connection_id = ctx.connection_id.get(),
6224 principal = %caller.principal,
6225 scope = %caller.scope,
6226 harness = %caller.harness,
6227 "route.open refused"
6228 );
6229 let refusal = if daemon_authored {
6230 control_error_body_frame(frame, body)?
6231 } else {
6232 control_relayed_error_body_frame(frame, body)?
6233 };
6234 Ok(vec![refusal])
6235 }
6236
6237 #[allow(clippy::too_many_arguments)]
6255 fn route_open_breaker_refusal_frame(
6256 &self,
6257 ctx: &RouteCtx,
6258 caller: &RouteOpenCaller,
6259 frame: &Frame,
6260 module_id: &str,
6261 consecutive_timeouts: u32,
6262 retry_in: Duration,
6263 probe_in_flight: bool,
6264 ) -> Result<Frame, RouterError> {
6265 self.counters
6266 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
6267 debug!(
6268 target: "control",
6269 code = "module_timeout",
6270 module_id = ?module_id,
6271 connection_id = ctx.connection_id.get(),
6272 principal = %caller.principal,
6273 scope = %caller.scope,
6274 harness = %caller.harness,
6275 consecutive_timeouts,
6276 retry_in_ms = retry_in.as_millis() as u64,
6277 probe_in_flight,
6278 "route.open refused by open bind-relay breaker"
6279 );
6280 let detail = if probe_in_flight {
6287 "one test route is already being tried; retry once it settles".to_string()
6288 } else {
6289 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
6290 };
6291 control_error_frame(
6292 frame,
6293 "module_timeout",
6294 format!(
6295 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
6296 timed out in a row); {detail}; its established routes are unaffected"
6297 ),
6298 )
6299 }
6300
6301 fn observe_route_open_refusal(
6314 &self,
6315 ctx: &RouteCtx,
6316 caller: &RouteOpenCaller,
6317 module_id: &str,
6318 reason: &'static str,
6319 code: &'static str,
6320 ) {
6321 self.counters.increment_route_open_refused(code);
6322 info!(
6323 target: "control",
6324 code,
6325 reason,
6326 module_id = ?module_id,
6327 connection_id = ctx.connection_id.get(),
6328 principal = %caller.principal,
6329 scope = %caller.scope,
6330 harness = %caller.harness,
6331 "route.open refused"
6332 );
6333 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
6334 self.route_outages.record_not_serving(module_id, reason);
6335 }
6336 }
6337
6338 fn observe_route_open_accept(
6383 &self,
6384 ctx: &RouteCtx,
6385 module_id: &str,
6386 principal: &str,
6387 scope: &str,
6388 ) {
6389 self.route_outages.record_accepted(module_id);
6390 self.counters.increment_route_open_accepted(principal);
6391 info!(
6392 target: "control",
6393 principal,
6394 module_id,
6395 connection_id = ctx.connection_id.get(),
6396 scope = %scope,
6397 "route.open accepted"
6398 );
6399 }
6400
6401 fn supervised_absent_route_open_refusal_frame(
6402 &self,
6403 ctx: &RouteCtx,
6404 caller: &RouteOpenCaller,
6405 frame: &Frame,
6406 module_id: &str,
6407 code: &'static str,
6408 status: &crate::supervise::ModuleStatus,
6409 ) -> Result<Frame, RouterError> {
6410 self.counters.increment_route_open_refused(code);
6411 info!(
6412 target: "control",
6413 code,
6414 reason = "supervised_not_registered",
6415 module_id = ?module_id,
6416 connection_id = ctx.connection_id.get(),
6417 principal = %caller.principal,
6418 scope = %caller.scope,
6419 harness = %caller.harness,
6420 state = %status.state,
6421 enabled = status.enabled,
6422 live = status.live,
6423 "route.open refused"
6424 );
6425 self.route_outages
6429 .record_not_serving(module_id, "supervised_not_registered");
6430 let message = if let Some(pid) = status.waiting_for_previous_pid {
6431 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
6432 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
6433 })
6434 } else {
6435 format!(
6436 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
6437 status.state, status.enabled, status.live
6438 )
6439 };
6440 let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
6444 control_error_body_frame(
6445 frame,
6446 ErrorBody {
6447 code: code.to_string(),
6448 message,
6449 detail: self.route_open_recovery_detail(
6450 module_id,
6451 frame.header.corr,
6452 None,
6453 expected_back,
6454 )?,
6455 },
6456 )
6457 }
6458
6459 async fn handle_route_open(
6460 &self,
6461 ctx: &RouteCtx,
6462 frame: Frame,
6463 request: RouteOpenRequest,
6464 timings: &mut RouteOpenTimings,
6465 ) -> Result<Vec<Frame>, RouterError> {
6466 let RouteOpenRequest {
6467 target,
6468 mut identity,
6469 consumer_identity,
6470 consumer_capabilities,
6471 role_versions,
6472 busy_refusals,
6473 admission_facts,
6474 scope,
6475 accepts_close_reasons,
6476 } = request;
6477 let target_module_id = target_module_id(&target).to_string();
6478 timings.module_id.clone_from(&target_module_id);
6479 if self
6480 .registry
6481 .get_module_by_connection(ctx.connection_id)
6482 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6483 .is_some()
6484 {
6485 return Ok(vec![control_error_frame(
6486 &frame,
6487 "invalid_request",
6488 "module connections cannot open client routes",
6489 )?]);
6490 }
6491 debug!(
6492 connection_id = ctx.connection_id.get(),
6493 corr = frame.header.corr,
6494 module_id = %target_module_id,
6495 "handling route.open"
6496 );
6497
6498 let attested =
6504 self.attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref());
6505 let mut caller = RouteOpenCaller::new(
6506 attested.as_ref().map(|attested| &attested.principal),
6507 scope.as_ref(),
6508 &identity,
6509 );
6510
6511 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
6517 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
6518 self.observe_route_open_refusal(
6519 ctx,
6520 &caller,
6521 &target_module_id,
6522 "invalid_role_versions",
6523 error_codes::INVALID_REQUEST,
6524 );
6525 return Ok(vec![control_error_body_frame(
6526 &frame,
6527 ErrorBody {
6528 code: error_codes::INVALID_REQUEST.to_string(),
6529 message: error.to_string(),
6530 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
6531 },
6532 )?]);
6533 }
6534
6535 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
6536 if status.waiting_for_previous_pid.is_some() {
6537 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6538 ctx,
6539 &caller,
6540 &frame,
6541 &target_module_id,
6542 "module_warming",
6543 &status,
6544 )?]);
6545 }
6546 }
6547
6548 let admission_registry = self
6567 .registry
6568 .route_admission_snapshot()
6569 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6570 let Some(registration) =
6571 crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
6572 else {
6573 if let Some((status, warming)) =
6574 self.supervisor_status(&target_module_id, frame.header.corr)?
6575 {
6576 if status.protocol == ModuleProtocol::None {
6583 return Ok(vec![self.route_open_refusal_frame(
6584 ctx,
6585 &caller,
6586 &frame,
6587 &target_module_id,
6588 "protocol_none",
6589 error_codes::MODULE_NO_PROTOCOL,
6590 format!(
6591 "module_id '{target_module_id}' is declared protocol: none; \
6592 it speaks no subc wire and serves no routes"
6593 ),
6594 )?]);
6595 }
6596 let code = if warming {
6597 "module_warming"
6598 } else {
6599 "target_unavailable"
6600 };
6601 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6602 ctx,
6603 &caller,
6604 &frame,
6605 &target_module_id,
6606 code,
6607 &status,
6608 )?]);
6609 }
6610 if let Some(removed_ago_ms) =
6611 self.supervisor.removal_tombstone_age_ms(&target_module_id)
6612 {
6613 return Ok(vec![self.route_open_refusal_frame(
6614 ctx,
6615 &caller,
6616 &frame,
6617 &target_module_id,
6618 "removed",
6619 error_codes::MODULE_REMOVED,
6620 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
6621 )?]);
6622 }
6623 return Ok(vec![self.route_open_refusal_frame(
6624 ctx,
6625 &caller,
6626 &frame,
6627 &target_module_id,
6628 "not_registered",
6629 error_codes::UNKNOWN_MODULE,
6630 format!("module_id '{target_module_id}' is not registered"),
6631 )?]);
6632 };
6633
6634 if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
6638 .is_err()
6639 {
6640 self.counters
6641 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
6642 info!(
6643 target: "control",
6644 code = error_codes::MODULE_WARMING,
6645 module_id = ?target_module_id,
6646 connection_id = ctx.connection_id.get(),
6647 principal = %caller.principal,
6648 scope = %caller.scope,
6649 harness = %caller.harness,
6650 reason = "declared_not_ready",
6651 "route.open refused"
6652 );
6653 self.route_outages
6656 .record_not_serving(&target_module_id, "declared_not_ready");
6657 return Ok(vec![control_error_body_frame(
6660 &frame,
6661 ErrorBody {
6662 code: error_codes::MODULE_WARMING.to_string(),
6663 message: format!(
6664 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
6665 ),
6666 detail: self.route_open_recovery_detail(
6667 &target_module_id,
6668 frame.header.corr,
6669 Some(serde_json::json!({ "reason": "declared_not_ready" })),
6670 Some(true),
6671 )?,
6672 },
6673 )?]);
6674 }
6675
6676 if let Some(capability) = self.first_unprovided_required_capability(registration) {
6695 self.counters
6696 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
6697 info!(
6698 target: "control",
6699 code = error_codes::MODULE_WARMING,
6700 module_id = ?target_module_id,
6701 connection_id = ctx.connection_id.get(),
6702 principal = %caller.principal,
6703 scope = %caller.scope,
6704 harness = %caller.harness,
6705 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6706 capability = %capability,
6707 "route.open refused"
6708 );
6709 return Ok(vec![control_error_body_frame(
6710 &frame,
6711 ErrorBody {
6712 code: error_codes::MODULE_WARMING.to_string(),
6713 message: format!(
6714 "module_id '{target_module_id}' requires capability '{capability}', \
6715 which no registered module provides; retry"
6716 ),
6717 detail: Some(serde_json::json!({
6718 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6719 "capability": capability,
6720 })),
6721 },
6722 )?]);
6723 }
6724
6725 if !target_has_required_role(&target, ®istration.manifest.provides) {
6726 return Ok(vec![self.route_open_refusal_frame(
6727 ctx,
6728 &caller,
6729 &frame,
6730 &target_module_id,
6731 "role_not_provided",
6732 "target_unavailable",
6733 format!("module_id '{target_module_id}' does not provide the requested target"),
6734 )?]);
6735 }
6736
6737 if registration.state != ChannelState::Active {
6738 return Ok(vec![self.route_open_refusal_frame(
6739 ctx,
6740 &caller,
6741 &frame,
6742 &target_module_id,
6743 "registration_not_active",
6744 "target_unavailable",
6745 format!("module_id '{target_module_id}' is not active"),
6746 )?]);
6747 }
6748
6749 if self
6750 .forwarding
6751 .module_is_draining(&target_module_id)
6752 .map_err(RouterError::Forwarding)?
6753 {
6754 return Ok(vec![self.route_open_refusal_frame(
6755 ctx,
6756 &caller,
6757 &frame,
6758 &target_module_id,
6759 "reloading",
6760 "module_reloading",
6761 format!("module_id '{target_module_id}' is reloading"),
6762 )?]);
6763 }
6764
6765 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
6766 process_liveness.process_live(&target_module_id) == Some(false)
6767 }) {
6768 if process_liveness.process_replacing(&target_module_id) {
6775 return Ok(vec![self.route_open_refusal_frame(
6776 ctx,
6777 &caller,
6778 &frame,
6779 &target_module_id,
6780 "reloading",
6781 "module_reloading",
6782 format!("module_id '{target_module_id}' is reloading"),
6783 )?]);
6784 }
6785 return Ok(vec![self.route_open_refusal_frame(
6786 ctx,
6787 &caller,
6788 &frame,
6789 &target_module_id,
6790 "supervisor_not_live",
6791 "target_unavailable",
6792 format!("module_id '{target_module_id}' is not live"),
6793 )?]);
6794 }
6795
6796 if !self
6797 .forwarding
6798 .has_live_module_connection(&target_module_id)
6799 .map_err(RouterError::Forwarding)?
6800 {
6801 return Ok(vec![self.route_open_refusal_frame(
6802 ctx,
6803 &caller,
6804 &frame,
6805 &target_module_id,
6806 "no_forwarding_connection",
6807 "target_unavailable",
6808 format!("module_id '{target_module_id}' has no live forwarding connection"),
6809 )?]);
6810 }
6811
6812 if let Some(error) =
6813 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
6814 {
6815 self.observe_route_open_refusal(
6816 ctx,
6817 &caller,
6818 &target_module_id,
6819 "op_not_allowed",
6820 "op_not_allowed",
6821 );
6822 return Ok(vec![error]);
6823 }
6824
6825 let principal = match (&attested, consumer_identity.is_some()) {
6826 (None, false) => {
6829 self.observe_route_open_refusal(
6830 ctx,
6831 &caller,
6832 &target_module_id,
6833 "registration_channel_closed",
6834 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6835 );
6836 return Ok(vec![control_error_frame(
6837 &frame,
6838 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6839 "the claim binding's launch has ended",
6840 )?]);
6841 }
6842 _ => match self.route_open_principal_attested(
6843 &frame,
6844 consumer_identity,
6845 attested.as_ref().map(|attested| attested.principal.clone()),
6846 )? {
6847 Ok(principal) => principal,
6848 Err(error) => {
6849 self.observe_route_open_refusal(
6850 ctx,
6851 &caller,
6852 &target_module_id,
6853 "bad_consumer_identity",
6854 "bad_consumer_identity",
6855 );
6856 return Ok(vec![error]);
6857 }
6858 },
6859 };
6860 let attested = attested.expect("an attested principal was resolved above");
6861 timings.principal = principal_label(&principal);
6862
6863 let Ok(mut caller_capture) = attested.capture() else {
6866 self.observe_route_open_refusal(
6867 ctx,
6868 &caller,
6869 &target_module_id,
6870 "registration_channel_closed",
6871 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6872 );
6873 return Ok(vec![control_error_frame(
6874 &frame,
6875 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6876 "the opener's launch has ended",
6877 )?]);
6878 };
6879 let caller_module_id = match &attested.spawn {
6883 Some((record, _)) => Some(record.module_id().to_owned()),
6884 None => match &principal {
6885 Principal::Reserved { module_id } => Some(module_id.clone()),
6886 _ => None,
6887 },
6888 };
6889
6890 let build_class = identity.build_class.take().filter(|build_class| {
6893 if caller_module_id.as_deref() == Some("callosum") {
6894 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
6895 } else {
6896 if self
6897 .build_class_warned
6898 .lock()
6899 .unwrap_or_else(|p| p.into_inner())
6900 .insert(ctx.connection_id)
6901 {
6902 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
6903 "ignoring build_class from a non-Callosum opener");
6904 }
6905 false
6906 }
6907 });
6908
6909 if let Some(opening_registration) = self.caller_registration(&attested, &admission_registry)
6913 {
6914 let opening_module_id = opening_registration.manifest.module_id.as_str();
6915 if let Some(capability) =
6916 denied_capability(&opening_registration.manifest, ®istration.manifest)
6917 {
6918 warn!(
6919 opening_module_id,
6920 target_module_id,
6921 capability,
6922 "refusing route.open because an attested capability deny edge matches"
6923 );
6924 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6925 &frame,
6926 &target_module_id,
6927 "capability_deny_edge",
6928 "capability_forbidden",
6929 format!(
6930 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
6931 ),
6932 )?]);
6933 }
6934 }
6935
6936 if admission_facts.is_some() {
6937 let carrier_matches = caller_module_id.is_some()
6938 && self.admission_facts_carrier_module_id.as_deref() == caller_module_id.as_deref();
6939 if !carrier_matches {
6940 return Ok(vec![self.route_open_refusal_frame(
6941 ctx,
6942 &caller,
6943 &frame,
6944 &target_module_id,
6945 "admission_facts_carrier_not_permitted",
6946 "admission_facts_not_permitted",
6947 "admission facts may only be carried by the configured reserved module",
6948 )?]);
6949 }
6950
6951 let target_allowed = self
6952 .admission_facts_targets
6953 .as_ref()
6954 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
6955 if !target_allowed {
6956 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6957 &frame,
6958 &target_module_id,
6959 "admission_facts_target_not_listed",
6960 "admission_facts_target_not_allowed",
6961 format!(
6962 "admission facts are not permitted for target module_id '{target_module_id}'"
6963 ),
6964 )?]);
6965 }
6966
6967 }
6971
6972 let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
6978 None => (None, None, false),
6979 Some(selector) => {
6980 let owner_configured = match &selector.owner {
6981 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
6987 _ => false,
6988 };
6989 let mut scopes = self
6990 .scopes
6991 .write()
6992 .unwrap_or_else(|poisoned| poisoned.into_inner());
6993 let admitted = crate::scopes::admission::admit_route(
6994 &scopes,
6995 &admission_registry,
6996 &principal,
6997 &target_module_id,
6998 Some(selector),
6999 owner_configured,
7000 crate::scopes::admission::AdmissionMode::Open,
7001 )
7002 .map(|route| {
7003 let admission = route.scope.expect("scoped admission");
7004 let bound = scopes.reserve_bind(
7005 &admission,
7006 &principal,
7007 ctx.connection_id,
7008 selector.hold_presence,
7009 );
7010 (admission, bound)
7011 });
7012 match admitted {
7013 Ok((admission, bound)) => {
7014 (Some(bound), Some(admission.stamp), admission.stamp_required)
7015 }
7016 Err(refusal) => {
7017 return Ok(vec![self.route_open_refusal_frame(
7018 ctx,
7019 &caller,
7020 &frame,
7021 &target_module_id,
7022 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7023 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7024 refusal.message,
7025 )?]);
7026 }
7027 }
7028 }
7029 };
7030 if let Some(stamp) = &scope_stamp {
7033 caller.scope = route_open_scope_label(stamp);
7034 }
7035 if scope_stamp.as_ref().is_some_and(scope_acts_for_another) {
7038 caller_capture = caller_capture.map(crate::spawn_authority::CallerCapture::relayed);
7039 }
7040
7041 timings.enter(RouteOpenPhase::ProjectRoot);
7049 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
7050 Ok(project) => project,
7051 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
7052 };
7053 timings.enter(RouteOpenPhase::Admission);
7054 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
7055 Ok(root) => root,
7056 Err(err) => {
7057 return Ok(vec![control_error_frame(
7058 &frame,
7059 "invalid_project_root",
7060 err.to_string(),
7061 )?])
7062 }
7063 };
7064 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
7070
7071 let _concurrency_guard = match self
7082 .route_bind_concurrency
7083 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
7084 {
7085 Ok(guard) => guard,
7086 Err(in_flight) => {
7087 return Ok(vec![self.route_open_target_capacity_refusal(
7088 ctx,
7089 &caller,
7090 &frame,
7091 &target_module_id,
7092 in_flight,
7093 )?]);
7094 }
7095 };
7096
7097 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
7100 RouteBindAdmission::Admitted { guard, probe } => {
7101 if probe {
7102 info!(
7103 module_id = %target_module_id,
7104 connection_id = ctx.connection_id.get(),
7105 "route.bind breaker half-open: admitting one probe"
7106 );
7107 }
7108 guard
7109 }
7110 RouteBindAdmission::Refused {
7111 consecutive_timeouts,
7112 retry_in,
7113 probe_in_flight,
7114 } => {
7115 return Ok(vec![self.route_open_breaker_refusal_frame(
7116 ctx,
7117 &caller,
7118 &frame,
7119 &target_module_id,
7120 consecutive_timeouts,
7121 retry_in,
7122 probe_in_flight,
7123 )?]);
7124 }
7125 };
7126
7127 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
7132 let relay_deadline = Instant::now() + route_bind_relay_timeout;
7133 let operator_app = self
7134 .operator_apps
7135 .lock()
7136 .unwrap_or_else(|p| p.into_inner())
7137 .get(&ctx.connection_id)
7138 .cloned();
7139 let production_peer_build =
7140 build_class.as_deref() == Some(subc_protocol::BUILD_CLASS_PRODUCTION);
7141 timings.enter(RouteOpenPhase::EgressReservation);
7142 let pending = match self
7143 .forwarding
7144 .begin_route_bind_relay_for(
7145 ctx.connection_id,
7146 ctx.egress.clone(),
7147 response_version(&frame),
7148 frame.header.corr,
7149 &target_module_id,
7150 principal.clone(),
7151 bound_scope,
7152 project_root,
7153 identity.harness.clone(),
7154 build_class,
7155 operator_app.clone(),
7156 crate::forwarding::RouteAuthority {
7157 incarnation: self.supervisor.daemon_incarnation(),
7158 caller: caller_capture
7159 .as_ref()
7160 .map(|caller| caller.authority.pair()),
7161 },
7162 relay_deadline,
7163 busy_refusals == Some(true),
7164 accepts_close_reasons,
7165 )
7166 .await
7167 {
7168 Ok(pending) => pending,
7169 Err(err) => {
7170 return Ok(vec![self.route_open_refusal_frame(
7171 ctx,
7172 &caller,
7173 &frame,
7174 &target_module_id,
7175 "relay_reservation_failed",
7176 forwarding_error_code(&err),
7177 err.to_string(),
7178 )?])
7179 }
7180 };
7181 timings.enter(RouteOpenPhase::Admission);
7182 let crate::forwarding::PendingRouteBindRelay {
7183 endpoint,
7184 module_sink,
7185 negotiated_ver,
7186 client_channel,
7187 client_epoch,
7188 module_channel,
7189 module_epoch,
7190 corr: relay_corr,
7191 receiver,
7192 } = pending;
7193 let mut reservation =
7194 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
7195
7196 let relay_registration = self
7201 .registry
7202 .get_module_by_connection(endpoint.connection_id)
7203 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7204 let relay_capabilities = relay_registration
7205 .as_ref()
7206 .and_then(|registration| registration.manifest.capabilities.as_ref());
7207 if let Some(selector) = &scope {
7208 if let Err(refusal) = self
7211 .scopes
7212 .read()
7213 .unwrap_or_else(|poisoned| poisoned.into_inner())
7214 .check_bind_proof(&principal, selector)
7215 {
7216 reservation.release_and_disarm();
7217 return Ok(vec![self.route_open_refusal_frame(
7218 ctx,
7219 &caller,
7220 &frame,
7221 &target_module_id,
7222 refusal.code,
7223 refusal.code,
7224 refusal.message,
7225 )?]);
7226 }
7227 }
7228 if let Some(stamp) = scope_stamp.as_ref() {
7229 if let Err(refusal) = crate::scopes::check_target_required_stamp(
7230 stamp_required,
7231 &target_module_id,
7232 relay_capabilities,
7233 )
7234 .and_then(|()| {
7235 crate::scopes::check_target_flow_support(
7236 stamp,
7237 &target_module_id,
7238 relay_capabilities,
7239 )
7240 })
7241 .and_then(|()| {
7242 crate::scopes::check_target_agent_run_support(
7243 stamp,
7244 &target_module_id,
7245 relay_capabilities,
7246 )
7247 }) {
7248 reservation.release_and_disarm();
7249 return Ok(vec![self.route_open_refusal_frame(
7250 ctx,
7251 &caller,
7252 &frame,
7253 &target_module_id,
7254 refusal.code,
7255 refusal.code,
7256 refusal.message,
7257 )?]);
7258 }
7259 }
7260
7261 if scope_stamp
7264 .as_ref()
7265 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
7266 && !crate::scopes::target_supports_federated(relay_capabilities)
7267 {
7268 scope_stamp = None;
7270 }
7271 if !relay_capabilities.is_some_and(|caps| {
7272 caps.provides
7273 .iter()
7274 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
7275 }) {
7276 if let Some(stamp) = scope_stamp.as_mut() {
7277 stamp.workdir = None;
7278 stamp.project_id = None;
7279 }
7280 }
7281 let worker_run = scope_stamp
7286 .as_ref()
7287 .is_some_and(crate::scopes::is_worker_run)
7288 && relay_capabilities.is_some_and(|caps| {
7289 caps.provides
7290 .iter()
7291 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
7292 });
7293 if !worker_run
7294 && !relay_capabilities.is_some_and(|caps| {
7295 caps.provides
7296 .iter()
7297 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
7298 })
7299 {
7300 if let Some(stamp) = scope_stamp.as_mut() {
7301 stamp.attributes.on_behalf_of_agent = None;
7302 }
7303 }
7304 if !relay_capabilities.is_some_and(|caps| {
7305 caps.provides
7306 .iter()
7307 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
7308 }) {
7309 if let Some(stamp) = scope_stamp.as_mut() {
7310 stamp.attributes.flow_version = None;
7311 stamp.attributes.flow_code_hash = None;
7312 stamp.attributes.flow_install_id = None;
7313 }
7314 }
7315 let call_mode = if worker_run
7316 || relay_capabilities
7317 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
7318 {
7319 Some(derive_call_mode(scope_stamp.as_ref()))
7320 } else {
7321 if let Some(stamp) = scope_stamp.as_mut() {
7322 stamp.expires_at_ms = None;
7323 }
7324 None
7325 };
7326
7327 debug!(
7328 connection_id = ctx.connection_id.get(),
7329 client_channel,
7330 client_epoch,
7331 module_channel,
7332 module_epoch,
7333 "reserved route handle pair"
7334 );
7335 let route_principal_label = principal_label(&principal);
7338 let relay = ModuleControlRequest::RouteBind {
7339 route_channel: module_channel,
7340 epoch: module_epoch,
7341 target,
7342 identity,
7343 principal: Some(principal),
7344 consumer_capabilities,
7345 role_versions,
7346 admission_facts,
7347 scope: scope_stamp,
7348 call_mode,
7349 };
7350 let peer = RouteBindPeer::new()
7351 .with_observed_peer(
7352 relay_capabilities
7353 .is_some_and(|caps| {
7354 caps.provides
7355 .iter()
7356 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
7357 })
7358 .then(|| {
7359 self.observed_peers
7360 .lock()
7361 .unwrap_or_else(|p| p.into_inner())
7362 .get(&ctx.connection_id)
7363 .cloned()
7364 .unwrap_or_else(ObservedPeer::unobserved)
7365 }),
7366 )
7367 .with_operator_app(
7368 self.connection_operator_app(operator_app.as_ref())
7369 .filter(|_| {
7370 relay_capabilities.is_some_and(|caps| {
7371 caps.provides
7372 .iter()
7373 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
7374 })
7375 }),
7376 )
7377 .with_caller(caller_for_target(
7378 caller_capture.as_ref(),
7379 relay_capabilities,
7380 &target_module_id,
7381 ))
7382 .with_peer_build(
7383 (production_peer_build
7384 && relay_capabilities.is_some_and(|caps| {
7385 caps.provides.iter().any(|cap| cap == PEER_BUILD_CAPABILITY)
7386 }))
7387 .then(|| subc_protocol::BUILD_CLASS_PRODUCTION.to_owned()),
7388 );
7389 let relay_body = crate::bind_compat::encode_bind(
7390 &relay,
7391 &peer,
7392 &legacy_bind_project,
7393 relay_capabilities,
7394 )
7395 .map_err(|err| {
7396 RouterError::backend(
7397 0,
7398 frame.header.corr,
7399 format!("failed to encode route.bind request: {err}"),
7400 )
7401 })?;
7402 let relay_frame = Frame::build_with_version(
7403 negotiated_ver,
7404 FrameType::Request,
7405 control_flags(),
7406 0,
7407 0,
7408 relay_corr,
7409 relay_body,
7410 )
7411 .map_err(RouterError::FrameBuild)?;
7412
7413 timings.enter(RouteOpenPhase::BindSend);
7414 let sent = module_sink.send(relay_frame).await;
7415 timings.enter(RouteOpenPhase::Admission);
7416 if let Err(err) = sent {
7417 reservation.release_and_disarm();
7418 return Ok(vec![self.route_open_refusal_frame(
7419 ctx,
7420 &caller,
7421 &frame,
7422 &target_module_id,
7423 "relay_send_failed",
7424 "target_unavailable",
7425 err.to_string(),
7426 )?]);
7427 }
7428
7429 #[cfg(test)]
7430 if let Some(hold) = &self.route_bind_relay_sent_hold {
7431 hold.wait().await;
7435 hold.wait().await;
7436 }
7437
7438 if self
7443 .forwarding
7444 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
7445 .map_err(RouterError::Forwarding)?
7446 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
7447 {
7448 self.send_abandoned_route_bind_goodbye(
7449 &module_sink,
7450 negotiated_ver,
7451 module_channel,
7452 module_epoch,
7453 );
7454 }
7455
7456 timings.enter(RouteOpenPhase::BindAckWait);
7457 let outcome = timeout_at(relay_deadline, receiver).await;
7458 timings.enter(RouteOpenPhase::Admission);
7459 match outcome {
7460 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
7461 reservation.disarm();
7462 if breaker.record_accepted() {
7463 info!(
7464 module_id = %target_module_id,
7465 "route.bind breaker closed: the probe was accepted"
7466 );
7467 }
7468 self.observe_route_open_accept(
7469 ctx,
7470 &target_module_id,
7471 &route_principal_label,
7472 &caller.scope,
7473 );
7474 Ok(Vec::new())
7475 }
7476 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
7477 reservation.release_and_disarm();
7478 breaker.record_inconclusive();
7482 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
7483 }
7484 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
7485 reservation.release_and_disarm();
7486 breaker.record_inconclusive();
7487 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
7488 }
7489 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
7490 reservation.release_and_disarm();
7491 breaker.record_inconclusive();
7492 tracing::warn!(
7499 module_id = %target_module_id,
7500 "route.bind relay abandoned: {message}"
7501 );
7502 Ok(vec![self.route_open_refusal_frame(
7503 ctx,
7504 &caller,
7505 &frame,
7506 &target_module_id,
7507 "relay_abandoned",
7508 "target_unavailable",
7509 message,
7510 )?])
7511 }
7512 Ok(Err(_)) => {
7513 reservation.release_and_disarm();
7514 breaker.record_inconclusive();
7515 Ok(vec![self.route_open_refusal_frame(
7516 ctx,
7517 &caller,
7518 &frame,
7519 &target_module_id,
7520 "relay_waiter_canceled",
7521 "target_unavailable",
7522 "route.bind relay waiter was canceled before the module responded",
7523 )?])
7524 }
7525 Err(_) => {
7526 reservation.release_and_disarm();
7527 if let Some(opened) = breaker.record_timeout(
7531 self.route_bind_breaker_threshold,
7532 self.route_bind_breaker_cooldown,
7533 ) {
7534 warn!(
7535 module_id = %target_module_id,
7536 consecutive_timeouts = opened.consecutive_timeouts,
7537 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
7538 reopened_after_probe = opened.reopened_after_probe,
7539 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
7540 );
7541 }
7542 tracing::warn!(
7550 module_id = %target_module_id,
7551 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
7552 "route.bind relay timed out: module did not ack within budget"
7553 );
7554 Ok(vec![self.route_open_refusal_frame(
7555 ctx,
7556 &caller,
7557 &frame,
7558 &target_module_id,
7559 "relay_timed_out",
7560 "module_timeout",
7561 format!(
7562 "module_id '{target_module_id}' did not answer route.bind within {:?}",
7563 route_bind_relay_timeout
7564 ),
7565 )?])
7566 }
7567 }
7568 }
7569
7570 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7571 let response = ClientControlResponse::SupervisorSpawnSnapshot {
7572 snapshot: self.supervisor.spawn_snapshot(),
7573 };
7574 Ok(vec![control_response_body_frame(
7575 &frame,
7576 &response,
7577 "ClientControlResponse::SupervisorSpawnSnapshot",
7578 )?])
7579 }
7580
7581 fn handle_supervisor_spawn_subscribe(
7582 &self,
7583 ctx: &RouteCtx,
7584 frame: Frame,
7585 since: Option<SpawnCursor>,
7586 ) -> Result<Vec<Frame>, RouterError> {
7587 match self.supervisor.subscribe_spawns(
7588 ctx.connection_id,
7589 frame.header.corr,
7590 response_version(&frame),
7591 since,
7592 ctx.egress.clone(),
7593 ) {
7594 Ok(()) => Ok(Vec::new()),
7595 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
7596 Ok(vec![control_error_body_frame(
7597 &frame,
7598 ErrorBody {
7599 code: "spawn_cursor_incarnation_mismatch".to_string(),
7600 message: "spawn cursor belongs to a different daemon incarnation"
7601 .to_string(),
7602 detail: Some(serde_json::json!({
7603 "current_daemon_incarnation": current
7604 })),
7605 },
7606 )?])
7607 }
7608 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
7609 &frame,
7610 ErrorBody {
7611 code: "spawn_cursor_too_old".to_string(),
7612 message: "spawn cursor predates the retained event ring".to_string(),
7613 detail: Some(serde_json::json!({
7614 "oldest_retained_cursor": oldest
7615 })),
7616 },
7617 )?]),
7618 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
7619 0,
7620 frame.header.corr,
7621 format!("failed to open supervisor spawn subscription: {error}"),
7622 )),
7623 }
7624 }
7625
7626 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7627 let handler = self.clone();
7630 let runtime = tokio::runtime::Handle::current();
7631 let subscriber = tracing::dispatcher::get_default(Clone::clone);
7634 let corr = frame.header.corr;
7635 tokio::task::spawn_blocking(move || {
7636 runtime.block_on(
7637 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
7638 .with_subscriber(subscriber),
7639 )
7640 })
7641 .await
7642 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
7643 }
7644
7645 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7646 use crate::config_gate::runtime::ApprovedDaemonView;
7647 let publication = self
7648 .config_runtime
7649 .as_ref()
7650 .and_then(|owner| owner.polled_snapshot().ok());
7651 let approved = publication
7652 .as_ref()
7653 .map(|view| view.approved_daemon.clone())
7654 .unwrap_or_else(|| {
7655 if self.config_runtime.is_some() {
7656 ApprovedDaemonView::Unavailable
7657 } else {
7658 ApprovedDaemonView::NotStamped
7659 }
7660 });
7661 let generation = self
7662 .registry
7663 .generation()
7664 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7665 let supervised = self.supervisor.list();
7666 let mut statuses = BTreeMap::new();
7667 let mut not_running = BTreeSet::new();
7668 for module in &supervised {
7669 let status = module.status_for_control("list").map_err(|err| {
7670 RouterError::backend(
7671 0,
7672 frame.header.corr,
7673 format!("failed to read supervisor status: {err}"),
7674 )
7675 })?;
7676 if !status.enabled || !status.process_alive {
7677 not_running.insert(module.module_id().to_owned());
7678 }
7679 statuses.insert(module.module_id().to_owned(), status);
7680 }
7681 let mut process_resources = BTreeMap::new();
7684 for module in &supervised {
7685 process_resources.insert(
7686 module.module_id().to_owned(),
7687 crate::subtree_resources::read_process(module.clone()).await,
7688 );
7689 }
7690 let mut image_readings = BTreeMap::new();
7693 for module in &supervised {
7694 image_readings.insert(
7695 module.module_id().to_owned(),
7696 module.running_image_agreement().await,
7697 );
7698 }
7699 let readings = self.resource_tracker.list(supervised.clone());
7700 let mut modules = Vec::new();
7701 for module in supervised {
7702 let status = statuses
7703 .remove(module.module_id())
7704 .expect("each supervised module has a status snapshot");
7705 let (configured, _) = module.configuration().map_err(|err| {
7706 RouterError::backend(
7707 0,
7708 frame.header.corr,
7709 format!("failed to read module configuration: {err}"),
7710 )
7711 })?;
7712 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
7713 RouterError::backend(
7714 0,
7715 frame.header.corr,
7716 format!("failed to read retry time: {err}"),
7717 )
7718 })?;
7719 let image = image_readings
7720 .remove(module.module_id())
7721 .expect("each supervised module has an image snapshot");
7722 let resources = process_resources.remove(module.module_id());
7723 let mut verdict =
7724 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
7725 verdict.spec = Some(launch_spec_agreement(
7726 &configured,
7727 status.process_alive,
7728 status.launch_spec_digest.as_ref(),
7729 ));
7730 let pending_reload = Some(verdict);
7731 let pending_entry = match &approved {
7732 ApprovedDaemonView::Approved { config, .. } => config
7733 .modules
7734 .iter()
7735 .find(|m| m.module_id == module.module_id()),
7736 _ => None,
7737 };
7738 let pending = self
7739 .pending_module_fields(
7740 &module,
7741 pending_entry,
7742 false,
7743 Self::registration_channel_in_force(
7744 publication
7745 .as_ref()
7746 .and_then(|view| view.registration_declaration(module.module_id()))
7747 .as_deref(),
7748 pending_entry.map(|entry| entry.registration_channel),
7749 ),
7750 )
7751 .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
7752 let script = configured.script.as_ref().map(|entry| {
7753 let approved = self
7754 .config_runtime
7755 .as_ref()
7756 .and_then(|owner| owner.approved_script(&status.module_id, &entry.path));
7757 crate::supervise::script::status(entry, approved.as_ref())
7758 });
7759 let configured_enabled = match &approved {
7760 ApprovedDaemonView::Approved {
7761 config: approved, ..
7762 } => approved
7763 .modules
7764 .iter()
7765 .find(|module| module.module_id == status.module_id)
7766 .map(|module| module.enabled),
7767 ApprovedDaemonView::NotStamped => self
7768 .capability_evaluator
7769 .configured_enabled(&status.module_id),
7770 ApprovedDaemonView::Unavailable => None,
7771 };
7772 let admission_refused = status
7773 .admission_refused
7774 .clone()
7775 .filter(|_| status.enabled && !status.process_alive)
7776 .filter(|_| {
7777 publication
7778 .as_ref()
7779 .and_then(|view| view.registration_declaration(&status.module_id))
7780 .as_deref()
7781 == Some("undeclared")
7782 });
7783 let mut entry = SupervisorEntry::new(
7786 status.module_id,
7787 status.state.to_string(),
7788 status.enabled,
7789 status.live,
7790 status.health.status,
7791 )
7792 .with_configured_enabled(configured_enabled)
7793 .with_script(script)
7794 .with_admission_refused(admission_refused)
7795 .with_next_retry_at_ms(next_retry_at_ms)
7796 .with_launch_nonce_env(Some(!cfg!(unix)))
7797 .with_launch_nonce_source(status.launch_nonce_source)
7798 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
7799 .with_protocol(status.protocol)
7800 .with_pending_reload(pending_reload)
7801 .with_last_probe_ms(status.health.last_probe_ms)
7802 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
7803 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
7804 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
7805 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
7806 .with_restart_count(Some(status.restart_count))
7807 .with_max_restarts(Some(status.max_restarts))
7808 .with_lifetime_restarts(Some(status.lifetime_restarts))
7809 .with_spawn_generation(Some(status.spawn_generation))
7810 .with_restart_window_secs(Some(status.restart_window.as_secs()))
7811 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
7812 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
7813 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
7814 .with_resources(resources);
7815 entry.pending = pending;
7816 modules.push(entry);
7817 }
7818 for module in &mut modules {
7819 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
7820 module.tree = tree.clone();
7821 module.cpu_last_hour = hour.clone();
7822 }
7823 module.tree_unavailable = readings
7824 .reasons
7825 .get(&module.module_id)
7826 .map(|reason| (*reason).to_owned());
7827 if module.tree.is_none() {
7828 module.tree_unavailable = Some(
7829 if not_running.contains(&module.module_id) {
7830 "not_running"
7831 } else if let Some(reason) = readings.reasons.get(&module.module_id) {
7832 reason
7833 } else {
7834 "not_yet_measured"
7835 }
7836 .to_owned(),
7837 );
7838 }
7839 }
7840 let response = ClientControlResponse::SupervisorList {
7841 generation,
7842 modules,
7843 daemon: readings.daemon,
7844 agent_commands: readings.agent_commands,
7845 };
7846 Ok(vec![control_response_body_frame(
7847 &frame,
7848 &response,
7849 "ClientControlResponse::SupervisorList",
7850 )?])
7851 }
7852
7853 fn handle_supervisor_stderr_tail(
7854 &self,
7855 frame: Frame,
7856 module_id: String,
7857 max_lines: Option<u32>,
7858 max_bytes: Option<u32>,
7859 ) -> Result<Vec<Frame>, RouterError> {
7860 let Some(module) = self.supervisor.get(&module_id) else {
7861 return Ok(vec![control_error_frame(
7862 &frame,
7863 "unknown_module",
7864 format!("module_id '{module_id}' is not supervised"),
7865 )?]);
7866 };
7867
7868 let snapshot = module.stderr_tail(
7869 max_lines.map(|value| value as usize),
7870 max_bytes.map(|value| value as usize),
7871 );
7872
7873 let response = ClientControlResponse::SupervisorStderrTail {
7874 module_id,
7875 tail: StderrTail {
7876 capture: match snapshot.capture {
7877 CaptureState::Captured => StderrCaptureState::Captured,
7878 CaptureState::Incomplete { reason } => {
7879 StderrCaptureState::Incomplete { reason }
7880 }
7881 CaptureState::NotCaptured { reason } => {
7882 StderrCaptureState::NotCaptured { reason }
7883 }
7884 },
7885 entries: snapshot
7886 .entries
7887 .into_iter()
7888 .map(|entry| match entry {
7889 TailEntry::Line {
7890 text,
7891 truncated,
7892 at_ms,
7893 } => StderrTailEntry::Line {
7894 text,
7895 truncated,
7896 at_ms,
7897 },
7898 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
7899 })
7900 .collect(),
7901 dropped_lines: snapshot.dropped_lines,
7902 },
7903 };
7904 Ok(vec![control_response_body_frame(
7905 &frame,
7906 &response,
7907 "ClientControlResponse::SupervisorStderrTail",
7908 )?])
7909 }
7910
7911 async fn observe_control_wait<F: std::future::Future>(
7912 &self,
7913 ctx: &RouteCtx,
7914 target: Option<&str>,
7915 kind: &'static str,
7916 future: F,
7917 ) -> F::Output {
7918 self.counters
7919 .observe_reader_wait(
7920 || {
7921 let module = self
7922 .forwarding
7923 .module_id_for_connection(ctx.connection_id)
7924 .ok()
7925 .flatten();
7926 let principal = if module.is_some() { "module" } else { "direct" };
7927 (
7928 self.resolve_reader_wait_target(kind, target)
7929 .or(module)
7930 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
7931 principal.into(),
7932 )
7933 },
7934 kind,
7935 future,
7936 )
7937 .await
7938 }
7939
7940 pub(crate) fn resolve_reader_wait_target(
7943 &self,
7944 op: &str,
7945 selector: Option<&str>,
7946 ) -> Option<String> {
7947 let selector = selector?;
7948 match op {
7949 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
7950 .supervisor
7951 .get(selector)
7952 .map(|module| module.module_id().to_owned()),
7953 ops::SUPERVISOR_HEALTH_PROBE => self
7954 .registry
7955 .get_module(selector)
7956 .ok()
7957 .flatten()
7958 .map(|registration| registration.manifest.module_id),
7959 _ => None,
7960 }
7961 }
7962
7963 async fn handle_supervisor_terminals(
7964 &self,
7965 ctx: &RouteCtx,
7966 frame: Frame,
7967 module_id: String,
7968 ) -> Result<Vec<Frame>, RouterError> {
7969 let Some(module) = self.supervisor.get(&module_id) else {
7970 return Ok(vec![control_error_frame(
7971 &frame,
7972 "unknown_module",
7973 format!("module_id '{module_id}' is not supervised"),
7974 )?]);
7975 };
7976
7977 let terminals = self
7980 .observe_control_wait(
7981 ctx,
7982 Some(module.module_id()),
7983 ops::SUPERVISOR_TERMINALS,
7984 module.read_durable_terminal_history(),
7985 )
7986 .await
7987 .map_err(|error| {
7988 RouterError::backend(
7989 0,
7990 frame.header.corr,
7991 format!("failed to read terminal history: {error}"),
7992 )
7993 })?;
7994 let response = ClientControlResponse::SupervisorTerminals {
7995 module_id,
7996 terminals,
7997 };
7998 Ok(vec![control_response_body_frame(
7999 &frame,
8000 &response,
8001 "ClientControlResponse::SupervisorTerminals",
8002 )?])
8003 }
8004
8005 fn handle_supervisor_routes(
8006 &self,
8007 frame: Frame,
8008 module_id: Option<String>,
8009 ) -> Result<Vec<Frame>, RouterError> {
8010 let modules = self
8011 .forwarding
8012 .route_census(module_id.as_deref())
8013 .map_err(RouterError::Forwarding)?
8014 .into_iter()
8015 .map(|(module_id, routes)| SupervisorRouteModule {
8016 module_id,
8017 routes: routes
8018 .into_iter()
8019 .map(|route| SupervisorRoute {
8020 consumer: match route.principal {
8021 Principal::Reserved { module_id } => {
8022 SupervisorRouteConsumer::Reserved { module_id }
8023 }
8024 Principal::Direct | Principal::Unverified => {
8025 SupervisorRouteConsumer::Direct {
8026 connection_id: route.goodbye_target.connection_id.get(),
8027 }
8028 }
8029 },
8030 age_ms: Instant::now()
8031 .saturating_duration_since(route.bound_at)
8032 .as_millis()
8033 .try_into()
8034 .unwrap_or(u64::MAX),
8035 draining: route.draining,
8036 drain_reason: route.drain_reason,
8037 })
8038 .collect(),
8039 })
8040 .collect();
8041 let response = ClientControlResponse::SupervisorRoutes { modules };
8042 Ok(vec![control_response_body_frame(
8043 &frame,
8044 &response,
8045 "ClientControlResponse::SupervisorRoutes",
8046 )?])
8047 }
8048
8049 fn running_registration_method(
8050 &self,
8051 module: &str,
8052 ) -> Option<subc_control::RegistrationMethod> {
8053 let registration = self.registry.get_module(module).ok()??;
8054 let connection = registration.connection_id;
8055 let channel = self.supervisor.registration_spawn(connection).is_some();
8056 let nonce = if channel {
8057 None
8058 } else {
8059 self.hello_launch_nonces
8060 .lock()
8061 .ok()?
8062 .nonce(connection)
8063 .map(str::to_owned)
8064 };
8065 if !channel && nonce.is_none() {
8066 return None;
8067 }
8068 self.supervisor
8069 .registration_snapshot_is_current_live(
8070 &self.registry,
8071 module,
8072 connection,
8073 nonce.as_deref(),
8074 )
8075 .then_some(if channel {
8076 subc_control::RegistrationMethod::Channel
8077 } else {
8078 subc_control::RegistrationMethod::Nonce
8079 })
8080 }
8081
8082 async fn handle_supervisor_provenance(
8083 &self,
8084 ctx: &RouteCtx,
8085 frame: Frame,
8086 module_id: Option<String>,
8087 ) -> Result<Vec<Frame>, RouterError> {
8088 let targeted = module_id.is_some();
8089 let selected = if let Some(module_id) = module_id {
8090 let Some(module) = self.supervisor.get(&module_id) else {
8091 return Ok(vec![control_error_frame(
8092 &frame,
8093 "unknown_module",
8094 format!("module_id '{module_id}' is not supervised"),
8095 )?]);
8096 };
8097 vec![module]
8098 } else {
8099 self.supervisor.list()
8100 };
8101
8102 let target = targeted.then(|| selected[0].module_id());
8103 let declarations = if let Some(owner) = &self.config_runtime {
8104 owner
8105 .on_blocking(|owner| owner.registration_declarations())
8106 .await
8107 .ok()
8108 } else {
8109 None
8110 };
8111 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
8112 let mut modules = Vec::with_capacity(selected.len());
8113 let mut extensions = Vec::with_capacity(selected.len());
8114 for module in &selected {
8115 let (status, observed_image) = module
8116 .status_and_running_image_agreement()
8117 .await
8118 .map_err(|err| {
8119 RouterError::backend(
8120 0,
8121 frame.header.corr,
8122 format!("failed to read supervisor status: {err}"),
8123 )
8124 })?;
8125 let module_declared = self
8126 .registry
8127 .get_module(&status.module_id)
8128 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
8129 .and_then(|registration| registration.manifest.provenance)
8130 .map(|build| ModuleDeclaredProvenance::Reported { build })
8131 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
8132 #[cfg(test)]
8133 let running_image = match &self.provenance_probe_override {
8134 Some(result) => result.clone(),
8135 None => observed_image,
8136 };
8137 #[cfg(not(test))]
8138 let running_image = observed_image;
8139 let registration = self.running_registration_method(&status.module_id);
8140 let mut extension = subc_control::SupervisorModuleProvenanceExt::new()
8141 .with_registration(registration);
8142 extension.registration_declaration = declarations
8143 .as_ref()
8144 .and_then(|entries| entries.get(&status.module_id))
8145 .cloned();
8146 if status.process_alive && status.registration_declaration.is_some() {
8147 extension.registration_declaration = status.registration_declaration.clone();
8148 }
8149 extensions.push(extension);
8150 modules.push(SupervisorModuleProvenance {
8151 module_id: status.module_id,
8152 module_declared,
8153 daemon_observed: SupervisorObservedProcess {
8154 launch_nonce_source: status.launch_nonce_source,
8155 pid: status.pid,
8156 spawned_at_ms: status.spawned_at_ms,
8157 spawned_from: status.spawned_from,
8158 running_image,
8159 },
8160 });
8161 }
8162 let daemon = SupervisorDaemonProvenance {
8163 daemon_build: self.daemon_provenance.build.clone(),
8164 daemon_observed: DaemonObservedProcess {
8165 pid: self.daemon_provenance.pid,
8166 started_at_ms: self
8167 .daemon_provenance
8168 .start_clock
8169 .map(|clock| clock.started_at_ms())
8170 .or(self.daemon_provenance.started_at_ms),
8171 running_image: self
8172 .daemon_provenance
8173 .probe
8174 .observe(
8175 self.daemon_provenance.pid,
8176 self.daemon_provenance.executable_path.as_deref(),
8177 self.daemon_provenance.executable_identity.clone(),
8178 self.daemon_provenance.process_start_time,
8179 )
8180 .await,
8181 },
8182 };
8183 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
8184 let mut response = serde_json::to_value(response)
8185 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
8186 for (entry, extension) in response["modules"]
8187 .as_array_mut()
8188 .expect("provenance modules are an array")
8189 .iter_mut()
8190 .zip(extensions)
8191 {
8192 extension.write_to(entry).map_err(|error| {
8193 RouterError::backend(0, frame.header.corr, error.to_string())
8194 })?;
8195 }
8196 Ok(vec![control_response_body_frame(
8197 &frame,
8198 &response,
8199 "ClientControlResponse::SupervisorProvenance",
8200 )?])
8201 })
8202 .await
8203 }
8204
8205 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
8206 self.refresh_capability_requirements();
8207 let publication = self
8208 .config_runtime
8209 .as_ref()
8210 .and_then(|owner| owner.polled_snapshot().ok());
8211 let generation = self
8212 .registry
8213 .generation()
8214 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
8215 let modules = self
8216 .supervisor
8217 .list()
8218 .into_iter()
8219 .map(|module| {
8220 let status = module.status_for_control("health").map_err(|err| {
8221 RouterError::backend(
8222 0,
8223 frame.header.corr,
8224 format!("failed to read supervisor health: {err}"),
8225 )
8226 })?;
8227 let module_id = status.module_id;
8228 let capability_detail = self
8229 .capability_evaluator
8230 .required_problem_detail(&module_id);
8231 let declaration_warning = self.registry.get_module(&module_id).ok().flatten()
8232 .filter(|registration| registration.manifest.registration_channel == Some(false))
8233 .filter(|_| publication.as_ref().and_then(|view| view.registration_declaration(&module_id))
8234 .as_deref().map(|next| if status.process_alive { status.registration_declaration.as_deref().unwrap_or(next) } else { next }) == Some("declared"))
8235 .map(|_| format!("{module_id}: HELLO registration_channel is false despite the approved channel declaration"));
8236 let capability_detail = append_capability_problem_detail(capability_detail, declaration_warning);
8237 Ok(SupervisorHealthEntry {
8238 module_id,
8239 status: status.health.status,
8240 detail: append_capability_problem_detail(
8241 status.health.detail,
8242 capability_detail,
8243 ),
8244 metrics: status.health.metrics,
8245 consecutive_failures: status.health.consecutive_failures,
8246 late_answer_count: status.health.late_answer_count,
8247 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
8248 last_action: status.health.last_action,
8249 last_action_ms: status.health.last_action_ms,
8250 last_probe_ms: status.health.last_probe_ms,
8251 })
8252 })
8253 .collect::<Result<Vec<_>, RouterError>>()?;
8254 let response = ClientControlResponse::SupervisorHealth {
8255 generation,
8256 modules,
8257 };
8258 Ok(vec![control_response_body_frame(
8259 &frame,
8260 &response,
8261 "ClientControlResponse::SupervisorHealth",
8262 )?])
8263 }
8264
8265 async fn handle_supervisor_restart(
8266 &self,
8267 frame: Frame,
8268 module_id: String,
8269 drain_timeout_ms: Option<u64>,
8270 requester: Principal,
8271 ) -> Result<Vec<Frame>, RouterError> {
8272 let operation_lock = self.supervisor.operation_lock();
8273 let _operation_guard = operation_lock.lock().await;
8274 let Some(module) = self.supervisor.get(&module_id) else {
8275 return Ok(vec![control_error_frame(
8276 &frame,
8277 "unknown_module",
8278 format!("module_id '{module_id}' is not supervised"),
8279 )?]);
8280 };
8281
8282 self.route_outages.mark_operator_action(&module_id);
8283 if let Err(err) = module
8284 .restart_requested(drain_timeout_ms, Some(requester))
8285 .await
8286 {
8287 self.route_outages
8288 .operator_action_ended_unrefused(&module_id);
8289 let (code, message) = match err {
8290 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8291 crate::supervise::SuperviseError::Disabled { .. } => {
8292 ("module_disabled", err.to_string())
8293 }
8294 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8295 ("swap_in_progress", err.to_string())
8296 }
8297 _ => (
8298 "target_unavailable",
8299 format!("failed to restart module_id '{module_id}': {err}"),
8300 ),
8301 };
8302 return Ok(vec![control_error_frame(&frame, code, message)?]);
8303 }
8304
8305 let response = ClientControlResponse::SupervisorAck {
8306 module_id,
8307 applied: true,
8308 };
8309 Ok(vec![control_response_body_frame(
8310 &frame,
8311 &response,
8312 "ClientControlResponse::SupervisorAck",
8313 )?])
8314 }
8315
8316 async fn handle_supervisor_swap(
8320 &self,
8321 frame: Frame,
8322 module_id: String,
8323 ready_timeout_ms: Option<u64>,
8324 requester: Principal,
8325 ) -> Result<Vec<Frame>, RouterError> {
8326 let module = {
8333 let operation_lock = self.supervisor.operation_lock();
8334 let _operation_guard = operation_lock.lock().await;
8335 self.supervisor.get(&module_id)
8336 };
8337 let Some(module) = module else {
8338 return Ok(vec![control_error_frame(
8339 &frame,
8340 "unknown_module",
8341 format!("module_id '{module_id}' is not supervised"),
8342 )?]);
8343 };
8344
8345 self.route_outages.mark_operator_action(&module_id);
8346 if let Err(err) = module
8347 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
8348 .await
8349 {
8350 self.route_outages
8351 .operator_action_ended_unrefused(&module_id);
8352 use crate::supervise::SuperviseError;
8353 let message = err.to_string();
8354 let error = match err {
8355 SuperviseError::ConfigHeld { .. } => ErrorBody::new("config_gated", message),
8356 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
8357 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
8358 code: "swap_refused".to_string(),
8359 message,
8360 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
8361 },
8362 SuperviseError::SwapFailed {
8363 arm,
8364 candidate_exit,
8365 ..
8366 } => ErrorBody {
8367 code: "swap_failed".to_string(),
8368 message,
8369 detail: Some(serde_json::json!({
8370 "arm": arm.as_str(),
8371 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
8372 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
8373 })),
8374 },
8375 _ => ErrorBody::new(
8376 "target_unavailable",
8377 format!("failed to swap module_id '{module_id}': {message}"),
8378 ),
8379 };
8380 return Ok(vec![control_error_body_frame(&frame, error)?]);
8381 }
8382 self.route_outages
8386 .operator_action_ended_unrefused(&module_id);
8387
8388 let response = ClientControlResponse::SupervisorAck {
8389 module_id,
8390 applied: true,
8391 };
8392 Ok(vec![control_response_body_frame(
8393 &frame,
8394 &response,
8395 "ClientControlResponse::SupervisorAck",
8396 )?])
8397 }
8398
8399 async fn handle_supervisor_reload(
8400 &self,
8401 frame: Frame,
8402 module_id: String,
8403 requester: Principal,
8404 ) -> Result<Vec<Frame>, RouterError> {
8405 let operation_lock = self.supervisor.operation_lock();
8406 let _operation_guard = operation_lock.lock().await;
8407 let Some(module) = self.supervisor.get(&module_id) else {
8408 return Ok(vec![control_error_frame(
8409 &frame,
8410 "unknown_module",
8411 format!("module_id '{module_id}' is not supervised"),
8412 )?]);
8413 };
8414
8415 self.route_outages.mark_operator_action(&module_id);
8416 if let Err(err) = module.reload_requested(Some(requester)).await {
8417 self.route_outages
8418 .operator_action_ended_unrefused(&module_id);
8419 let (code, message) = match err {
8420 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8421 crate::supervise::SuperviseError::Disabled { .. } => {
8422 ("module_disabled", err.to_string())
8423 }
8424 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8425 ("swap_in_progress", err.to_string())
8426 }
8427 _ => (
8428 "reload_failed",
8429 format!("failed to reload module_id '{module_id}': {err}"),
8430 ),
8431 };
8432 return Ok(vec![control_error_frame(&frame, code, message)?]);
8433 }
8434
8435 let response = ClientControlResponse::SupervisorAck {
8436 module_id,
8437 applied: true,
8438 };
8439 Ok(vec![control_response_body_frame(
8440 &frame,
8441 &response,
8442 "ClientControlResponse::SupervisorAck",
8443 )?])
8444 }
8445
8446 async fn handle_supervisor_rescan(
8450 &self,
8451 frame: Frame,
8452 mode: RescanMode,
8453 requester: Principal,
8454 ) -> Result<Vec<Frame>, RouterError> {
8455 let preview = mode.previews();
8456 let mut approved_preview = None;
8457 if matches!(mode, RescanMode::Preview) {
8458 if let Some(owner) = &self.config_runtime {
8459 match owner
8460 .on_blocking(|owner| {
8461 owner.observe_file_proposal()?;
8462 owner.approved_daemon_configuration()
8463 })
8464 .await
8465 {
8466 Ok(config) => approved_preview = config,
8467 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8468 }
8469 }
8470 }
8471 let Some(context) = self.rescan.clone() else {
8472 return Ok(vec![control_error_frame(
8473 &frame,
8474 "rescan_unavailable",
8475 "the daemon was not started with a reloadable config path".to_string(),
8476 )?]);
8477 };
8478
8479 let operation_lock = self.supervisor.operation_lock();
8480 let _operation_guard = operation_lock.lock().await;
8481 let loaded = match self.load_rescan_config(&mode, &context) {
8485 Ok(config) => config,
8486 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
8487 };
8488 let Some((config, activity_config)) = loaded else {
8496 return Ok(vec![control_error_frame(
8497 &frame,
8498 "invalid_daemon_config",
8499 format!(
8500 "daemon config not found at {}; refusing to rescan (an absent config would \
8501 retire every supervised module)",
8502 context.config_path.display()
8503 ),
8504 )?]);
8505 };
8506 let (
8507 configured_port,
8508 storage_config,
8509 admission_facts_carrier_module_id,
8510 admission_facts_targets,
8511 scope_authority_owners,
8512 federated_scope_owner,
8513 privileged_operator_prompt,
8514 modules,
8515 reserved_capabilities,
8516 ) = (
8517 config.port,
8518 config.storage,
8519 config.admission_facts_carrier_module_id,
8520 config.admission_facts_targets,
8521 config.scope_authority_owners,
8522 config.federated_scope_owner,
8523 config.privileged_operator_prompt,
8524 config.modules,
8525 config.reserved_capabilities,
8526 );
8527
8528 let mut restart_required = Vec::new();
8537 for section in RestartRequiredSection::ALL {
8538 let changed = match section {
8539 RestartRequiredSection::Port => configured_port != context.configured_port,
8540 RestartRequiredSection::Storage => storage_config != context.storage_config,
8541 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
8542 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
8543 }
8544 RestartRequiredSection::AdmissionFactsTargets => {
8545 admission_facts_targets != context.admission_facts_targets
8546 }
8547 RestartRequiredSection::ScopeAuthorityOwners => {
8548 scope_authority_owners != context.scope_authority_owners
8549 }
8550 RestartRequiredSection::FederatedScopeOwner => {
8551 federated_scope_owner != context.federated_scope_owner
8552 }
8553 RestartRequiredSection::PrivilegedOperatorPrompt => {
8554 privileged_operator_prompt != context.privileged_operator_prompt
8555 }
8556 };
8557 if changed {
8558 restart_required.push(section.label().to_string());
8559 }
8560 }
8561 if !restart_required.is_empty() {
8562 warn!(
8563 config_path = %context.config_path.display(),
8564 sections = %restart_required.join(", "),
8565 "daemon config changed outside the modules section; restart the daemon to apply those changes"
8566 );
8567 }
8568
8569 for configured in &modules {
8570 if let Err(err) = validate_spec(&configured.module_spec()) {
8571 return Ok(vec![control_error_frame(
8572 &frame,
8573 "invalid_daemon_config",
8574 format!("supervisor rescan rejected daemon config: {err}"),
8575 )?]);
8576 }
8577 }
8578
8579 let configured_capabilities = modules
8580 .iter()
8581 .map(|module| (module.module_id.clone(), module.enabled))
8582 .collect::<Vec<_>>();
8583 let preview_capability_warnings = if preview {
8584 let (_, registrations) = self.runtime_capability_snapshot()?;
8585 let current_modules = self
8586 .supervisor
8587 .list()
8588 .into_iter()
8589 .map(|module| module.module_id().to_string())
8590 .collect::<BTreeSet<_>>();
8591 let resulting_modules = configured_capabilities.clone();
8592 let removed = current_modules
8593 .into_iter()
8594 .filter(|module_id| {
8595 !resulting_modules
8596 .iter()
8597 .any(|(configured_id, _)| configured_id == module_id)
8598 })
8599 .collect::<Vec<_>>();
8600 let disabled = match &mode {
8603 RescanMode::Candidate(_) => resulting_modules
8604 .iter()
8605 .filter(|(_, enabled)| !enabled)
8606 .map(|(module_id, _)| module_id.clone())
8607 .collect::<Vec<_>>(),
8608 _ => Vec::new(),
8609 };
8610 self.capability_evaluator.preview_warnings(
8611 resulting_modules,
8612 &removed,
8613 &disabled,
8614 ®istrations,
8615 )
8616 } else {
8617 Vec::new()
8618 };
8619 let reconciled = match (&mode, approved_preview.as_ref()) {
8620 (RescanMode::Preview, Some(approved)) => Ok((
8621 config_gate::preview_approved_daemon(approved, &modules),
8622 Vec::new(),
8623 )),
8624 (RescanMode::Candidate(candidate), _) => self
8625 .preview_candidate(&context, modules, candidate)
8626 .map(|result| (result, Vec::new())),
8627 _ => {
8628 self.reconcile_supervised_modules(
8629 &context.supervisor,
8630 modules,
8631 preview,
8632 requester.clone(),
8633 )
8634 .await
8635 }
8636 };
8637 let (result, changed) = match reconciled {
8638 Ok(result) => result,
8639 Err(message) => {
8640 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
8641 }
8642 };
8643 if !preview {
8644 if let Some(ledger) = &self.activity {
8645 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
8646 ledger.config_changed(&result.added, &result.removed, &changed, requester);
8647 }
8648 if let Err(code) = self.configure_activity(activity_config).await {
8649 warn!(
8650 code,
8651 "activity retention could not be applied during rescan"
8652 );
8653 }
8654 self.capability_evaluator
8655 .configure(configured_capabilities, reserved_capabilities);
8656 self.capability_evaluator.wake_deadline_loop();
8657 self.refresh_capability_requirements();
8658 }
8659 let mut result = result;
8660 result.restart_required = restart_required;
8661 result.capability_warnings = preview_capability_warnings;
8662 let response = ClientControlResponse::SupervisorRescan { result };
8663 Ok(vec![control_response_body_frame(
8664 &frame,
8665 &response,
8666 "ClientControlResponse::SupervisorRescan",
8667 )?])
8668 }
8669
8670 async fn handle_supervisor_release_reserved(
8671 &self,
8672 frame: Frame,
8673 module_id: String,
8674 ) -> Result<Vec<Frame>, RouterError> {
8675 let Some(context) = self.rescan.clone() else {
8676 return Ok(vec![control_error_frame(
8677 &frame,
8678 "release_unavailable",
8679 "reserved-id release requires a daemon started with a reloadable config path",
8680 )?]);
8681 };
8682 let operation_lock = self.supervisor.operation_lock();
8683 let _operation_guard = operation_lock.lock().await;
8684 let approved = if let Some(owner) = &self.config_runtime {
8685 match owner
8688 .on_blocking(|owner| {
8689 owner.observe_file_proposal()?;
8690 owner.approved_daemon_configuration()
8691 })
8692 .await
8693 {
8694 Ok(config) => config,
8695 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8696 }
8697 } else {
8698 None
8699 };
8700 let loaded = if let Some(approved) = approved {
8701 Ok(Some(approved))
8702 } else {
8703 crate::daemon_config::load(&context.config_path)
8704 };
8705 let loaded = match loaded {
8706 Ok(Some(config)) => config,
8707 Ok(None) => {
8708 return Ok(vec![control_error_frame(
8709 &frame,
8710 "invalid_daemon_config",
8711 format!(
8712 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
8713 context.config_path.display()
8714 ),
8715 )?])
8716 }
8717 Err(err) => {
8718 return Ok(vec![control_error_frame(
8719 &frame,
8720 "invalid_daemon_config",
8721 format!("unable to verify reserved-id release against daemon config: {err}"),
8722 )?])
8723 }
8724 };
8725 if loaded
8726 .modules
8727 .iter()
8728 .any(|configured| configured.module_id == module_id)
8729 {
8730 return Ok(vec![control_error_frame(
8731 &frame,
8732 "reserved_module_configured",
8733 format!(
8734 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
8735 ),
8736 )?]);
8737 }
8738 if !self.supervisor.release_retained_reserved_gate(&module_id) {
8739 return Ok(vec![control_error_frame(
8740 &frame,
8741 "reserved_gate_not_retained",
8742 format!(
8743 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
8744 ),
8745 )?]);
8746 }
8747
8748 let response = ClientControlResponse::SupervisorAck {
8749 module_id,
8750 applied: true,
8751 };
8752 Ok(vec![control_response_body_frame(
8753 &frame,
8754 &response,
8755 "ClientControlResponse::SupervisorAck",
8756 )?])
8757 }
8758
8759 fn plan_rescan(
8763 &self,
8764 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8765 ) -> Result<RescanPlan, String> {
8766 let mut current = BTreeMap::new();
8767 for module in self.supervisor.list() {
8768 let (spec, health) = module.configuration().map_err(|err| {
8769 format!(
8770 "failed to read configuration for module_id '{}': {err}",
8771 module.module_id()
8772 )
8773 })?;
8774 let enabled = module
8775 .status()
8776 .map_err(|err| {
8777 format!(
8778 "failed to read status for module_id '{}': {err}",
8779 module.module_id()
8780 )
8781 })?
8782 .enabled;
8783 current.insert(
8784 module.module_id().to_string(),
8785 (module, spec, health, enabled),
8786 );
8787 }
8788 let configured = configured_modules
8789 .into_iter()
8790 .map(|module| (module.module_id.clone(), module))
8791 .collect::<BTreeMap<_, _>>();
8792
8793 let added = configured
8794 .keys()
8795 .filter(|module_id| !current.contains_key(*module_id))
8796 .cloned()
8797 .collect::<Vec<_>>();
8798 let removed = current
8799 .keys()
8800 .filter(|module_id| !configured.contains_key(*module_id))
8801 .cloned()
8802 .collect::<Vec<_>>();
8803 let mut changed_pending_reload = Vec::new();
8804 let mut changed_pending = Vec::new();
8805 let mut configuration_changes = BTreeSet::new();
8806 let mut enabled_changes = BTreeSet::new();
8807 let mut unchanged = 0_u32;
8808 let publication = self
8809 .config_runtime
8810 .as_ref()
8811 .and_then(|owner| owner.polled_snapshot().ok());
8812
8813 for (module_id, configured_module) in &configured {
8814 let Some((module, current_spec, current_health, current_enabled)) =
8815 current.get(module_id)
8816 else {
8817 continue;
8818 };
8819 let launch_changed = *current_spec != configured_module.module_spec();
8823 let configuration_changed =
8824 launch_changed || *current_health != configured_module.health;
8825 let pending = self.pending_module_fields(
8826 module,
8827 Some(configured_module),
8828 configuration_changed,
8829 Self::registration_channel_in_force(
8830 publication
8831 .as_ref()
8832 .and_then(|view| view.registration_declaration(module_id))
8833 .as_deref(),
8834 publication
8835 .as_ref()
8836 .and_then(|view| match &view.approved_daemon {
8837 crate::config_gate::runtime::ApprovedDaemonView::Approved {
8838 config,
8839 ..
8840 } => config
8841 .modules
8842 .iter()
8843 .find(|entry| entry.module_id == *module_id)
8844 .map(|entry| entry.registration_channel),
8845 _ => None,
8846 }),
8847 ),
8848 )?;
8849 let has_pending = !pending.is_empty();
8850 if has_pending {
8851 changed_pending.push(subc_control::SupervisorPendingModule {
8852 module_id: module_id.clone(),
8853 fields: pending
8854 .into_iter()
8855 .map(|mut field| {
8856 field.value = None;
8857 field
8858 })
8859 .collect(),
8860 });
8861 }
8862 let enabled_changed = *current_enabled != configured_module.enabled
8865 || self
8866 .capability_evaluator
8867 .configured_enabled(module_id)
8868 .is_some_and(|enabled| enabled != configured_module.enabled);
8869 if configuration_changed {
8870 configuration_changes.insert(module_id.clone());
8871 }
8872 if launch_changed {
8873 changed_pending_reload.push(module_id.clone());
8874 }
8875 if enabled_changed {
8876 enabled_changes.insert(module_id.clone());
8877 }
8878 if !configuration_changed && !enabled_changed && !has_pending {
8879 unchanged = unchanged.saturating_add(1);
8880 }
8881 }
8882 Ok(RescanPlan {
8883 current,
8884 configured,
8885 added,
8886 removed,
8887 changed_pending_reload,
8888 changed_pending,
8889 configuration_changes,
8890 enabled_changes,
8891 unchanged,
8892 })
8893 }
8894
8895 async fn reconcile_supervised_modules(
8904 &self,
8905 supervisor: &Supervisor,
8906 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8907 preview: bool,
8908 requester: Principal,
8909 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
8910 let plan = self.plan_rescan(configured_modules)?;
8911
8912 if preview {
8918 return Ok((plan.preview_result(), Vec::new()));
8919 }
8920 let changed = plan
8921 .configuration_changes
8922 .union(&plan.enabled_changes)
8923 .cloned()
8924 .collect::<Vec<_>>();
8925 let RescanPlan {
8926 current,
8927 configured,
8928 added,
8929 removed,
8930 changed_pending_reload,
8931 changed_pending,
8932 configuration_changes,
8933 enabled_changes,
8934 unchanged,
8935 } = plan;
8936 if !(added.is_empty()
8941 && removed.is_empty()
8942 && configuration_changes.is_empty()
8943 && changed_pending.is_empty()
8944 && enabled_changes.is_empty())
8945 {
8946 if let Some(context) = &self.rescan {
8947 context
8948 .generation
8949 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8950 }
8951 }
8952
8953 for module_id in &removed {
8954 let module = ¤t
8955 .get(module_id)
8956 .expect("removed module came from current supervisor state")
8957 .0;
8958 module
8959 .retire_requested(Some(requester.clone()))
8960 .await
8961 .map_err(|err| {
8962 format!("failed to retire module_id '{module_id}' during rescan: {err}")
8963 })?;
8964 self.supervisor.record_rescan_removal(module_id);
8998 self.supervisor.retire(module_id);
8999 self.route_outages.forget(module_id);
9000 }
9001
9002 for module_id in configured.keys() {
9003 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
9004 continue;
9005 };
9006 let configured_module = configured
9007 .get(module_id)
9008 .expect("configured module id came from configured map");
9009 if configuration_changes.contains(module_id) {
9010 module
9011 .update_configuration(
9012 configured_module.module_spec(),
9013 configured_module.health.clone(),
9014 configured_module.drain_timeout_ms,
9015 )
9016 .await
9017 .map_err(|err| {
9018 format!(
9019 "failed to update module_id '{module_id}' configuration during rescan: {err}"
9020 )
9021 })?;
9022 }
9023 if *current_enabled != configured_module.enabled {
9026 self.route_outages.mark_operator_action(module_id);
9029 module
9030 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
9031 .await
9032 .map_err(|err| {
9033 self.route_outages.operator_action_ended_unrefused(module_id);
9034 format!(
9035 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
9036 configured_module.enabled
9037 )
9038 })?;
9039 }
9040 }
9041
9042 for module_id in &added {
9043 let configured_module = configured
9044 .get(module_id)
9045 .expect("added module id came from configured map");
9046 supervisor
9047 .supervise_configured_requested_async(
9048 configured_module.module_spec(),
9049 configured_module.enabled,
9050 configured_module.health.clone(),
9051 configured_module.drain_timeout_ms,
9052 configured_module.restart,
9053 configured_module.register_timeout,
9054 Some(requester.clone()),
9055 )
9056 .await
9057 .map_err(|err| {
9058 format!("failed to add module_id '{module_id}' during rescan: {err}")
9059 })?;
9060 }
9061
9062 if let Some(context) = &self.rescan {
9063 let mut entries = context
9064 .module_entries
9065 .lock()
9066 .unwrap_or_else(|e| e.into_inner());
9067 entries.record(&configured, &added, &removed);
9068 }
9069
9070 Ok((
9071 SupervisorRescanResult {
9072 added,
9073 removed,
9074 changed_pending_reload,
9075 changed_pending,
9076 enabled_changes: enabled_changes.iter().cloned().collect(),
9077 unchanged,
9078 preview: false,
9079 restart_required: Vec::new(),
9082 capability_warnings: Vec::new(),
9083 module_actions: Vec::new(),
9084 earlier_pending: Vec::new(),
9085 expect: None,
9086 },
9087 changed,
9088 ))
9089 }
9090
9091 async fn handle_supervisor_set_enabled(
9092 &self,
9093 frame: Frame,
9094 module_id: String,
9095 enabled: bool,
9096 requester: Principal,
9097 ) -> Result<Vec<Frame>, RouterError> {
9098 let operation_lock = self.supervisor.operation_lock();
9099 let _operation_guard = operation_lock.lock().await;
9100 let Some(module) = self.supervisor.get(&module_id) else {
9101 return Ok(vec![control_error_frame(
9102 &frame,
9103 "unknown_module",
9104 format!("module_id '{module_id}' is not supervised"),
9105 )?]);
9106 };
9107
9108 self.route_outages.mark_operator_action(&module_id);
9111 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
9112 Ok(applied) => applied,
9113 Err(err) => {
9114 self.route_outages
9115 .operator_action_ended_unrefused(&module_id);
9116 return Ok(vec![control_error_frame(
9117 &frame,
9118 if err.is_config_gate_error() {
9119 "config_gated"
9120 } else {
9121 "target_unavailable"
9122 },
9123 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
9124 )?]);
9125 }
9126 };
9127 if !applied {
9128 self.route_outages
9131 .operator_action_ended_unrefused(&module_id);
9132 }
9133
9134 self.capability_evaluator.wake_deadline_loop();
9135 self.refresh_capability_requirements();
9136 let response = ClientControlResponse::SupervisorAck { module_id, applied };
9137 Ok(vec![control_response_body_frame(
9138 &frame,
9139 &response,
9140 "ClientControlResponse::SupervisorAck",
9141 )?])
9142 }
9143
9144 async fn handle_supervisor_health_probe(
9145 &self,
9146 ctx: &RouteCtx,
9147 frame: Frame,
9148 module_id: String,
9149 ) -> Result<Vec<Frame>, RouterError> {
9150 self.refresh_capability_requirements();
9151 let Some(registration) = self
9152 .registry
9153 .get_module(&module_id)
9154 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
9155 else {
9156 return Ok(vec![control_error_frame(
9157 &frame,
9158 "unknown_module",
9159 format!("module_id '{module_id}' is not registered"),
9160 )?]);
9161 };
9162
9163 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
9176 {
9177 return Ok(vec![control_error_frame(
9178 &frame,
9179 "health_not_advertised",
9180 format!("module_id '{module_id}' did not advertise health.check"),
9181 )?]);
9182 }
9183
9184 self.observe_control_wait(
9185 ctx,
9186 Some(®istration.manifest.module_id),
9187 ops::SUPERVISOR_HEALTH_PROBE,
9188 async {
9189 let deadline = Instant::now() + self.health_probe_timeout;
9190 let pending = match self.forwarding.begin_module_control_rpc_for(
9191 &module_id,
9192 MODULE_CONTROL_OP_HEALTH_CHECK,
9193 deadline,
9194 ) {
9195 Ok(pending) => pending,
9196 Err(err) => {
9197 return Ok(vec![control_error_frame(
9198 &frame,
9199 forwarding_error_code(&err),
9200 err.to_string(),
9201 )?])
9202 }
9203 };
9204
9205 let PendingModuleControlRpc {
9206 endpoint,
9207 module_sink,
9208 negotiated_ver,
9209 corr: probe_corr,
9210 receiver,
9211 } = pending;
9212 let mut guard =
9213 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
9214 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
9215 .map_err(|err| {
9216 RouterError::backend(
9217 0,
9218 frame.header.corr,
9219 format!("failed to encode health.check request: {err}"),
9220 )
9221 })?;
9222 let probe_frame = Frame::build_with_version(
9223 negotiated_ver,
9224 FrameType::Request,
9225 control_flags(),
9226 0,
9227 0,
9228 probe_corr,
9229 probe_body,
9230 )
9231 .map_err(RouterError::FrameBuild)?;
9232
9233 match timeout_at(deadline, module_sink.send(probe_frame)).await {
9236 Ok(Ok(())) => {}
9237 Ok(Err(err)) => {
9238 return Ok(vec![control_error_frame(
9239 &frame,
9240 "target_unavailable",
9241 err.to_string(),
9242 )?]);
9243 }
9244 Err(_) => {
9245 return Ok(vec![control_error_frame(
9246 &frame,
9247 "module_timeout",
9248 format!(
9249 "module_id '{module_id}' did not answer health.check within {:?}",
9250 self.health_probe_timeout
9251 ),
9252 )?]);
9253 }
9254 }
9255
9256 match timeout_at(deadline, receiver).await {
9257 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => {
9258 Ok(vec![control_error_frame(
9259 &frame,
9260 "invalid_control_body",
9261 "health.check returned an unexpected config response",
9262 )?])
9263 }
9264 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
9265 guard.disarm();
9266 let Some(report) = response.health_report() else {
9267 return Ok(vec![control_error_frame(
9268 &frame,
9269 "invalid_control_body",
9270 "health.check RPC returned a non-health response",
9271 )?]);
9272 };
9273 let HealthReport {
9278 status,
9279 detail,
9280 metrics,
9281 } = report;
9282 let capability_detail = self
9283 .capability_evaluator
9284 .required_problem_detail(&module_id);
9285 let response = ClientControlResponse::SupervisorHealthProbe {
9286 module_id,
9287 status,
9288 detail: append_capability_problem_detail(detail, capability_detail),
9289 metrics,
9290 };
9291 Ok(vec![control_response_body_frame(
9292 &frame,
9293 &response,
9294 "ClientControlResponse::SupervisorHealthProbe",
9295 )?])
9296 }
9297 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
9298 guard.disarm();
9299 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
9304 }
9305 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
9306 guard.disarm();
9307 Ok(vec![control_error_frame(
9308 &frame,
9309 "target_unavailable",
9310 message,
9311 )?])
9312 }
9313 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
9314 guard.disarm();
9315 Ok(vec![control_error_frame(
9316 &frame,
9317 "invalid_control_body",
9318 message,
9319 )?])
9320 }
9321 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
9322 guard.disarm();
9323 Ok(vec![control_error_frame(
9324 &frame,
9325 "invalid_control_body",
9326 format!("expected module-control op '{expected}', got '{actual}'"),
9327 )?])
9328 }
9329 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
9330 guard.disarm();
9331 Ok(vec![control_error_frame(
9332 &frame,
9333 "module_timeout",
9334 format!(
9335 "module_id '{module_id}' answered health.check after {:?}",
9336 self.health_probe_timeout
9337 ),
9338 )?])
9339 }
9340 Ok(Err(_)) => Ok(vec![control_error_frame(
9341 &frame,
9342 "target_unavailable",
9343 "health.check waiter was canceled before the module responded",
9344 )?]),
9345 Err(_) => Ok(vec![control_error_frame(
9346 &frame,
9347 "module_timeout",
9348 format!(
9349 "module_id '{module_id}' did not answer health.check within {:?}",
9350 self.health_probe_timeout
9351 ),
9352 )?]),
9353 }
9354 },
9355 )
9356 .await
9357 }
9358
9359 fn supervisor_status(
9360 &self,
9361 module_id: &str,
9362 corr: u64,
9363 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
9364 self.supervisor
9365 .get(module_id)
9366 .map(|module| {
9367 let warming = module.is_warming_for_control("status").map_err(|err| {
9368 RouterError::backend(
9369 0,
9370 corr,
9371 format!(
9372 "failed to read supervisor warming state for module_id '{module_id}': {err}"
9373 ),
9374 )
9375 })?;
9376 module.status_for_control("status").map_err(|err| {
9377 RouterError::backend(
9378 0,
9379 corr,
9380 format!(
9381 "failed to read supervisor status for module_id '{module_id}': {err}"
9382 ),
9383 )
9384 }).map(|status| (status, warming))
9385 })
9386 .transpose()
9387 }
9388
9389 fn guard_module_control_op(
9390 &self,
9391 frame: &Frame,
9392 module_id: &str,
9393 op: &str,
9394 ) -> Result<Option<Frame>, RouterError> {
9395 if self.module_grants_op(module_id, op, frame.header.corr)? {
9396 return Ok(None);
9397 }
9398
9399 Ok(Some(control_error_frame(
9400 frame,
9401 "op_not_allowed",
9402 format!("module_id '{module_id}' did not grant control op '{op}'"),
9403 )?))
9404 }
9405
9406 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
9407 let Some(registration) = self
9408 .registry
9409 .get_module(module_id)
9410 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
9411 else {
9412 return Ok(false);
9413 };
9414 Ok(module_registration_grants_op(®istration.control_ops, op))
9415 }
9416
9417 fn handle_status_update(
9418 &self,
9419 endpoint: ModuleEndpointId,
9420 frame: Frame,
9421 ) -> Result<Vec<Frame>, RouterError> {
9422 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
9423 Ok(update) => update,
9424 Err(err) => {
9425 if is_known_module_push_op(&frame.body) {
9430 return Ok(vec![control_error_frame(
9431 &frame,
9432 "invalid_control_body",
9433 format!("malformed module control push body: {err}"),
9434 )?]);
9435 }
9436 return Ok(Vec::new());
9437 }
9438 };
9439
9440 match update {
9441 ModuleControlPush::RouteStatus {
9442 route_channel,
9443 route_epoch,
9444 status,
9445 } => {
9446 self.forwarding
9447 .cache_status(endpoint, route_channel, route_epoch, status)
9448 .map_err(RouterError::Forwarding)?;
9449 }
9450 }
9451 Ok(Vec::new())
9452 }
9453
9454 fn handle_route_poll(
9455 &self,
9456 ctx: &RouteCtx,
9457 frame: Frame,
9458 route_channel: u16,
9459 route_epoch: u32,
9460 kind: PollKind,
9461 ) -> Result<Vec<Frame>, RouterError> {
9462 let snapshot = self
9463 .forwarding
9464 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
9465 .map_err(RouterError::Forwarding)?;
9466 let response = match (kind, snapshot) {
9467 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
9468 ClientControlResponse::RoutePoll {
9469 route_channel,
9470 route_epoch,
9471 status,
9472 live: None,
9473 }
9474 }
9475 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9476 route_channel,
9477 route_epoch,
9478 status: None,
9479 live: None,
9480 },
9481 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
9482 let live = self
9508 .process_liveness
9509 .as_ref()
9510 .and_then(|source| source.process_live(&module_id))
9511 .unwrap_or(true);
9512 ClientControlResponse::RoutePoll {
9513 route_channel,
9514 route_epoch,
9515 status: None,
9516 live: Some(live),
9517 }
9518 }
9519 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9520 route_channel,
9521 route_epoch,
9522 status: None,
9523 live: Some(false),
9524 },
9525 };
9526
9527 Ok(vec![control_response_body_frame(
9528 &frame,
9529 &response,
9530 "ClientControlResponse::RoutePoll",
9531 )?])
9532 }
9533
9534 pub(crate) fn observe_module_control_completion(
9535 &self,
9536 completion: ModuleControlRpcCompletion,
9537 ) -> bool {
9538 match completion {
9539 ModuleControlRpcCompletion::Unknown => false,
9540 ModuleControlRpcCompletion::Settled => true,
9541 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
9542 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
9543 info!(
9544 module_id = %module_id,
9545 latency_ms,
9546 "late health.check answer proves the module is alive"
9547 );
9548 match self
9549 .supervisor
9550 .record_late_health_answer(&module_id, latency_ms)
9551 {
9552 Ok(true) => {}
9553 Ok(false) => debug!(
9554 module_id = %module_id,
9555 latency_ms,
9556 "late health.check answer has no active supervisor snapshot"
9557 ),
9558 Err(err) => warn!(
9559 module_id = %module_id,
9560 latency_ms,
9561 error = %err,
9562 "failed to record late health.check answer"
9563 ),
9564 }
9565 true
9566 }
9567 }
9568 }
9569
9570 fn refuse_to_end_module_connection_for_a_client(
9591 &self,
9592 module_connection_id: ConnectionId,
9593 corr: u64,
9594 err: ForwardingError,
9595 ) -> Result<(), RouterError> {
9596 if let ForwardingError::ConnectionClosing { connection_id } = err {
9597 if connection_id != module_connection_id {
9598 warn!(
9599 module_connection_id = module_connection_id.get(),
9600 client_connection_id = connection_id.get(),
9601 corr,
9602 "dropping a route.bind response for a closing client; the module connection keeps serving"
9603 );
9604 return Ok(());
9605 }
9606 }
9607 Err(RouterError::Forwarding(err))
9608 }
9609
9610 fn handle_module_relay_response(
9611 &self,
9612 connection_id: ConnectionId,
9613 frame: Frame,
9614 ) -> Result<Vec<Frame>, RouterError> {
9615 let mut secondary_error = None;
9616 let outcome = match frame.header.ty {
9617 FrameType::Response => match module_control_response_probe(&frame.body) {
9618 Ok(probe) if probe.op == "route.bind" => {
9619 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9620 Ok(ModuleControlResponse::RouteBindAck {}) => {
9621 RouteBindRelayOutcome::Accepted
9622 }
9623 Ok(other) => {
9624 let message =
9625 format!("route.bind response carried unexpected body: {other:?}");
9626 secondary_error = Some(control_error_frame(
9627 &frame,
9628 "invalid_control_body",
9629 message.clone(),
9630 )?);
9631 RouteBindRelayOutcome::ModuleGone(message)
9632 }
9633 Err(err) => {
9634 let message = format!("malformed route.bind response body: {err}");
9635 secondary_error = Some(control_error_frame(
9636 &frame,
9637 "invalid_control_body",
9638 message.clone(),
9639 )?);
9640 RouteBindRelayOutcome::ModuleGone(message)
9641 }
9642 }
9643 }
9644 Ok(probe) => {
9645 let outcome = if matches!(
9646 probe.op.as_str(),
9647 "config.review" | "consent/v1" | "projects.enumerate"
9648 ) {
9649 ModuleControlRpcOutcome::RawResponse(frame.body.clone())
9650 } else {
9651 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9652 Ok(response) => ModuleControlRpcOutcome::Response(response),
9653 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9654 "malformed {} response body: {err}",
9655 probe.op
9656 )),
9657 }
9658 };
9659 let completion = self
9660 .forwarding
9661 .complete_module_control_rpc(
9662 connection_id,
9663 frame.header.corr,
9664 Some(&probe.op),
9665 outcome,
9666 )
9667 .map_err(RouterError::Forwarding)?;
9668 if !self.observe_module_control_completion(completion) {
9669 debug!(
9670 connection_id = connection_id.get(),
9671 corr = frame.header.corr,
9672 op = %probe.op,
9673 "dropping late or unknown module-control RPC response"
9674 );
9675 }
9676 return Ok(Vec::new());
9677 }
9678 Err(err) => {
9679 if let Some(expected_op) = self
9680 .forwarding
9681 .pending_module_control_op(connection_id, frame.header.corr)
9682 .map_err(RouterError::Forwarding)?
9683 {
9684 let completion = self
9685 .forwarding
9686 .complete_module_control_rpc(
9687 connection_id,
9688 frame.header.corr,
9689 None,
9690 ModuleControlRpcOutcome::MalformedResponse(format!(
9691 "malformed {expected_op} response body: {err}"
9692 )),
9693 )
9694 .map_err(RouterError::Forwarding)?;
9695 if !self.observe_module_control_completion(completion) {
9696 debug!(
9697 connection_id = connection_id.get(),
9698 corr = frame.header.corr,
9699 "dropping late malformed module-control RPC response"
9700 );
9701 }
9702 return Ok(Vec::new());
9703 }
9704 let message = format!("malformed route.bind response body: {err}");
9705 secondary_error = Some(control_error_frame(
9706 &frame,
9707 "invalid_control_body",
9708 message.clone(),
9709 )?);
9710 RouteBindRelayOutcome::ModuleGone(message)
9711 }
9712 },
9713 FrameType::Error => {
9714 if self
9715 .forwarding
9716 .pending_module_control_op(connection_id, frame.header.corr)
9717 .map_err(RouterError::Forwarding)?
9718 .is_some()
9719 {
9720 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
9721 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
9722 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9723 "malformed module-control ERROR body: {err}"
9724 )),
9725 };
9726 let completion = self
9727 .forwarding
9728 .complete_module_control_rpc(
9729 connection_id,
9730 frame.header.corr,
9731 None,
9732 outcome,
9733 )
9734 .map_err(RouterError::Forwarding)?;
9735 if !self.observe_module_control_completion(completion) {
9736 debug!(
9737 connection_id = connection_id.get(),
9738 corr = frame.header.corr,
9739 "dropping late or unknown module-control RPC error"
9740 );
9741 }
9742 return Ok(Vec::new());
9743 }
9744 match serde_json::from_slice::<ErrorBody>(&frame.body) {
9745 Ok(body) => RouteBindRelayOutcome::Rejected(body),
9746 Err(err) => {
9747 let message = format!("malformed route.bind ERROR body: {err}");
9748 secondary_error = Some(control_error_frame(
9749 &frame,
9750 "invalid_control_body",
9751 message.clone(),
9752 )?);
9753 RouteBindRelayOutcome::ModuleGone(message)
9754 }
9755 }
9756 }
9757 ty => {
9758 return Ok(vec![control_error_frame(
9759 &frame,
9760 "unsupported_control_frame",
9761 format!("unsupported module channel-0 frame {ty:?}"),
9762 )?])
9763 }
9764 };
9765
9766 let settled =
9767 self.forwarding
9768 .complete_pending_relay(connection_id, frame.header.corr, outcome);
9769 let completion = match settled {
9770 Ok(completion) => completion,
9771 Err(err) => {
9772 self.refuse_to_end_module_connection_for_a_client(
9773 connection_id,
9774 frame.header.corr,
9775 err,
9776 )?;
9777 return Ok(secondary_error.into_iter().collect());
9778 }
9779 };
9780 if let Some(target) = completion.abandoned.as_ref() {
9781 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
9782 }
9783 if !completion.settled {
9784 debug!(
9785 connection_id = connection_id.get(),
9786 corr = frame.header.corr,
9787 frame_type = ?frame.header.ty,
9788 "dropping late or unknown route.bind relay response"
9789 );
9790 }
9791 Ok(secondary_error.into_iter().collect())
9792 }
9793
9794 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
9795 debug!(connection_id = connection_id.get(), "handling GOODBYE");
9796 self.cleanup_connection_with_end_reason(
9800 connection_id,
9801 RegistrationEndReason::ExplicitGoodbye,
9802 )
9803 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
9804 Ok(Vec::new())
9805 }
9806}
9807
9808impl Default for ControlHandler {
9809 fn default() -> Self {
9810 Self::new(Arc::new(Registry::default()))
9811 }
9812}
9813
9814impl crate::supervise::SwapPromotionObserver for ControlHandler {
9815 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
9816 self.apply_registration_capabilities(registration);
9817 }
9818}
9819
9820fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
9821 CapabilityRequirementStatus {
9822 consumer: status.consumer,
9823 capability: status.capability,
9824 need: match status.need {
9825 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
9826 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
9827 },
9828 verdict: status.verdict.as_str().to_string(),
9829 episode_seq: status.episode_seq,
9830 config_satisfiable: status.config_satisfiable,
9831 runtime_available: status.runtime_available,
9832 detail: status.detail,
9833 }
9834}
9835
9836fn append_capability_problem_detail(
9837 detail: Option<String>,
9838 capability_detail: Option<String>,
9839) -> Option<String> {
9840 match (detail, capability_detail) {
9841 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
9842 (Some(detail), None) => Some(detail),
9843 (None, Some(capability_detail)) => Some(capability_detail),
9844 (None, None) => None,
9845 }
9846}
9847
9848fn subc_ops() -> Vec<String> {
9849 SUBC_CONTROL_OPS
9850 .iter()
9851 .map(|op| (*op).to_string())
9852 .collect()
9853}
9854
9855fn module_subc_ops() -> Vec<String> {
9856 SUBC_CONTROL_OPS
9857 .iter()
9858 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
9859 .map(|op| (*op).to_string())
9860 .collect()
9861}
9862
9863#[cfg(test)]
9864fn module_baseline_control_ops() -> Vec<String> {
9865 MODULE_BASELINE_CONTROL_OPS
9866 .iter()
9867 .map(|op| (*op).to_string())
9868 .collect()
9869}
9870
9871fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
9872 let mut seen = HashSet::new();
9873 let mut effective = Vec::new();
9874 for op in MODULE_BASELINE_CONTROL_OPS {
9875 if seen.insert((*op).to_string()) {
9876 effective.push((*op).to_string());
9877 }
9878 }
9879 for op in declared.unwrap_or_default() {
9880 if seen.insert(op.clone()) {
9881 effective.push(op);
9882 }
9883 }
9884 effective
9885}
9886
9887fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
9888 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
9889}
9890
9891fn target_module_id(target: &RouteTarget) -> &str {
9892 match target {
9893 RouteTarget::ToolProvider { module_id }
9894 | RouteTarget::ManagementSurface { module_id }
9895 | RouteTarget::InternalService { module_id, .. } => module_id,
9896 }
9897}
9898
9899fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
9900 roles.iter().any(|role| match (target, role) {
9901 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
9902 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
9903 (
9904 RouteTarget::InternalService { service_id, .. },
9905 ProviderRole::InternalService {
9906 service_id: provided,
9907 ..
9908 },
9909 ) => service_id == provided,
9910 _ => false,
9911 })
9912}
9913
9914fn is_routable_role(role: &ProviderRole) -> bool {
9915 matches!(
9916 role,
9917 ProviderRole::ToolProvider { .. }
9918 | ProviderRole::ManagementSurface { .. }
9919 | ProviderRole::InternalService { .. }
9920 )
9921}
9922
9923#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9924enum ControlRequestBodyError {
9925 UnknownOp,
9926 InvalidBody,
9927}
9928
9929#[derive(Debug, Deserialize)]
9930struct ControlOpProbe {
9931 op: String,
9932}
9933
9934fn module_control_response_probe(body: &[u8]) -> Result<ControlOpProbe, serde_json::Error> {
9935 serde_json::from_slice(body).or_else(|original| {
9936 #[derive(Deserialize)]
9937 struct ConsentProbe {
9938 protocol: String,
9939 }
9940 match serde_json::from_slice::<ConsentProbe>(body) {
9941 Ok(probe) if probe.protocol == subc_protocol::consent::CONSENT_V1 => {
9942 Ok(ControlOpProbe { op: probe.protocol })
9943 }
9944 _ => Err(original),
9945 }
9946 })
9947}
9948
9949const MODULE_PUSH_OPS: &[&str] = &["route.status"];
9952
9953fn is_known_module_push_op(body: &[u8]) -> bool {
9954 serde_json::from_slice::<ControlOpProbe>(body)
9955 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
9956 .unwrap_or(false)
9957}
9958
9959fn is_known_module_request_op(body: &[u8]) -> bool {
9960 serde_json::from_slice::<ControlOpProbe>(body)
9961 .map(|probe| is_module_to_subc_op(&probe.op))
9962 .unwrap_or(false)
9963}
9964
9965fn is_module_to_subc_op(op: &str) -> bool {
9966 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
9967}
9968
9969fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
9970 debug!(
9971 op = %op,
9972 connection_id = connection_id.get(),
9973 corr,
9974 "control dispatch"
9975 );
9976}
9977
9978fn log_slow_control_dispatch(
9979 dispatch_started_at: Option<StdInstant>,
9980 op: &'static str,
9981 connection_id: ConnectionId,
9982 corr: u64,
9983 route_open_timings: Option<&RouteOpenTimings>,
9984) {
9985 let Some(dispatch_started_at) = dispatch_started_at else {
9986 return;
9987 };
9988 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
9990 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
9991 if let Some(timings) = route_open_timings {
9992 warn!(
9993 op = %op,
9994 connection_id = connection_id.get(),
9995 corr,
9996 elapsed_ms = elapsed.as_millis() as u64,
9997 module_id = ?timings.module_id,
9998 principal = %timings.principal,
9999 admission_ms = timings.admission.as_millis() as u64,
10000 project_root_ms = timings.project_root.as_millis() as u64,
10001 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
10002 bind_send_ms = timings.bind_send.as_millis() as u64,
10003 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
10004 "slow control dispatch"
10005 );
10006 return;
10007 }
10008 warn!(
10009 op = %op,
10010 connection_id = connection_id.get(),
10011 corr,
10012 elapsed_ms = elapsed.as_millis() as u64,
10013 "slow control dispatch"
10014 );
10015 }
10016}
10017
10018#[cfg(test)]
10020type TestCallerLookup =
10021 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
10022
10023fn privileged_operation(request: &impl PrivilegedRequest) -> Option<PrivilegedOperation<'_>> {
10025 request.privileged_operation()
10026}
10027
10028trait PrivilegedRequest {
10029 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>>;
10030}
10031
10032impl PrivilegedRequest for ClientControlRequest {
10033 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>> {
10034 let request = self;
10035 match request {
10036 ClientControlRequest::SupervisorRestart { module_id, .. } => {
10037 Some(PrivilegedOperation::Restart { module_id })
10038 }
10039 ClientControlRequest::SupervisorSwap { module_id, .. } => {
10040 Some(PrivilegedOperation::Swap { module_id })
10041 }
10042 ClientControlRequest::SupervisorReload { module_id } => {
10043 Some(PrivilegedOperation::Reload { module_id })
10044 }
10045 ClientControlRequest::SupervisorRescan { preview } => {
10046 if *preview {
10047 None
10048 } else {
10049 Some(PrivilegedOperation::Rescan)
10050 }
10051 }
10052 ClientControlRequest::SupervisorRescanChecked {
10057 preview,
10058 candidate,
10059 expect,
10060 } => (!*preview && candidate.is_none() && expect.is_some())
10061 .then_some(PrivilegedOperation::Rescan),
10062 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
10063 Some(PrivilegedOperation::SetEnabled {
10064 module_id,
10065 enabled: *enabled,
10066 })
10067 }
10068 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
10069 Some(PrivilegedOperation::ReleaseReserved { module_id })
10070 }
10071 ClientControlRequest::ServerDescribe {}
10072 | ClientControlRequest::DaemonShutdownIntent { .. }
10075 | ClientControlRequest::CatalogList { .. }
10076 | ClientControlRequest::RouteOpen { .. }
10077 | ClientControlRequest::RoutePoll { .. }
10078 | ClientControlRequest::SupervisorList {}
10079 | ClientControlRequest::SupervisorHealth {}
10080 | ClientControlRequest::SupervisorHealthProbe { .. }
10081 | ClientControlRequest::SupervisorRoutes { .. }
10082 | ClientControlRequest::SupervisorProvenance { .. }
10083 | ClientControlRequest::SupervisorStderrTail { .. }
10084 | ClientControlRequest::SupervisorTerminals { .. }
10085 | ClientControlRequest::SupervisorSpawnSnapshot {}
10086 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
10087 }
10088 }
10089}
10090
10091fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
10092 match request {
10093 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
10094 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
10095 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
10096 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
10097 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
10098 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
10099 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
10100 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
10101 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
10102 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
10103 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
10104 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
10105 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
10106 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
10107 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
10108 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
10109 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
10110 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
10111 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
10112 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
10113 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
10114 }
10115}
10116
10117struct RouteOpenCaller {
10125 principal: String,
10129 scope: String,
10132 harness: String,
10135}
10136
10137impl RouteOpenCaller {
10138 fn new(
10139 principal: Option<&Principal>,
10140 scope: Option<&ScopeSelector>,
10141 identity: &BindIdentity,
10142 ) -> Self {
10143 Self {
10144 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
10145 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
10146 harness: bounded_log_text(&identity.harness),
10147 }
10148 }
10149
10150 fn unknown() -> Self {
10153 Self {
10154 principal: "unknown".to_string(),
10155 scope: "unknown".to_string(),
10156 harness: "unknown".to_string(),
10157 }
10158 }
10159}
10160
10161const MAX_LOGGED_TEXT_CHARS: usize = 64;
10164
10165fn bounded_log_text(value: &str) -> String {
10169 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
10170 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
10171 bounded.push('…');
10172 }
10173 format!("{bounded:?}")
10174}
10175
10176fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
10182 use subc_protocol::scope::ScopeKind;
10183 let kind = match stamp.kind {
10184 ScopeKind::Head => "head",
10185 ScopeKind::Worker => "worker",
10186 ScopeKind::Ephemeral => "ephemeral",
10187 ScopeKind::Federated => "federated",
10188 _ => "unknown",
10189 };
10190 format!(
10191 "{kind}/{}/{}@{}",
10192 principal_label(&stamp.owner),
10193 bounded_log_text(&stamp.scope_ref),
10194 stamp.scope_epoch
10195 )
10196}
10197
10198fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
10205 let epoch = selector
10206 .scope_epoch
10207 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
10208 format!(
10209 "requested/{}/{}@{epoch}",
10210 bounded_log_text(&principal_label(&selector.owner)),
10211 bounded_log_text(&selector.scope_ref),
10212 )
10213}
10214
10215fn principal_label(principal: &Principal) -> String {
10216 match principal {
10217 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
10218 Principal::Direct => "direct".to_string(),
10219 other => format!("{other:?}"),
10220 }
10221}
10222
10223fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
10224 match request {
10225 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
10226 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
10227 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
10228 }
10229 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
10230 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
10231 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
10232 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
10233 ModuleControlRequestFromModule::RouteAuthorize { .. } => {
10234 subc_protocol::scope::ROUTE_AUTHORIZE_OP
10235 }
10236 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
10237 _ => "unsupported",
10238 }
10239}
10240
10241fn parse_client_control_request(
10242 body: &[u8],
10243) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
10244 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
10245 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10246 Ok(probe)
10247 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
10248 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
10249 {
10250 ControlRequestBodyError::InvalidBody
10251 }
10252 Ok(_) => ControlRequestBodyError::UnknownOp,
10253 Err(_) => ControlRequestBodyError::InvalidBody,
10254 };
10255 (err, classification)
10256 })
10257}
10258
10259fn parse_module_control_request_from_module(
10260 body: &[u8],
10261) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
10262 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
10263 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10264 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
10265 Ok(_) => ControlRequestBodyError::UnknownOp,
10266 Err(_) => ControlRequestBodyError::InvalidBody,
10267 };
10268 (err, classification)
10269 })
10270}
10271
10272#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
10273enum ProviderRoleKind {
10274 ToolProvider,
10275 PipelineStage,
10276 ManagementSurface,
10277 InternalService,
10278}
10279
10280fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
10281 match role {
10282 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
10283 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
10284 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
10285 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
10286 }
10287}
10288
10289fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
10290 roles.iter().map(provider_role_kind).collect()
10291}
10292
10293const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
10296
10297#[derive(Debug, Default, PartialEq, Eq)]
10299struct ScopeOutcomeCounts {
10300 created: usize,
10301 replaced: usize,
10302 updated: usize,
10303 unchanged: usize,
10304 refused: usize,
10305}
10306
10307impl ScopeOutcomeCounts {
10308 fn of(results: &[ScopeRecordResult]) -> Self {
10309 let mut counts = Self::default();
10310 for result in results {
10311 let slot = match result.outcome {
10312 ScopeRecordOutcome::Created => &mut counts.created,
10313 ScopeRecordOutcome::Replaced => &mut counts.replaced,
10314 ScopeRecordOutcome::Updated => &mut counts.updated,
10315 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
10316 ScopeRecordOutcome::Refused => &mut counts.refused,
10317 };
10318 *slot += 1;
10319 }
10320 counts
10321 }
10322}
10323
10324#[cfg(test)]
10325mod scope_outcome_count_tests {
10326 use super::*;
10327
10328 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
10329 ScopeRecordResult {
10330 scope_ref: "r".to_string(),
10331 scope_epoch: 1,
10332 outcome,
10333 code: None,
10334 message: None,
10335 version: None,
10336 parent_state: None,
10337 }
10338 }
10339
10340 #[test]
10343 fn every_outcome_is_counted_in_its_own_field() {
10344 let results = [
10345 result(ScopeRecordOutcome::Created),
10346 result(ScopeRecordOutcome::Created),
10347 result(ScopeRecordOutcome::Replaced),
10348 result(ScopeRecordOutcome::Updated),
10349 result(ScopeRecordOutcome::Unchanged),
10350 result(ScopeRecordOutcome::Refused),
10351 result(ScopeRecordOutcome::Refused),
10352 result(ScopeRecordOutcome::Refused),
10353 ];
10354 assert_eq!(
10355 ScopeOutcomeCounts::of(&results),
10356 ScopeOutcomeCounts {
10357 created: 2,
10358 replaced: 1,
10359 updated: 1,
10360 unchanged: 1,
10361 refused: 3,
10362 }
10363 );
10364 }
10365}
10366
10367fn capability_census_trigger(
10371 old: Option<&CapabilityDeclarations>,
10372 new: Option<&CapabilityDeclarations>,
10373) -> bool {
10374 let old_provides = old
10375 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
10376 .unwrap_or_default();
10377 let old_denies = old
10378 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
10379 .unwrap_or_default();
10380 let new = new.cloned().unwrap_or(CapabilityDeclarations {
10381 provides: Vec::new(),
10382 requires: Vec::new(),
10383 must_never_reach: Vec::new(),
10384 });
10385
10386 new.provides
10387 .iter()
10388 .any(|capability| !old_provides.contains(capability))
10389 || new
10390 .must_never_reach
10391 .iter()
10392 .any(|capability| !old_denies.contains(capability))
10393}
10394
10395fn catalog_update_frozen_field_message(
10396 registered: &ModuleManifest,
10397 provides: &[ProviderRole],
10398) -> Option<String> {
10399 let old_has_provides = !registered.provides.is_empty();
10400 let new_has_provides = !provides.is_empty();
10401 if old_has_provides != new_has_provides {
10402 return Some(format!(
10403 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
10404 registered.module_id
10405 ));
10406 }
10407
10408 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
10409 return Some(format!(
10410 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
10411 registered.module_id
10412 ));
10413 }
10414
10415 let registered_concurrency = manifest_concurrency(registered);
10416 let mut candidate = registered.clone();
10417 candidate.provides = provides.to_vec();
10418 let candidate_concurrency = manifest_concurrency(&candidate);
10419 if candidate_concurrency != registered_concurrency {
10420 return Some(format!(
10421 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
10422 registered.module_id, registered_concurrency, candidate_concurrency
10423 ));
10424 }
10425
10426 None
10429}
10430
10431fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
10432 manifest.provides.iter().any(is_routable_role)
10433}
10434
10435fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
10441 manifest
10442 .provides
10443 .iter()
10444 .find_map(|provider| match provider {
10445 ProviderRole::ToolProvider { concurrency, .. }
10446 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
10447 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
10448 })
10449 .unwrap_or(Concurrency::ModuleManaged)
10450}
10451
10452fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
10459 let has_management_surface = manifest
10460 .provides
10461 .iter()
10462 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
10463 if !has_management_surface {
10464 return false;
10465 }
10466 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
10467 return false;
10468 };
10469 let Some(provides) = raw
10470 .get("manifest")
10471 .and_then(|manifest| manifest.get("provides"))
10472 .and_then(serde_json::Value::as_array)
10473 else {
10474 return false;
10475 };
10476 provides.iter().any(|role| {
10481 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
10482 && role.get("concurrency").is_none()
10483 })
10484}
10485
10486fn negotiate_version(peer_version: u8) -> Result<u8, String> {
10487 if peer_version != PROTOCOL_VERSION {
10488 return Err(format!(
10489 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
10490 ));
10491 }
10492 Ok(PROTOCOL_VERSION)
10493}
10494
10495fn pong(frame: &Frame) -> Result<Frame, RouterError> {
10496 Frame::build_with_version(
10497 response_version(frame),
10498 FrameType::Pong,
10499 frame.header.flags,
10500 0,
10501 0,
10502 frame.header.corr,
10503 Vec::new(),
10504 )
10505 .map_err(RouterError::FrameBuild)
10506}
10507
10508pub(crate) fn control_error_frame(
10509 frame: &Frame,
10510 code: &'static str,
10511 message: impl Into<String>,
10512) -> Result<Frame, RouterError> {
10513 control_error_body_frame(
10514 frame,
10515 ErrorBody {
10516 code: code.to_string(),
10517 message: message.into(),
10518 detail: None,
10519 },
10520 )
10521}
10522
10523pub(crate) fn control_error_body_frame(
10531 frame: &Frame,
10532 error: ErrorBody,
10533) -> Result<Frame, RouterError> {
10534 control_error_frame_with_flags(frame, error, daemon_error_flags())
10535}
10536
10537fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
10543 control_error_frame_with_flags(frame, error, control_flags())
10544}
10545
10546fn control_error_frame_with_flags(
10547 frame: &Frame,
10548 error: ErrorBody,
10549 flags: Flags,
10550) -> Result<Frame, RouterError> {
10551 let body = serde_json::to_vec(&error).map_err(|err| {
10552 RouterError::backend(
10553 0,
10554 frame.header.corr,
10555 format!("failed to encode control ERROR: {err}"),
10556 )
10557 })?;
10558
10559 Frame::build_with_version(
10560 response_version(frame),
10561 FrameType::Error,
10562 flags,
10563 0,
10564 0,
10565 frame.header.corr,
10566 body,
10567 )
10568 .map_err(RouterError::FrameBuild)
10569}
10570
10571pub(crate) fn control_response_body_frame<T: Serialize>(
10572 frame: &Frame,
10573 reply: &T,
10574 label: &'static str,
10575) -> Result<Frame, RouterError> {
10576 let body = serde_json::to_vec(reply).map_err(|err| {
10577 RouterError::backend(
10578 0,
10579 frame.header.corr,
10580 format!("failed to encode {label}: {err}"),
10581 )
10582 })?;
10583
10584 Frame::build_with_version(
10585 response_version(frame),
10586 FrameType::Response,
10587 control_flags(),
10588 0,
10589 0,
10590 frame.header.corr,
10591 body,
10592 )
10593 .map_err(RouterError::FrameBuild)
10594}
10595
10596struct HelloAuthority {
10629 record: Arc<crate::spawn_authority::SpawnAuthorityRecord>,
10630 requested: ProviderDeclarations,
10631 froze: std::sync::atomic::AtomicBool,
10634}
10635
10636fn forwarding_error_code(err: &ForwardingError) -> &'static str {
10637 match err {
10638 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
10639 ForwardingError::NoModuleConnection => "target_unavailable",
10640 ForwardingError::ModuleReloading { .. } => "module_reloading",
10641 ForwardingError::ClientRouteChannelExhausted { .. }
10642 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
10643 ForwardingError::StaleModuleEndpoint
10644 | ForwardingError::UnknownReservation { .. }
10645 | ForwardingError::ConnectionClosing { .. }
10646 | ForwardingError::ClientEgressClosed { .. }
10647 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
10648 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
10651 ForwardingError::RelayCorrelationExhausted
10652 | ForwardingError::RouteOpenBuild(_)
10653 | ForwardingError::Poisoned => "forwarding_error",
10654 }
10655}
10656
10657fn response_version(frame: &Frame) -> u8 {
10658 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
10659 frame.header.ver
10660 } else {
10661 PROTOCOL_VERSION
10662 }
10663}
10664
10665pub(crate) fn control_flags() -> Flags {
10666 Flags::new(false, Priority::Passive, false)
10667}
10668
10669fn send_goodbye_target_best_effort(
10674 counters: &DaemonCounters,
10675 target: &GoodbyeTarget,
10676 context: &'static str,
10677) {
10678 let Ok(frame) = Frame::build_with_version(
10679 target.negotiated_ver,
10680 FrameType::Goodbye,
10681 control_flags(),
10682 target.channel,
10683 target.epoch,
10684 0,
10685 Vec::new(),
10686 ) else {
10687 return;
10688 };
10689 crate::forwarding::send_module_route_goodbye(
10690 counters,
10691 &target.sink,
10692 frame,
10693 target.module_id.as_deref(),
10694 context,
10695 );
10696}
10697
10698fn route_close_reason_for_opener(
10699 reason: RouteCloseReason,
10700 accepts_close_reasons: Option<&[String]>,
10701) -> RouteCloseReason {
10702 if reason == RouteCloseReason::AuthorityChanged
10703 && !accepts_close_reasons
10704 .is_some_and(|reasons| reasons.iter().any(|reason| reason == "authority_changed"))
10705 {
10706 RouteCloseReason::Reload
10709 } else {
10710 reason
10711 }
10712}
10713
10714pub(crate) fn send_route_control_pushes(
10715 forwarding: &ForwardingTable,
10716 routes: Vec<EndpointRoute>,
10717 push: ClientControlPush,
10718) {
10719 let reason = match &push {
10720 ClientControlPush::RouteClosing { reason, .. }
10721 | ClientControlPush::RouteClosed { reason, .. } => *reason,
10722 };
10723 let mut targets: Vec<(GoodbyeTarget, RouteCloseReason, Vec<u16>)> = Vec::new();
10724 for route in routes {
10725 let recipient_reason =
10726 route_close_reason_for_opener(reason, route.accepts_close_reasons.as_deref());
10727 let target = route.goodbye_target;
10728 if let Some((existing, _, channels)) = targets.iter_mut().find(|(existing, reason, _)| {
10729 existing.connection_id == target.connection_id && *reason == recipient_reason
10730 }) {
10731 debug_assert_eq!(
10732 existing.negotiated_ver, target.negotiated_ver,
10733 "one connection cannot negotiate multiple frame versions"
10734 );
10735 if !channels.contains(&target.channel) {
10736 channels.push(target.channel);
10737 }
10738 continue;
10739 }
10740 let channel = target.channel;
10741 targets.push((target, recipient_reason, vec![channel]));
10742 }
10743 for (target, recipient_reason, mut channels) in targets {
10744 channels.sort_unstable();
10745 let mut push = push.clone();
10746 match &mut push {
10747 ClientControlPush::RouteClosing {
10748 channels: covered,
10749 reason,
10750 ..
10751 }
10752 | ClientControlPush::RouteClosed {
10753 channels: covered,
10754 reason,
10755 ..
10756 } => {
10757 *covered = channels;
10758 *reason = recipient_reason;
10759 }
10760 }
10761 let body = match serde_json::to_vec(&push) {
10762 Ok(body) => body,
10763 Err(err) => {
10764 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
10765 continue;
10766 }
10767 };
10768 let frame = match Frame::build_with_version(
10769 target.negotiated_ver,
10770 FrameType::Push,
10771 control_flags(),
10772 0,
10773 0,
10774 0,
10775 body.clone(),
10776 ) {
10777 Ok(frame) => frame,
10778 Err(err) => {
10779 warn!(
10780 route_channel = target.channel,
10781 error = %err,
10782 "failed to build route lifecycle control PUSH frame"
10783 );
10784 continue;
10785 }
10786 };
10787 if let Err(err) = target.sink.try_send(frame) {
10788 if target.close_on_delivery_failure() {
10789 warn!(
10790 target_connection_id = target.connection_id.get(),
10791 route_channel = target.channel,
10792 error = %err,
10793 "route lifecycle control PUSH was not delivered to client; closing target connection"
10794 );
10795 let _ = forwarding.escalate_client_delivery_failure(
10796 target.connection_id,
10797 target.channel,
10798 target.epoch,
10799 CloseReason::new(
10800 "route_lifecycle_push_delivery_failed",
10801 format!(
10802 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
10803 target.channel
10804 ),
10805 ),
10806 crate::forwarding::UndeliveredFrame {
10807 module_id: target.module_id.as_deref(),
10808 sink: &target.sink,
10809 },
10810 );
10811 }
10812 }
10813 }
10814}
10815
10816#[cfg(test)]
10817mod tests {
10818 include!("control/agent_commands_tests.rs");
10819 include!("control/resource_revision_tests.rs");
10820 include!("control/route_open_recovery_tests.rs");
10821 #[tokio::test]
10822 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
10823 let registry = Arc::new(Registry::default());
10824 let handler = ControlHandler::new(registry);
10825 assert_eq!(
10826 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10827 None
10828 );
10829 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
10830 hello_via_sink(
10831 &handler,
10832 &ctx,
10833 &mut rx,
10834 hello_frame("daemon", PROTOCOL_VERSION, 1),
10835 )
10836 .await;
10837 assert_eq!(
10838 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10839 Some("daemon".into())
10840 );
10841 assert_eq!(
10842 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
10843 None
10844 );
10845 assert_eq!(
10846 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
10847 None,
10848 "a registered module is not necessarily supervised"
10849 );
10850 assert_eq!(
10851 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
10852 None,
10853 "aggregate provenance has no single target"
10854 );
10855 assert_eq!(
10856 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
10857 None
10858 );
10859 }
10860
10861 mod peer_manifest {
10862 include!("peer_manifest_tests.rs");
10863 }
10864 mod catalog_declarations {
10865 include!("control/catalog_tests.rs");
10866 }
10867 mod activity_read {
10868 include!("activity_read_tests.rs");
10869 }
10870
10871 mod lifecycle_activity {
10872 include!("control_lifecycle_tests.rs");
10873 }
10874 #[cfg(unix)]
10875 #[tokio::test]
10876 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
10877 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
10878 let path = dir.join("subc.jsonc");
10879 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
10880 "program":"/bin/sleep","args":["60"],"protocol":"none",
10881 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
10882 }}}).to_string()).unwrap();
10883 let mut configured = crate::daemon_config::load(&path)
10884 .unwrap()
10885 .unwrap()
10886 .modules
10887 .pop()
10888 .unwrap();
10889 let registry = std::sync::Arc::new(crate::Registry::default());
10890 let handle = crate::SupervisorHandle::new();
10891 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
10892 .with_handle(handle.clone());
10893 let module = supervisor
10894 .supervise_configured_with_health(
10895 configured.module_spec(),
10896 true,
10897 configured.health.clone(),
10898 None,
10899 configured.restart,
10900 )
10901 .unwrap();
10902 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
10903 let before = module.status().unwrap().pid;
10904 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
10905 configured.health.cadence = std::time::Duration::from_secs(3600);
10906 let health_only = handler
10907 .reconcile_supervised_modules(
10908 &supervisor,
10909 vec![configured.clone()],
10910 false,
10911 Principal::Direct,
10912 )
10913 .await
10914 .unwrap();
10915 assert!(
10916 health_only.0.changed_pending_reload.is_empty(),
10917 "health policy is already applied live"
10918 );
10919 assert_eq!(module.status().unwrap().pid, before);
10920 assert_eq!(
10921 module.configuration().unwrap().1.http,
10922 configured.health.http
10923 );
10924 configured.args = vec!["61".into()];
10925 let launch = handler
10926 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
10927 .await
10928 .unwrap();
10929 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
10930 assert_eq!(
10931 module.status().unwrap().pid,
10932 before,
10933 "a launch edit is stored until reload"
10934 );
10935 module.drain().await.unwrap();
10936 }
10937 use cortexkit_test_support::ScratchDir;
10938 use std::{
10939 collections::BTreeMap,
10940 fmt,
10941 path::PathBuf,
10942 sync::{Arc, Mutex},
10943 time::Duration,
10944 };
10945
10946 use serde_json::{json, Value};
10947 use subc_protocol::{
10948 manifest::{
10949 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
10950 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
10951 },
10952 session::HealthStatus,
10953 FrameType,
10954 };
10955
10956 use super::*;
10957 use crate::{
10958 forwarding::{DataRoute, DataRouteState},
10959 registry::ChannelState,
10960 router::FrameSink,
10961 stderr_tail::DEFAULT_MAX_LINE_BYTES,
10962 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
10963 RouteCtx, Router,
10964 };
10965 use tokio::{
10966 sync::mpsc,
10967 time::{sleep, Instant},
10968 };
10969 use tracing::{
10970 field::{Field, Visit},
10971 Event, Subscriber,
10972 };
10973 use tracing_subscriber::{layer::Context, prelude::*, Layer};
10974
10975 fn fake_aft_stub_path() -> PathBuf {
10994 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
10995 assert!(
10996 path.exists(),
10997 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
10998 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
10999 path.display()
11000 );
11001 path
11002 }
11003
11004 fn client_retries(code: &str) -> bool {
11009 subc_protocol::error_codes::is_retryable_route_open(code)
11010 }
11011
11012 #[test]
11023 fn retryability_of_forwarding_codes_matches_the_failure() {
11024 let transient = [
11027 ForwardingError::NoModuleConnection,
11028 ForwardingError::ModuleReloading {
11029 module_id: "m".into(),
11030 },
11031 ForwardingError::StaleModuleEndpoint,
11032 ForwardingError::UnknownReservation {
11033 client_channel: 1,
11034 module_channel: 1,
11035 },
11036 ForwardingError::ConnectionClosing {
11037 connection_id: ConnectionId::new(1),
11038 },
11039 ForwardingError::ClientEgressClosed {
11040 connection_id: ConnectionId::new(1),
11041 },
11042 ForwardingError::ModuleEgressUnavailable {
11043 connection_id: ConnectionId::new(1),
11044 },
11045 ];
11046 for err in transient {
11047 let code = forwarding_error_code(&err);
11048 assert!(
11049 client_retries(code),
11050 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
11051 );
11052 }
11053
11054 let permanent = [
11059 ForwardingError::ConnectionRoleConflict {
11060 connection_id: ConnectionId::new(1),
11061 },
11062 ForwardingError::ClientRouteChannelExhausted {
11063 connection_id: ConnectionId::new(1),
11064 },
11065 ForwardingError::ModuleRouteChannelExhausted {
11066 endpoint: ModuleEndpointId {
11067 connection_id: ConnectionId::new(1),
11068 generation: 1,
11069 },
11070 },
11071 ForwardingError::RelayCorrelationExhausted,
11072 ForwardingError::RouteOpenBuild("x".into()),
11073 ForwardingError::Poisoned,
11074 ];
11075 for err in permanent {
11076 let code = forwarding_error_code(&err);
11077 assert!(
11078 !client_retries(code),
11079 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
11080 );
11081 }
11082 }
11083
11084 #[tokio::test]
11112 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
11113 let handler = ControlHandler::default();
11114 let frame =
11115 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
11116
11117 let stamped = handler
11121 .route_open_principal(ConnectionId::new(1), &frame, None)
11122 .unwrap()
11123 .unwrap();
11124 assert_eq!(
11125 stamped,
11126 Principal::Direct,
11127 "a caller that proved nothing must not be stamped as a supervised module"
11128 );
11129
11130 let forged = handler
11135 .route_open_principal(
11136 ConnectionId::new(1),
11137 &frame,
11138 Some(ConsumerIdentity {
11139 module_id: "aft".to_string(),
11140 launch_nonce: "not-a-real-nonce".to_string(),
11141 }),
11142 )
11143 .unwrap();
11144 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
11145 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
11146 }
11147
11148 #[test]
11165 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
11166 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
11167 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
11168 let ClientControlRequest::RouteOpen {
11169 consumer_identity, ..
11170 } = parsed
11171 else {
11172 panic!("route.open body must parse as RouteOpen");
11173 };
11174 assert_eq!(
11175 consumer_identity,
11176 Some(ConsumerIdentity {
11177 module_id: "aft".to_string(),
11178 launch_nonce: "n".to_string(),
11179 }),
11180 "the wire field name must reach the value route_open_principal reads"
11181 );
11182 }
11183
11184 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
11185 ModuleManifest::builder(module_id, "0.1.0")
11186 .protocol_ver(protocol_ver)
11187 .provides(vec![ProviderRole::ToolProvider {
11188 tools: vec![Tool {
11189 name: "read".to_string(),
11190 description: None,
11191 execution_mode: ExecutionMode::Pure,
11192 schema: json!({"type": "object"}),
11193 }],
11194 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
11195 concurrency: Concurrency::ModuleManaged,
11196 emits_push: true,
11197 sub_supervises: true,
11198 }])
11199 .build()
11200 }
11201
11202 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
11203 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
11204 }
11205
11206 fn hello_frame_with_control_ops(
11207 module_id: &str,
11208 protocol_ver: u8,
11209 corr: u64,
11210 control_ops: Option<Vec<String>>,
11211 ) -> Frame {
11212 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
11213 }
11214
11215 fn hello_frame_with_nonce(
11216 module_id: &str,
11217 protocol_ver: u8,
11218 corr: u64,
11219 launch_nonce: Option<&str>,
11220 ) -> Frame {
11221 hello_frame_full(
11222 module_id,
11223 protocol_ver,
11224 corr,
11225 None,
11226 launch_nonce.map(ToOwned::to_owned),
11227 )
11228 }
11229
11230 fn hello_frame_full(
11231 module_id: &str,
11232 protocol_ver: u8,
11233 corr: u64,
11234 control_ops: Option<Vec<String>>,
11235 launch_nonce: Option<String>,
11236 ) -> Frame {
11237 let body = serde_json::to_vec(&ModuleHelloBody {
11238 manifest: manifest(module_id, protocol_ver),
11239 protocol_ver,
11240 control_ops,
11241 launch_nonce,
11242 })
11243 .unwrap();
11244 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11245 }
11246
11247 fn hello_frame_with_rich_ui(
11248 module_id: &str,
11249 protocol_ver: u8,
11250 corr: u64,
11251 launch_nonce: Option<&str>,
11252 rich_ui: Value,
11253 ) -> Frame {
11254 let mut body = serde_json::to_value(ModuleHelloBody {
11255 manifest: manifest(module_id, protocol_ver),
11256 protocol_ver,
11257 control_ops: None,
11258 launch_nonce: launch_nonce.map(str::to_string),
11259 })
11260 .unwrap();
11261 body["manifest"]["rich_ui"] = rich_ui;
11262 Frame::build(
11263 FrameType::Hello,
11264 control_flags(),
11265 0,
11266 0,
11267 corr,
11268 serde_json::to_vec(&body).unwrap(),
11269 )
11270 .unwrap()
11271 }
11272
11273 fn non_routable_hello_frame_with_control_ops(
11274 module_id: &str,
11275 corr: u64,
11276 control_ops: Option<Vec<String>>,
11277 ) -> Frame {
11278 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11279 manifest.provides.clear();
11280 let body = serde_json::to_vec(&ModuleHelloBody {
11281 manifest,
11282 protocol_ver: PROTOCOL_VERSION,
11283 control_ops,
11284 launch_nonce: None,
11285 })
11286 .unwrap();
11287 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11288 }
11289
11290 fn capability_grammar_hello_frame(
11291 capabilities: Value,
11292 runtime_computed: Option<Value>,
11293 corr: u64,
11294 ) -> Frame {
11295 let mut body = serde_json::to_value(ModuleHelloBody {
11296 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
11297 protocol_ver: PROTOCOL_VERSION,
11298 control_ops: None,
11299 launch_nonce: None,
11300 })
11301 .expect("HELLO body serializes");
11302 body["manifest"]["capabilities"] = capabilities;
11303 if let Some(runtime_computed) = runtime_computed {
11304 body["runtime_computed"] = runtime_computed;
11305 }
11306 Frame::build(
11307 FrameType::Hello,
11308 control_flags(),
11309 0,
11310 0,
11311 corr,
11312 serde_json::to_vec(&body).expect("HELLO body reserializes"),
11313 )
11314 .expect("HELLO frame builds")
11315 }
11316
11317 fn channel_request(channel: u16, corr: u64) -> Frame {
11318 Frame::build(
11319 FrameType::Request,
11320 Flags::new(true, Priority::Interactive, false),
11321 channel,
11322 0,
11323 corr,
11324 b"opaque".to_vec(),
11325 )
11326 .unwrap()
11327 }
11328
11329 fn route_ctx(
11330 connection_id: ConnectionId,
11331 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11332 let (tx, rx) = mpsc::channel(8);
11333 (
11334 RouteCtx {
11335 connection_id,
11336 egress: FrameSink::new(tx),
11337 },
11338 rx,
11339 )
11340 }
11341
11342 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
11343 serde_json::from_slice(&frame.body).unwrap()
11344 }
11345
11346 async fn hello_via_sink(
11351 handler: &ControlHandler,
11352 ctx: &RouteCtx,
11353 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11354 hello: Frame,
11355 ) -> Frame {
11356 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
11357 assert!(
11358 replies.is_empty(),
11359 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
11360 );
11361 let ack = rx
11362 .try_recv()
11363 .expect("HELLO_ACK is queued on the module sink")
11364 .frame;
11365 assert_eq!(ack.header.ty, FrameType::HelloAck);
11366 ack
11367 }
11368
11369 fn parse_error(frame: &Frame) -> Value {
11370 serde_json::from_slice(&frame.body).unwrap()
11371 }
11372
11373 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
11374 serde_json::from_slice(&frame.body).unwrap()
11375 }
11376
11377 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
11378 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
11379 route_channel,
11380 route_epoch: 0,
11381 kind,
11382 })
11383 .unwrap();
11384 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11385 }
11386
11387 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
11388 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
11389 module_id: module_id.to_string(),
11390 })
11391 .unwrap();
11392 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11393 }
11394
11395 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
11396 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
11397 }
11398
11399 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
11401 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
11402 let mut frame = route_open_frame(corr, module_id, root);
11403 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
11404 body["identity"]["project_root"] = serde_json::json!(path.as_path());
11405 frame.body = serde_json::to_vec(&body).unwrap();
11406 frame
11407 }
11408
11409 fn route_open_frame_with_consumer_capabilities(
11410 corr: u64,
11411 module_id: &str,
11412 _project_root: ScratchDir,
11413 consumer_capabilities: Option<Vec<String>>,
11414 ) -> Frame {
11415 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11416 busy_refusals: None,
11417 target: RouteTarget::ToolProvider {
11418 module_id: module_id.to_string(),
11419 },
11420 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
11421 consumer_identity: None,
11422 consumer_capabilities,
11423 role_versions: None,
11424 admission_facts: None,
11425 scope: None,
11426 })
11427 .unwrap();
11428 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11429 }
11430
11431 fn route_open_frame_with_role_versions(
11432 corr: u64,
11433 module_id: &str,
11434 _project_root: ScratchDir,
11435 role_versions: Option<BTreeMap<String, String>>,
11436 ) -> Frame {
11437 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11438 busy_refusals: None,
11439 target: RouteTarget::ToolProvider {
11440 module_id: module_id.to_string(),
11441 },
11442 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11443 consumer_identity: None,
11444 consumer_capabilities: None,
11445 role_versions,
11446 admission_facts: None,
11447 scope: None,
11448 })
11449 .unwrap();
11450 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11451 }
11452
11453 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
11454 entries
11455 .iter()
11456 .map(|(role, version)| (role.to_string(), version.to_string()))
11457 .collect()
11458 }
11459
11460 fn route_open_frame_with_admission_facts(
11461 corr: u64,
11462 module_id: &str,
11463 _project_root: ScratchDir,
11464 consumer_identity: Option<subc_control::ConsumerIdentity>,
11465 facts: Option<Value>,
11466 ) -> Frame {
11467 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11468 busy_refusals: None,
11469 target: RouteTarget::ToolProvider {
11470 module_id: module_id.to_string(),
11471 },
11472 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11473 consumer_identity,
11474 consumer_capabilities: None,
11475 role_versions: None,
11476 admission_facts: facts,
11477 scope: None,
11478 })
11479 .unwrap();
11480 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11481 }
11482
11483 #[derive(Clone, Default)]
11484 struct EventCapture {
11485 events: Arc<Mutex<Vec<CapturedEvent>>>,
11486 }
11487
11488 #[derive(Clone, Debug)]
11489 struct CapturedEvent {
11490 target: String,
11491 level: tracing::Level,
11492 fields: BTreeMap<String, String>,
11493 }
11494
11495 impl EventCapture {
11496 fn events(&self) -> Vec<CapturedEvent> {
11497 self.events.lock().unwrap().clone()
11498 }
11499 }
11500
11501 impl<S> Layer<S> for EventCapture
11502 where
11503 S: Subscriber,
11504 {
11505 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
11506 let mut visitor = EventFieldVisitor::default();
11507 event.record(&mut visitor);
11508 self.events.lock().unwrap().push(CapturedEvent {
11509 target: event.metadata().target().to_string(),
11510 level: *event.metadata().level(),
11511 fields: visitor.fields,
11512 });
11513 }
11514 }
11515
11516 #[derive(Default)]
11517 struct EventFieldVisitor {
11518 fields: BTreeMap<String, String>,
11519 }
11520
11521 impl Visit for EventFieldVisitor {
11522 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
11523 self.fields
11524 .insert(field.name().to_string(), format!("{value:?}"));
11525 }
11526 }
11527
11528 fn health_response(corr: u64, status: HealthStatus) -> Frame {
11529 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
11530 status,
11531 detail: Some("warming".to_string()),
11532 metrics: Some(json!({"queue_depth": 3})),
11533 })
11534 .unwrap();
11535 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11536 }
11537
11538 fn route_bind_ack(corr: u64) -> Frame {
11539 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
11540 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11541 }
11542
11543 fn unique_project_root(label: &str) -> ScratchDir {
11544 ScratchDir::new(label)
11545 }
11546
11547 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
11548 match parse_route_poll(frame) {
11549 ClientControlResponse::RoutePoll {
11550 status: None,
11551 live: Some(live),
11552 ..
11553 } => assert_eq!(live, expected_live),
11554 other => panic!("unexpected route.poll response: {other:?}"),
11555 }
11556 }
11557
11558 fn bind_liveness_route(
11559 registry: &Registry,
11560 forwarding: &ForwardingTable,
11561 module_id: &str,
11562 ) -> (RouteCtx, u16, u32) {
11563 let module_connection = ConnectionId::new(101);
11564 let client_connection = ConnectionId::new(202);
11565 let registration = registry
11566 .register_with_control_ops(
11567 manifest(module_id, PROTOCOL_VERSION),
11568 PROTOCOL_VERSION,
11569 module_connection,
11570 module_baseline_control_ops(),
11571 )
11572 .unwrap();
11573 let (module_tx, _module_rx) = mpsc::channel(8);
11574 let endpoint = forwarding
11575 .register_module_connection(
11576 module_connection,
11577 module_id.to_string(),
11578 PROTOCOL_VERSION,
11579 manifest_concurrency(®istration.manifest),
11580 FrameSink::new(module_tx),
11581 )
11582 .unwrap();
11583 let (client_ctx, _client_rx) = route_ctx(client_connection);
11584 let pending = forwarding
11585 .begin_route_bind_relay_for_test(
11586 client_connection,
11587 client_ctx.egress.clone(),
11588 1,
11589 module_id,
11590 )
11591 .unwrap();
11592 assert_eq!(pending.endpoint, endpoint);
11593 let route_channel = pending.client_channel;
11594 let route_epoch = pending.client_epoch;
11595 forwarding
11596 .complete_pending_relay(
11597 module_connection,
11598 pending.corr,
11599 RouteBindRelayOutcome::Accepted,
11600 )
11601 .unwrap();
11602 (client_ctx, route_channel, route_epoch)
11603 }
11604
11605 struct FakeProcessLiveness {
11606 live: Option<bool>,
11607 }
11608
11609 impl ModuleProcessLiveness for FakeProcessLiveness {
11610 fn process_live(&self, _module_id: &str) -> Option<bool> {
11611 self.live
11612 }
11613 }
11614
11615 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11616 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
11617 {
11618 let registry = Arc::new(Registry::default());
11619 let supervisor_handle = SupervisorHandle::new();
11620 let supervisor = Supervisor::new_for_test(
11621 Arc::clone(®istry),
11622 RestartPolicy::new(1, Duration::from_millis(10)),
11623 )
11624 .with_handle(supervisor_handle.clone());
11625 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
11626 let module = supervisor
11627 .spawn(ModuleSpec {
11628 script: None,
11629 carry_containment: false,
11630 module_id: "stderr-tail-wire".to_string(),
11631 program: fake_aft_stub_path(),
11632 args: Vec::new(),
11633 env: vec![
11634 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
11635 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
11636 ],
11637 reserved: false,
11638 reserved_prefixes: Vec::new(),
11639 protocol: ModuleProtocol::Subc,
11640 overlap: Default::default(),
11641 })
11642 .unwrap();
11643
11644 let deadline = Instant::now() + Duration::from_secs(5);
11645 loop {
11646 let tail = module.stderr_tail(None, None);
11647 if tail
11648 .entries
11649 .iter()
11650 .any(|entry| matches!(entry, TailEntry::ProcessStart))
11651 && tail.entries.iter().any(|entry| {
11652 matches!(
11653 entry,
11654 TailEntry::Line {
11655 truncated: true,
11656 ..
11657 }
11658 )
11659 })
11660 {
11661 break;
11662 }
11663 assert!(
11664 Instant::now() < deadline,
11665 "module did not produce a truncated line and restart boundary: {tail:?}"
11666 );
11667 sleep(Duration::from_millis(10)).await;
11668 }
11669
11670 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11671 let request = ClientControlRequest::SupervisorStderrTail {
11672 module_id: "stderr-tail-wire".to_string(),
11673 max_lines: None,
11674 max_bytes: None,
11675 };
11676 let frame = Frame::build(
11677 FrameType::Request,
11678 control_flags(),
11679 0,
11680 0,
11681 1,
11682 serde_json::to_vec(&request).unwrap(),
11683 )
11684 .unwrap();
11685 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11686 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11687 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
11688 serde_json::from_slice(&responses[0].body).unwrap()
11689 else {
11690 panic!("expected supervisor.stderr_tail response");
11691 };
11692
11693 assert!(
11694 tail.entries
11695 .iter()
11696 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
11697 "the control response lost the restart boundary"
11698 );
11699 let Some(StderrTailEntry::Line {
11700 text,
11701 truncated,
11702 at_ms,
11703 }) = tail.entries.iter().find(|entry| {
11704 matches!(
11705 entry,
11706 StderrTailEntry::Line {
11707 truncated: true,
11708 ..
11709 }
11710 )
11711 })
11712 else {
11713 panic!("the control response lost the truncated line");
11714 };
11715 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
11716 assert!(*truncated);
11717 assert!(
11718 at_ms.is_some(),
11719 "the control response lost the line's capture time"
11720 );
11721 }
11722
11723 #[tokio::test(flavor = "current_thread")]
11728 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
11729 let dir = ScratchDir::new("terminals-off-worker");
11730 let journal_path = dir.join("terminals.jsonl");
11731 let registry = Arc::new(Registry::default());
11732 let supervisor_handle = SupervisorHandle::new();
11733 let supervisor =
11734 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11735 .with_handle(supervisor_handle.clone())
11736 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
11737 let module = supervisor
11738 .spawn(ModuleSpec {
11739 script: None,
11740 carry_containment: false,
11741 module_id: "terminal-off-worker".to_string(),
11742 program: fake_aft_stub_path(),
11743 args: Vec::new(),
11744 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11745 reserved: false,
11746 reserved_prefixes: Vec::new(),
11747 protocol: ModuleProtocol::Subc,
11748 overlap: Default::default(),
11749 })
11750 .unwrap();
11751 let deadline = Instant::now() + Duration::from_secs(5);
11752 while module.terminal_history().entries.len() != 2 {
11753 assert!(Instant::now() < deadline, "module did not record two exits");
11754 sleep(Duration::from_millis(10)).await;
11755 }
11756
11757 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
11758 let handler =
11759 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
11760 let frame = Frame::build(
11761 FrameType::Request,
11762 control_flags(),
11763 0,
11764 0,
11765 1,
11766 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
11767 module_id: "terminal-off-worker".to_string(),
11768 })
11769 .unwrap(),
11770 )
11771 .unwrap();
11772 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11773 let spawned_at = std::time::Instant::now();
11774 let read = tokio::spawn({
11775 let handler = Arc::clone(&handler);
11776 async move { handler.handle_control_frame(&ctx, frame).await }
11777 });
11778 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
11781 .await
11782 .unwrap()
11783 .expect("the history read reached its pause");
11784 let elapsed = spawned_at.elapsed();
11785 assert!(
11786 elapsed < Duration::from_secs(2) && !read.is_finished(),
11787 "this task could not run while the history read was paused \
11788 (resumed after {elapsed:?}, read finished: {})",
11789 read.is_finished()
11790 );
11791
11792 drop(release);
11793 let responses = read.await.unwrap().unwrap();
11794 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11795 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
11796 panic!("expected supervisor.terminals response");
11797 };
11798 assert_eq!(terminals.entries.len(), 2);
11799 assert_eq!(terminals.journal_skipped_lines, 0);
11800 assert_eq!(terminals.journal_read_errors, 0);
11801 }
11802
11803 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11804 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
11805 let registry = Arc::new(Registry::default());
11806 let supervisor_handle = SupervisorHandle::new();
11807 let supervisor =
11808 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11809 .with_handle(supervisor_handle.clone());
11810 let module = supervisor
11811 .spawn(ModuleSpec {
11812 script: None,
11813 carry_containment: false,
11814 module_id: "terminal-golden".to_string(),
11815 program: fake_aft_stub_path(),
11816 args: Vec::new(),
11817 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11818 reserved: false,
11819 reserved_prefixes: Vec::new(),
11820 protocol: ModuleProtocol::Subc,
11821 overlap: Default::default(),
11822 })
11823 .unwrap();
11824
11825 let deadline = Instant::now() + Duration::from_secs(5);
11826 while module.terminal_history().entries.len() != 2 {
11827 assert!(
11828 Instant::now() < deadline,
11829 "module did not retain two terminal exits: {:?}",
11830 module.terminal_history()
11831 );
11832 sleep(Duration::from_millis(10)).await;
11833 }
11834
11835 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11836 let request = ClientControlRequest::SupervisorTerminals {
11837 module_id: "terminal-golden".to_string(),
11838 };
11839 let frame = Frame::build(
11840 FrameType::Request,
11841 control_flags(),
11842 0,
11843 0,
11844 1,
11845 serde_json::to_vec(&request).unwrap(),
11846 )
11847 .unwrap();
11848 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11849 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11850 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11851 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
11852 panic!("expected supervisor.terminals response");
11853 };
11854 assert_eq!(terminals.entries.len(), 2);
11855 assert_eq!(terminals.dropped, 0);
11856
11857 let mut rendered = serde_json::to_value(response).unwrap();
11858 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
11861 for (index, entry) in rendered["entries"]
11862 .as_array_mut()
11863 .expect("terminal response entries array")
11864 .iter_mut()
11865 .enumerate()
11866 {
11867 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
11868 }
11869
11870 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11871 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
11872 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
11873 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11874 std::fs::write(&golden_path, &serialized).unwrap();
11875 }
11876 let expected: Value =
11877 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
11878 assert_eq!(rendered, expected);
11879 }
11880
11881 #[test]
11882 fn hello_registers_manifest_and_returns_ack() {
11883 let registry = Arc::new(Registry::default());
11884 let handler = ControlHandler::new(Arc::clone(®istry));
11885 let conn = ConnectionId::new(1);
11886
11887 let responses = handler
11888 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
11889 .unwrap();
11890
11891 assert_eq!(responses.len(), 1);
11892 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11893 assert_eq!(responses[0].header.channel, 0);
11894 assert_eq!(responses[0].header.corr, 7);
11895 let ack = parse_ack(&responses[0]);
11896 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11897 assert!(ack
11898 .subc_capabilities
11899 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
11900 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
11901 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
11902 assert!(ack
11903 .subc_ops
11904 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
11905 assert!(ack
11906 .subc_ops
11907 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
11908 assert!(ack
11909 .subc_ops
11910 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
11911
11912 let registration = registry.get_module("aft").unwrap().unwrap();
11913 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
11914 assert_eq!(registration.state, ChannelState::Active);
11915 assert_eq!(registration.connection_id, conn);
11916 assert_eq!(registration.control_ops, module_baseline_control_ops());
11917 assert!(registration.manifest.display_name.is_none());
11918 }
11919
11920 #[test]
11921 fn hello_display_name_refuses_invalid_manifest_by_field() {
11922 let registry = Arc::new(Registry::default());
11923 let handler = ControlHandler::new(Arc::clone(®istry));
11924 for (index, invalid) in [
11925 json!(""),
11926 json!("a".repeat(41)),
11927 json!(" Routing"),
11928 json!("a\nb"),
11929 json!("a\u{202e}b"),
11930 json!(7),
11931 ]
11932 .into_iter()
11933 .enumerate()
11934 {
11935 let module_id = format!("bad-display-name-{index}");
11936 let mut body = serde_json::to_value(ModuleHelloBody {
11937 manifest: manifest(&module_id, PROTOCOL_VERSION),
11938 protocol_ver: PROTOCOL_VERSION,
11939 control_ops: None,
11940 launch_nonce: None,
11941 })
11942 .unwrap();
11943 body["manifest"]["display_name"] = invalid;
11944 let hello = Frame::build(
11945 FrameType::Hello,
11946 control_flags(),
11947 0,
11948 0,
11949 7,
11950 serde_json::to_vec(&body).unwrap(),
11951 )
11952 .unwrap();
11953 let response = handler
11954 .handle_control(ConnectionId::new(100 + index as u64), hello)
11955 .unwrap();
11956 assert_eq!(response[0].header.ty, FrameType::Error);
11957 let error = parse_error(&response[0]);
11958 assert_eq!(error["code"], "invalid_manifest");
11959 assert!(error["message"].as_str().unwrap().contains("display_name"));
11960 assert!(registry.get_module(&module_id).unwrap().is_none());
11961 }
11962 }
11963
11964 #[test]
11965 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
11966 let invalid_identifiers = [
11967 ("case_change", "credentials-Provider/v1"),
11968 ("leading_zero", "credentials-provider/v01"),
11969 ("trailing_hyphen", "credentials-provider-/v1"),
11970 ("consecutive_hyphens", "credentials--provider/v1"),
11971 ("uppercase", "Credentials-provider/v1"),
11972 ("missing_v", "credentials-provider/1"),
11973 ("whitespace", "credentials provider/v1"),
11974 ("zero_version", "credentials-provider/v0"),
11975 ("out_of_range_version", "credentials-provider/v4294967296"),
11976 (
11977 "overlength_name",
11978 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
11979 ),
11980 ];
11981 let mut cases = invalid_identifiers
11982 .into_iter()
11983 .map(|(name, identifier)| {
11984 (
11985 format!("identifier_{name}"),
11986 "capabilities.provides[0]".to_string(),
11987 identifier.to_string(),
11988 json!({ "provides": [identifier] }),
11989 None,
11990 )
11991 })
11992 .collect::<Vec<_>>();
11993 cases.extend([
11994 (
11995 "unknown_need".to_string(),
11996 "capabilities.requires[0].need".to_string(),
11997 "deferred".to_string(),
11998 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
11999 None,
12000 ),
12001 (
12002 "duplicate_provides".to_string(),
12003 "capabilities.provides[1]".to_string(),
12004 "credentials-provider/v1".to_string(),
12005 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
12006 None,
12007 ),
12008 (
12009 "duplicate_must_never_reach".to_string(),
12010 "capabilities.must_never_reach[1]".to_string(),
12011 "credentials-provider/v1".to_string(),
12012 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
12013 None,
12014 ),
12015 (
12016 "duplicate_requires_same_need".to_string(),
12017 "capabilities.requires[1]".to_string(),
12018 "credentials-provider/v1".to_string(),
12019 json!({ "requires": [
12020 { "capability": "credentials-provider/v1", "need": "required" },
12021 { "capability": "credentials-provider/v1", "need": "required" }
12022 ] }),
12023 None,
12024 ),
12025 (
12026 "duplicate_requires_conflicting_need".to_string(),
12027 "capabilities.requires[1]".to_string(),
12028 "credentials-provider/v1".to_string(),
12029 json!({ "requires": [
12030 { "capability": "credentials-provider/v1", "need": "required" },
12031 { "capability": "credentials-provider/v1", "need": "optional" }
12032 ] }),
12033 None,
12034 ),
12035 (
12036 "capabilities_root_pointer".to_string(),
12037 "runtime_computed[0]".to_string(),
12038 "/capabilities".to_string(),
12039 json!({}),
12040 Some(json!(["/capabilities"])),
12041 ),
12042 (
12043 "capabilities_descendant_pointer".to_string(),
12044 "runtime_computed[0]".to_string(),
12045 "/capabilities/provides".to_string(),
12046 json!({}),
12047 Some(json!(["/capabilities/provides"])),
12048 ),
12049 (
12050 "malformed_pointer_without_leading_slash".to_string(),
12051 "runtime_computed[0]".to_string(),
12052 "capabilities".to_string(),
12053 json!({}),
12054 Some(json!(["capabilities"])),
12055 ),
12056 (
12057 "malformed_pointer_escape".to_string(),
12058 "runtime_computed[0]".to_string(),
12059 "/roles/~2/tools".to_string(),
12060 json!({}),
12061 Some(json!(["/roles/~2/tools"])),
12062 ),
12063 (
12064 "unknown_capabilities_field".to_string(),
12065 "capabilities.future".to_string(),
12066 "<array>".to_string(),
12067 json!({ "future": [] }),
12068 None,
12069 ),
12070 ]);
12071
12072 for (index, (name, field, value, capabilities, runtime_computed)) in
12073 cases.into_iter().enumerate()
12074 {
12075 let registry = Arc::new(Registry::default());
12076 let handler = ControlHandler::new(Arc::clone(®istry));
12077 let response = handler
12078 .handle_control(
12079 ConnectionId::new((index + 1) as u64),
12080 capability_grammar_hello_frame(
12081 capabilities,
12082 runtime_computed,
12083 index as u64 + 1,
12084 ),
12085 )
12086 .expect("invalid HELLO returns a refusal");
12087
12088 assert_eq!(response.len(), 1, "{name} must emit one refusal");
12089 let error = parse_error(&response[0]);
12090 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
12091 let message = error["message"]
12092 .as_str()
12093 .expect("error message is a string");
12094 assert!(
12095 message.contains(&field),
12096 "{name}: field missing from {message}"
12097 );
12098 assert!(
12099 message.contains(&value),
12100 "{name}: value missing from {message}"
12101 );
12102 assert_eq!(
12103 registry
12104 .active_registration_count()
12105 .expect("registry reads"),
12106 0,
12107 "{name}: refused HELLO must not create a catalog entry"
12108 );
12109 }
12110 }
12111
12112 #[test]
12113 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
12114 let registry = Arc::new(Registry::default());
12115 let handler = ControlHandler::new(Arc::clone(®istry));
12116 let capabilities = json!({
12117 "provides": ["credentials-provider/v1"],
12118 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
12119 "must_never_reach": ["federation-transport/v1"]
12120 });
12121 let response = handler
12122 .handle_control(
12123 ConnectionId::new(99),
12124 capability_grammar_hello_frame(
12125 capabilities.clone(),
12126 Some(json!(["/roles/0/tools"])),
12127 99,
12128 ),
12129 )
12130 .expect("valid HELLO registers");
12131 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12132
12133 let request = Frame::build(
12134 FrameType::Request,
12135 control_flags(),
12136 0,
12137 0,
12138 100,
12139 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12140 .expect("catalog request serializes"),
12141 )
12142 .expect("catalog request frame builds");
12143 let response = handler
12144 .handle_catalog_list(request, None)
12145 .expect("catalog list succeeds");
12146 let ClientControlResponse::CatalogList { modules, .. } =
12147 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
12148 else {
12149 panic!("catalog request must return catalog.list");
12150 };
12151 assert_eq!(modules.len(), 1);
12152 assert_eq!(
12153 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
12154 capabilities
12155 );
12156 }
12157
12158 #[test]
12159 fn catalog_list_mirrors_management_operation_description() {
12160 let registry = Arc::new(Registry::default());
12161 let handler = ControlHandler::new(Arc::clone(®istry));
12162 let description = "List managed records and return their identifiers and metadata.";
12163 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
12164 manifest.provides = vec![ProviderRole::ManagementSurface {
12165 operations: vec![ManagementOperation {
12166 name: "records.list".to_string(),
12167 kind: ManagementOperationKind::Query,
12168 description: Some(description.to_string()),
12169 }],
12170 config_schema: json!({"type": "object"}),
12171 observability: vec![ObservabilitySurface {
12172 name: "records.stats".to_string(),
12173 kind: ObservabilityKind::Snapshot,
12174 }],
12175 identity_scope: vec![IdentityScope::Project],
12176 concurrency: Concurrency::ModuleManaged,
12177 }];
12178 registry
12179 .register_with_control_ops(
12180 manifest,
12181 PROTOCOL_VERSION,
12182 ConnectionId::new(99),
12183 Vec::new(),
12184 )
12185 .expect("described management manifest registers");
12186
12187 let request = Frame::build(
12188 FrameType::Request,
12189 control_flags(),
12190 0,
12191 0,
12192 100,
12193 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12194 .expect("catalog request serializes"),
12195 )
12196 .expect("catalog request frame builds");
12197 let response = handler
12198 .handle_catalog_list(request, None)
12199 .expect("catalog list succeeds");
12200 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
12201 assert_eq!(
12202 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
12203 "catalog.list must preserve the declared operation description verbatim"
12204 );
12205 }
12206
12207 #[test]
12208 fn catalog_list_mirrors_registered_operation_delivery() {
12209 let registry = Arc::new(Registry::default());
12210 let handler = ControlHandler::new(Arc::clone(®istry));
12211 let delivery =
12212 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
12213 let mut body = serde_json::to_value(ModuleHelloBody {
12214 manifest: manifest("watch-management", PROTOCOL_VERSION),
12215 protocol_ver: PROTOCOL_VERSION,
12216 control_ops: None,
12217 launch_nonce: None,
12218 })
12219 .unwrap();
12220 body["manifest"]["provides"] = json!([{
12221 "role": "management_surface",
12222 "operations": [
12223 {"name": "status", "kind": "query"},
12224 {"name": "future", "kind": "query"},
12225 {"name": "reset", "kind": "mutate"}
12226 ],
12227 "config_schema": {}, "observability": [], "identity_scope": []
12228 }]);
12229 body["manifest"]["operation_delivery"] = delivery.clone();
12230 let hello = Frame::build(
12231 FrameType::Hello,
12232 control_flags(),
12233 0,
12234 0,
12235 99,
12236 serde_json::to_vec(&body).unwrap(),
12237 )
12238 .unwrap();
12239 let response = handler
12240 .handle_control(ConnectionId::new(99), hello)
12241 .unwrap();
12242 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12243
12244 let (_, registrations) = registry.list_modules().unwrap();
12245 let registered = ®istrations[0].manifest;
12246 assert!(registered.is_watch_operation("status"));
12247 assert!(!registered.is_watch_operation("future"));
12248 assert!(!registered.is_watch_operation("reset"));
12249 assert!(!registered.is_watch_operation("missing"));
12250
12251 let request = Frame::build(
12252 FrameType::Request,
12253 control_flags(),
12254 0,
12255 0,
12256 100,
12257 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12258 )
12259 .unwrap();
12260 let response = handler.handle_catalog_list(request, None).unwrap();
12261 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
12262 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
12263 let ClientControlResponse::CatalogList { modules, .. } =
12264 serde_json::from_slice(&response[0].body).unwrap()
12265 else {
12266 panic!("expected catalog.list");
12267 };
12268 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
12269 }
12270
12271 #[test]
12272 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
12273 let registry = Arc::new(Registry::default());
12274 let supervisor = SupervisorHandle::new();
12275 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12276 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12277 let descriptor = json!({
12278 "schema": "ckios.richui.manifest/1",
12279 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12280 "bytes": 4096,
12281 "chunks": 4
12282 });
12283 let connection = ConnectionId::new(201);
12284 let hello = handler
12285 .handle_control(
12286 connection,
12287 hello_frame_with_rich_ui(
12288 "vault",
12289 PROTOCOL_VERSION,
12290 201,
12291 Some("the-real-nonce"),
12292 descriptor.clone(),
12293 ),
12294 )
12295 .unwrap();
12296 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12297
12298 let request =
12299 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
12300 let provides = registry
12301 .get_module_by_connection(connection)
12302 .unwrap()
12303 .unwrap()
12304 .manifest
12305 .provides;
12306 handler
12307 .handle_catalog_update(connection, request, provides, None, Some(false))
12308 .unwrap();
12309
12310 let request = Frame::build(
12311 FrameType::Request,
12312 control_flags(),
12313 0,
12314 0,
12315 203,
12316 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12317 )
12318 .unwrap();
12319 let response = handler.handle_catalog_list(request, None).unwrap();
12320 let ClientControlResponse::CatalogList { modules, .. } =
12321 serde_json::from_slice(&response[0].body).unwrap()
12322 else {
12323 panic!("expected catalog.list");
12324 };
12325 assert_eq!(
12326 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
12327 descriptor
12328 );
12329 assert!(modules[0].nonce_verified);
12330 }
12331
12332 fn hello_with_role_declarations(declarations: Value) -> Frame {
12333 let mut body = serde_json::to_value(ModuleHelloBody {
12334 manifest: manifest("role-test", PROTOCOL_VERSION),
12335 protocol_ver: PROTOCOL_VERSION,
12336 control_ops: None,
12337 launch_nonce: None,
12338 })
12339 .unwrap();
12340 body["manifest"]["role_declarations"] = declarations;
12341 Frame::build(
12342 FrameType::Hello,
12343 control_flags(),
12344 0,
12345 0,
12346 301,
12347 serde_json::to_vec(&body).unwrap(),
12348 )
12349 .unwrap()
12350 }
12351
12352 fn update_role_declarations_frame(declarations: Value) -> Frame {
12353 Frame::build(
12354 FrameType::Request,
12355 control_flags(),
12356 0,
12357 0,
12358 303,
12359 serde_json::to_vec(&json!({
12360 "op": "catalog.update_role_declarations",
12361 "role_declarations": declarations,
12362 }))
12363 .unwrap(),
12364 )
12365 .unwrap()
12366 }
12367
12368 #[tokio::test]
12369 async fn role_declarations_update_replaces_then_clears_catalog() {
12370 let registry = Arc::new(Registry::default());
12371 let handler = ControlHandler::new(Arc::clone(®istry));
12372 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12373 let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
12374 hello_via_sink(
12375 &handler,
12376 &ctx,
12377 &mut rx,
12378 hello_with_role_declarations(initial.clone()),
12379 )
12380 .await;
12381 assert_eq!(
12382 role_declarations_catalog(&handler).0["role_declarations"],
12383 initial
12384 );
12385 let before = registry.get_module("role-test").unwrap().unwrap();
12386 for declarations in [
12387 json!({"media-provider/v1": {"engines": ["new"]}}),
12388 json!({}),
12389 ] {
12390 let response = handler
12391 .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
12392 .await
12393 .unwrap();
12394 assert_eq!(response[0].header.ty, FrameType::Response);
12395 assert_eq!(
12396 serde_json::from_slice::<Value>(&response[0].body).unwrap(),
12397 json!({"op": "catalog.update_role_declarations"})
12398 );
12399 let (raw, entry) = role_declarations_catalog(&handler);
12400 assert_eq!(
12401 serde_json::to_value(entry.role_declarations).unwrap(),
12402 declarations
12403 );
12404 if declarations.as_object().unwrap().is_empty() {
12405 assert!(raw.get("role_declarations").is_none());
12406 } else {
12407 assert_eq!(raw["role_declarations"], declarations);
12408 }
12409 let after = registry.get_module("role-test").unwrap().unwrap();
12410 let mut expected = before.manifest.clone();
12411 expected.role_declarations = serde_json::from_value(declarations).unwrap();
12412 assert_eq!(after.manifest, expected);
12413 assert_eq!(after.ready, before.ready);
12414 }
12415 }
12416
12417 #[tokio::test]
12418 async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
12419 let registry = Arc::new(Registry::default());
12420 let handler = ControlHandler::new(registry);
12421 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12422 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12423 hello_via_sink(
12424 &handler,
12425 &ctx,
12426 &mut rx,
12427 hello_with_role_declarations(initial.clone()),
12428 )
12429 .await;
12430 for (invalid, reason) in [
12431 (json!(null), "role_declarations must be an object"),
12432 (json!([]), "role_declarations must be an object"),
12433 (
12434 json!({"bad key": {}}),
12435 "role_declarations key must use capability-name grammar",
12436 ),
12437 (
12438 json!({"media-provider/v1": []}),
12439 "role_declarations values must be objects",
12440 ),
12441 (
12442 json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
12443 "role_declarations exceeds 16384 compact JSON bytes",
12444 ),
12445 ] {
12446 let response = handler
12447 .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
12448 .await
12449 .unwrap();
12450 let error = parse_error(&response[0]);
12451 assert_eq!(error["code"], "invalid_role_declarations");
12452 assert_eq!(error["message"], reason);
12453 assert_eq!(
12454 role_declarations_catalog(&handler).0["role_declarations"],
12455 initial
12456 );
12457 }
12458 }
12459
12460 #[tokio::test]
12461 async fn role_declarations_update_is_owned_by_the_registered_connection() {
12462 let registry = Arc::new(Registry::default());
12463 let handler = ControlHandler::new(Arc::clone(®istry));
12464 let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
12465 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12466 hello_via_sink(
12467 &handler,
12468 &owner,
12469 &mut owner_rx,
12470 hello_with_role_declarations(initial.clone()),
12471 )
12472 .await;
12473 let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
12474 let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
12475 let response = handler
12476 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12477 .await
12478 .unwrap();
12479 assert_eq!(parse_error(&response[0])["code"], "not_registered");
12480 hello_via_sink(
12481 &handler,
12482 &other,
12483 &mut other_rx,
12484 hello_frame("other", PROTOCOL_VERSION, 304),
12485 )
12486 .await;
12487 let response = handler
12488 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12489 .await
12490 .unwrap();
12491 assert_eq!(response[0].header.ty, FrameType::Response);
12492 assert_eq!(
12493 serde_json::to_value(
12494 registry
12495 .get_module("role-test")
12496 .unwrap()
12497 .unwrap()
12498 .manifest
12499 .role_declarations
12500 )
12501 .unwrap(),
12502 initial
12503 );
12504 assert_eq!(
12505 serde_json::to_value(
12506 registry
12507 .get_module("other")
12508 .unwrap()
12509 .unwrap()
12510 .manifest
12511 .role_declarations
12512 )
12513 .unwrap(),
12514 replacement
12515 );
12516 }
12517
12518 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
12519 let request = Frame::build(
12520 FrameType::Request,
12521 control_flags(),
12522 0,
12523 0,
12524 302,
12525 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12526 )
12527 .unwrap();
12528 let response = handler.handle_catalog_list(request, None).unwrap();
12529 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
12530 let ClientControlResponse::CatalogList { mut modules, .. } =
12531 serde_json::from_slice(&response[0].body).unwrap()
12532 else {
12533 panic!("expected catalog.list");
12534 };
12535 (raw["modules"][0].clone(), modules.remove(0))
12536 }
12537
12538 #[test]
12539 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
12540 let registry = Arc::new(Registry::default());
12541 let handler = ControlHandler::new(Arc::clone(®istry));
12542 let declarations = json!({
12543 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
12544 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
12545 });
12546 let connection = ConnectionId::new(301);
12547 let response = handler
12548 .handle_control(
12549 connection,
12550 hello_with_role_declarations(declarations.clone()),
12551 )
12552 .unwrap();
12553 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12554 let registered = registry
12555 .get_module_by_connection(connection)
12556 .unwrap()
12557 .unwrap()
12558 .manifest;
12559 assert_eq!(
12563 serde_json::to_value(®istered.role_declarations).unwrap(),
12564 declarations
12565 );
12566 let (raw, entry) = role_declarations_catalog(&handler);
12567 assert_eq!(raw["role_declarations"], declarations);
12568 assert_eq!(entry.role_declarations, registered.role_declarations);
12569
12570 let request =
12571 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
12572 handler
12573 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
12574 .unwrap();
12575 let (raw, entry) = role_declarations_catalog(&handler);
12576 assert_eq!(raw["role_declarations"], declarations);
12577 assert_eq!(entry.role_declarations, registered.role_declarations);
12578 }
12579
12580 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
12581 let registry = Arc::new(Registry::default());
12582 let handler = ControlHandler::new(Arc::clone(®istry));
12583 let capture = EventCapture::default();
12584 let _subscriber =
12585 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12586 let response = handler
12587 .handle_control(
12588 ConnectionId::new(301),
12589 hello_with_role_declarations(declarations),
12590 )
12591 .unwrap();
12592 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12593 assert!(registry
12594 .get_module_by_connection(ConnectionId::new(301))
12595 .unwrap()
12596 .unwrap()
12597 .manifest
12598 .role_declarations
12599 .is_empty());
12600 let (raw, entry) = role_declarations_catalog(&handler);
12601 assert!(raw.get("role_declarations").is_none());
12602 assert!(entry.role_declarations.is_empty());
12603 let warnings: Vec<_> = capture
12604 .events()
12605 .into_iter()
12606 .filter(|event| {
12607 event.fields.get("message").map(String::as_str)
12608 == Some("dropping whole role_declarations map from HELLO")
12609 })
12610 .collect();
12611 assert_eq!(warnings.len(), 1);
12612 assert_eq!(warnings[0].level, tracing::Level::WARN);
12613 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
12614 assert_eq!(warnings[0].fields["reason"], reason);
12615 }
12616
12617 #[test]
12618 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
12619 for value in [
12620 json!(null),
12621 json!(false),
12622 json!(0),
12623 json!("opaque"),
12624 json!([]),
12625 ] {
12626 assert_hello_drops_role_declarations_with_warning(
12627 json!({"routing/v1": {}, "broken/v1": value}),
12628 "role_declarations values must be objects",
12629 );
12630 }
12631 assert_hello_drops_role_declarations_with_warning(
12632 json!([]),
12633 "role_declarations must be an object",
12634 );
12635 }
12636
12637 #[test]
12638 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
12639 assert_hello_drops_role_declarations_with_warning(
12640 json!({"routing/v1": {}, "bad key": {}}),
12641 "role_declarations key must use capability-name grammar",
12642 );
12643 }
12644
12645 #[test]
12646 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
12647 assert_hello_drops_role_declarations_with_warning(
12648 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
12649 "role_declarations exceeds 16384 compact JSON bytes",
12650 );
12651 }
12652
12653 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
12654 let mut body = serde_json::to_value(ModuleHelloBody {
12655 manifest: manifest("flow-ops", PROTOCOL_VERSION),
12656 protocol_ver: PROTOCOL_VERSION,
12657 control_ops: None,
12658 launch_nonce: None,
12659 })
12660 .unwrap();
12661 body["manifest"]["provides"] = json!([{
12662 "role": "management_surface",
12663 "operations": [
12664 {"name": "diagnostics.findings", "kind": "query"},
12665 {"name": "reset", "kind": "mutate"}
12666 ],
12667 "config_schema": {}, "observability": [], "identity_scope": []
12668 }]);
12669 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
12670 if let Some(flow_operations) = flow_operations {
12671 body["manifest"]["flow_operations"] = flow_operations;
12672 }
12673 Frame::build(
12674 FrameType::Hello,
12675 control_flags(),
12676 0,
12677 0,
12678 401,
12679 serde_json::to_vec(&body).unwrap(),
12680 )
12681 .unwrap()
12682 }
12683
12684 #[test]
12685 fn catalog_list_mirrors_flow_operations_unchanged() {
12686 for (capability, flow_operations) in [
12689 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
12690 ("call-modes/v1", json!(["diagnostics.findings"])),
12691 ("flow-scopes/v1", json!([])),
12692 ] {
12693 let registry = Arc::new(Registry::default());
12694 let handler = ControlHandler::new(Arc::clone(®istry));
12695 let connection = ConnectionId::new(401);
12696 let response = handler
12697 .handle_control(
12698 connection,
12699 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
12700 )
12701 .unwrap();
12702 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12703 let registered = registry
12704 .get_module_by_connection(connection)
12705 .unwrap()
12706 .unwrap()
12707 .manifest;
12708 assert_eq!(
12709 serde_json::to_value(®istered.flow_operations).unwrap(),
12710 flow_operations
12711 );
12712 let (raw, entry) = role_declarations_catalog(&handler);
12713 assert_eq!(raw["flow_operations"], flow_operations);
12714 assert_eq!(entry.flow_operations, registered.flow_operations);
12715 }
12716
12717 let registry = Arc::new(Registry::default());
12718 let handler = ControlHandler::new(Arc::clone(®istry));
12719 let response = handler
12720 .handle_control(
12721 ConnectionId::new(401),
12722 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
12723 )
12724 .unwrap();
12725 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12726 let (raw, entry) = role_declarations_catalog(&handler);
12727 assert!(raw.get("flow_operations").is_none());
12728 assert_eq!(entry.flow_operations, None);
12729 }
12730
12731 #[test]
12732 fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
12733 let oversized: serde_json::Map<String, Value> = (0..257)
12734 .map(|n| (format!("tool:t{n}"), json!(true)))
12735 .collect();
12736 for (capabilities, map, valid) in [
12737 (
12738 json!(["scope-federated/v1"]),
12739 json!({"tool:read":true,"management:reset":true}),
12740 true,
12741 ),
12742 (json!(["scope-federated/v1"]), json!({}), true),
12743 (json!([]), json!({"tool:read":true}), false),
12744 (
12745 json!(["scope-federated/v1"]),
12746 json!({"tool:valid":true,"tool:bad":false}),
12747 false,
12748 ),
12749 (
12750 json!(["scope-federated/v1"]),
12751 json!({"tool:valid":true,"management:missing":true}),
12752 false,
12753 ),
12754 (
12755 json!(["scope-federated/v1"]),
12756 Value::Object(oversized),
12757 false,
12758 ),
12759 ] {
12760 let capture = EventCapture::default();
12761 let _subscriber = tracing::subscriber::set_default(
12762 tracing_subscriber::registry().with(capture.clone()),
12763 );
12764 let registry = Arc::new(Registry::default());
12765 let handler = ControlHandler::new(Arc::clone(®istry));
12766 let hello = hello_with_flow_operations(capabilities, None);
12767 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12768 body["manifest"]["operation_deferral"] = map.clone();
12769 let hello = Frame::build(
12770 FrameType::Hello,
12771 control_flags(),
12772 0,
12773 0,
12774 401,
12775 serde_json::to_vec(&body).unwrap(),
12776 )
12777 .unwrap();
12778 let response = handler
12779 .handle_control(ConnectionId::new(401), hello)
12780 .unwrap();
12781 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12782 let (raw, entry) = role_declarations_catalog(&handler);
12783 if valid {
12784 assert_eq!(raw["operation_deferral"], map);
12785 assert_eq!(
12786 serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
12787 map
12788 );
12789 } else {
12790 assert!(entry.operation_deferral.is_none());
12791 assert!(raw.get("operation_deferral").is_none());
12792 assert_eq!(
12793 capture
12794 .events()
12795 .iter()
12796 .filter(|event| event.fields.get("message").map(String::as_str)
12797 == Some("dropping operation_deferral from HELLO"))
12798 .count(),
12799 1
12800 );
12801 }
12802 }
12803 }
12804
12805 fn assert_hello_drops_flow_operations_with_warning(
12806 capabilities: Value,
12807 flow_operations: Value,
12808 reason: &str,
12809 ) {
12810 let registry = Arc::new(Registry::default());
12811 let handler = ControlHandler::new(Arc::clone(®istry));
12812 let capture = EventCapture::default();
12813 let _subscriber =
12814 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12815 let response = handler
12816 .handle_control(
12817 ConnectionId::new(401),
12818 hello_with_flow_operations(capabilities, Some(flow_operations)),
12819 )
12820 .unwrap();
12821 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12822 let registered = registry
12823 .get_module_by_connection(ConnectionId::new(401))
12824 .unwrap()
12825 .expect("the module still registers")
12826 .manifest;
12827 assert_eq!(registered.module_id, "flow-ops");
12828 assert_eq!(registered.flow_operations, None);
12829 let (raw, entry) = role_declarations_catalog(&handler);
12830 assert_eq!(raw["module_id"], "flow-ops");
12831 assert!(raw.get("flow_operations").is_none());
12832 assert_eq!(entry.flow_operations, None);
12833 let warnings: Vec<_> = capture
12834 .events()
12835 .into_iter()
12836 .filter(|event| {
12837 event.fields.get("message").map(String::as_str)
12838 == Some("dropping flow_operations from HELLO")
12839 })
12840 .collect();
12841 assert_eq!(warnings.len(), 1);
12842 assert_eq!(warnings[0].level, tracing::Level::WARN);
12843 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
12844 assert_eq!(warnings[0].fields["reason"], reason);
12845 }
12846
12847 #[test]
12848 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
12849 assert_hello_drops_flow_operations_with_warning(
12850 json!(["flow-scopes/v1"]),
12851 json!(["diagnostics.findings", "missing"]),
12852 "flow_operations names 'missing', which is not a declared management operation",
12853 );
12854 }
12855
12856 #[test]
12857 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
12858 assert_hello_drops_flow_operations_with_warning(
12859 json!([]),
12860 json!(["diagnostics.findings"]),
12861 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12862 );
12863 assert_hello_drops_flow_operations_with_warning(
12865 json!(["agent-run-scopes/v1"]),
12866 json!([]),
12867 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12868 );
12869 }
12870
12871 #[test]
12872 fn hello_malformed_flow_operations_is_dropped_with_warning() {
12873 assert_hello_drops_flow_operations_with_warning(
12874 json!(["flow-scopes/v1"]),
12875 json!(["reset", "reset"]),
12876 "flow_operations names 'reset' more than once",
12877 );
12878 assert_hello_drops_flow_operations_with_warning(
12879 json!(["flow-scopes/v1"]),
12880 json!(vec!["reset"; 257]),
12881 "flow_operations lists 257 names, more than 256",
12882 );
12883 assert_hello_drops_flow_operations_with_warning(
12884 json!(["flow-scopes/v1"]),
12885 json!("reset"),
12886 "flow_operations must be an array of operation names",
12887 );
12888 }
12889
12890 #[test]
12891 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
12892 let registry = Arc::new(Registry::default());
12893 let handler = ControlHandler::new(registry.clone());
12894 let descriptor = json!({
12895 "schema": "ckios.richui.manifest/1",
12896 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12897 "bytes": 128,
12898 "chunks": 2
12899 });
12900 let malformed = [
12901 ("schema", json!("")),
12902 ("schema", json!("a".repeat(65))),
12903 ("schema", json!("Bad schema")),
12904 (
12905 "manifest_digest",
12906 json!(format!("sha256:{}", "A".repeat(64))),
12907 ),
12908 ("manifest_digest", json!("sha256:abc")),
12909 ("bytes", json!(0)),
12910 ("bytes", json!(16_777_217)),
12911 ("bytes", json!("128")),
12912 ("chunks", json!(0)),
12913 ("chunks", json!(1025)),
12914 ("chunks", json!("2")),
12915 ];
12916 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
12917 let mut invalid = descriptor.clone();
12918 invalid[field] = invalid_value;
12919 let response = handler
12920 .handle_control(
12921 ConnectionId::new(210 + index as u64),
12922 hello_frame_with_rich_ui(
12923 &format!("bad-rich-ui-{index}"),
12924 PROTOCOL_VERSION,
12925 210 + index as u64,
12926 None,
12927 invalid,
12928 ),
12929 )
12930 .unwrap();
12931 assert_eq!(response[0].header.ty, FrameType::Error);
12932 let error = parse_error(&response[0]);
12933 assert_eq!(error["code"], "invalid_manifest");
12934 assert!(error["message"]
12935 .as_str()
12936 .unwrap()
12937 .contains(&format!("rich_ui.{field}")));
12938 assert!(registry
12939 .get_module(&format!("bad-rich-ui-{index}"))
12940 .unwrap()
12941 .is_none());
12942 }
12943 }
12944
12945 #[test]
12946 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
12947 let registry = Arc::new(Registry::default());
12948 let handler = ControlHandler::new(Arc::clone(®istry));
12949 let connection = ConnectionId::new(219);
12950 registry
12951 .register_with_control_ops(
12952 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
12953 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
12954 "bad schema",
12955 format!("sha256:{}", "a".repeat(64)),
12956 128,
12957 2,
12958 )))
12959 .build(),
12960 PROTOCOL_VERSION,
12961 connection,
12962 Vec::new(),
12963 )
12964 .unwrap();
12965 let request =
12966 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
12967 let response = handler
12968 .handle_catalog_update(connection, request, Vec::new(), None, None)
12969 .unwrap();
12970 assert_eq!(response[0].header.ty, FrameType::Error);
12971 let error = parse_error(&response[0]);
12972 assert_eq!(error["code"], "invalid_manifest");
12973 assert!(error["message"]
12974 .as_str()
12975 .unwrap()
12976 .contains("rich_ui.schema"));
12977 }
12978
12979 #[test]
12980 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
12981 let registry = Arc::new(Registry::default());
12982 let supervisor = SupervisorHandle::new();
12983 supervisor.open_swap("provider", "candidate-nonce".to_string());
12984 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12985 let connection = ConnectionId::new(218);
12986 let hello = handler
12987 .handle_control(
12988 connection,
12989 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
12990 )
12991 .unwrap();
12992 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12993 assert!(registry.list_modules().unwrap().1.is_empty());
12994
12995 registry.promote_candidate("provider").unwrap().unwrap();
12996 let request = Frame::build(
12997 FrameType::Request,
12998 control_flags(),
12999 0,
13000 0,
13001 219,
13002 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13003 )
13004 .unwrap();
13005 let response = handler.handle_catalog_list(request, None).unwrap();
13006 let ClientControlResponse::CatalogList { modules, .. } =
13007 serde_json::from_slice(&response[0].body).unwrap()
13008 else {
13009 panic!("expected catalog.list");
13010 };
13011 assert_eq!(modules.len(), 1);
13012 assert!(modules[0].nonce_verified);
13013 }
13014
13015 #[test]
13016 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
13017 for provenance_json in [
13018 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
13019 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
13020 ] {
13021 let registry = Arc::new(Registry::default());
13022 let handler = ControlHandler::new(registry);
13023 let hello = Frame::build(
13024 FrameType::Hello,
13025 control_flags(),
13026 0,
13027 0,
13028 220,
13029 serde_json::to_vec(&ModuleHelloBody {
13030 manifest: ModuleManifest::builder("built", "0.1.0")
13031 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
13032 .build(),
13033 protocol_ver: PROTOCOL_VERSION,
13034 control_ops: None,
13035 launch_nonce: None,
13036 })
13037 .unwrap(),
13038 )
13039 .unwrap();
13040 assert_eq!(
13041 handler
13042 .handle_control(ConnectionId::new(220), hello)
13043 .unwrap()[0]
13044 .header
13045 .ty,
13046 FrameType::HelloAck
13047 );
13048 let request = Frame::build(
13049 FrameType::Request,
13050 control_flags(),
13051 0,
13052 0,
13053 221,
13054 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13055 )
13056 .unwrap();
13057 let response = handler.handle_catalog_list(request, None).unwrap();
13058 let ClientControlResponse::CatalogList { modules, .. } =
13059 serde_json::from_slice(&response[0].body).unwrap()
13060 else {
13061 panic!("expected catalog.list");
13062 };
13063 assert_eq!(modules.len(), 1);
13064 assert_eq!(
13065 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13066 format!(
13067 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
13068 )
13069 .as_bytes()
13070 );
13071 }
13072 }
13073
13074 #[test]
13075 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
13076 let registry = Arc::new(Registry::default());
13077 let handler = ControlHandler::new(Arc::clone(®istry));
13078 let hello = Frame::build(
13079 FrameType::Hello,
13080 control_flags(),
13081 0,
13082 0,
13083 220,
13084 serde_json::to_vec(&ModuleHelloBody {
13085 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
13086 protocol_ver: PROTOCOL_VERSION,
13087 control_ops: None,
13088 launch_nonce: Some("unverified".to_string()),
13089 })
13090 .unwrap(),
13091 )
13092 .unwrap();
13093 assert_eq!(
13094 handler
13095 .handle_control(ConnectionId::new(220), hello)
13096 .unwrap()[0]
13097 .header
13098 .ty,
13099 FrameType::HelloAck
13100 );
13101 let request = Frame::build(
13102 FrameType::Request,
13103 control_flags(),
13104 0,
13105 0,
13106 221,
13107 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13108 )
13109 .unwrap();
13110 let response = handler.handle_catalog_list(request, None).unwrap();
13111 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
13112 assert!(wire["modules"][0].get("provenance").is_none());
13113 let ClientControlResponse::CatalogList { modules, .. } =
13114 serde_json::from_slice(&response[0].body).unwrap()
13115 else {
13116 panic!("expected catalog.list");
13117 };
13118 assert_eq!(
13119 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13120 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
13121 );
13122 assert!(!modules[0].nonce_verified);
13123 assert!(modules[0].provenance.is_none());
13124 }
13125
13126 fn catalog_legacy_projection(mut entry: CatalogEntry) -> CatalogEntry {
13128 assert_eq!(entry.first_party, Some(false));
13129 entry.incarnation = None;
13130 entry.spawn = None;
13131 entry.first_party = None;
13132 entry.origin = None;
13133 entry.requested_declarations_digest = None;
13134 entry.approved_declarations_digest = None;
13135 entry
13136 }
13137
13138 #[test]
13139 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
13140 let registry = Arc::new(Registry::default());
13141 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13142 [("vault".to_string(), true), ("squatter".to_string(), true)],
13143 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13144 );
13145 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
13146 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13147 provides: vec!["credentials-provider/v1".to_string()],
13148 requires: Vec::new(),
13149 must_never_reach: Vec::new(),
13150 });
13151 let frame = Frame::build(
13152 FrameType::Hello,
13153 control_flags(),
13154 0,
13155 0,
13156 77,
13157 serde_json::to_vec(&ModuleHelloBody {
13158 manifest: squatter,
13159 protocol_ver: PROTOCOL_VERSION,
13160 control_ops: None,
13161 launch_nonce: None,
13162 })
13163 .expect("HELLO serializes"),
13164 )
13165 .expect("HELLO frame builds");
13166 let response = handler
13167 .handle_control(ConnectionId::new(77), frame)
13168 .expect("reserved claim receives a typed refusal");
13169 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13170 assert_eq!(
13171 registry
13172 .active_registration_count()
13173 .expect("registry reads"),
13174 0,
13175 "a reserved capability refusal must not leave a catalog entry"
13176 );
13177 }
13178
13179 #[test]
13180 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
13181 for settlement in ["timeout", "inconclusive", "drop"] {
13182 let breakers = RouteBindBreakers::default();
13183 let RouteBindAdmission::Admitted {
13184 guard: mut old,
13185 probe: false,
13186 } = breakers.admit("prov")
13187 else {
13188 panic!("ordinary relay admitted")
13189 };
13190 let RouteBindAdmission::Admitted {
13191 guard: mut opener, ..
13192 } = breakers.admit("prov")
13193 else {
13194 panic!("second relay admitted")
13195 };
13196 assert!(
13197 !opener
13198 .record_timeout(1, Duration::ZERO)
13199 .unwrap()
13200 .reopened_after_probe
13201 );
13202 let RouteBindAdmission::Admitted {
13203 guard: mut probe,
13204 probe: true,
13205 } = breakers.admit("prov")
13206 else {
13207 panic!("one cooldown probe admitted")
13208 };
13209 match settlement {
13210 "timeout" => assert!(
13211 !old.record_timeout(1, Duration::ZERO)
13212 .unwrap()
13213 .reopened_after_probe
13214 ),
13215 "inconclusive" => old.record_inconclusive(),
13216 "drop" => drop(old),
13217 _ => unreachable!(),
13218 }
13219 assert!(
13220 matches!(
13221 breakers.admit("prov"),
13222 RouteBindAdmission::Refused {
13223 probe_in_flight: true,
13224 ..
13225 }
13226 ),
13227 "{settlement} of a pre-open relay cannot release the real probe"
13228 );
13229 assert!(
13230 probe
13231 .record_timeout(1, Duration::ZERO)
13232 .unwrap()
13233 .reopened_after_probe
13234 );
13235 assert!(matches!(
13236 breakers.admit("prov"),
13237 RouteBindAdmission::Admitted { probe: true, .. }
13238 ));
13239 }
13240 let breakers = RouteBindBreakers::default();
13241 let admit = || match breakers.admit("prov") {
13242 RouteBindAdmission::Admitted { guard, .. } => guard,
13243 _ => panic!("relay admitted"),
13244 };
13245 admit().record_timeout(1, Duration::ZERO);
13246 let mut old_probe = admit();
13247 breakers.reset_for_new_module_connection("prov");
13248 admit().record_timeout(1, Duration::ZERO);
13249 let _new_probe = admit();
13250 old_probe.record_inconclusive();
13251 assert!(matches!(
13252 breakers.admit("prov"),
13253 RouteBindAdmission::Refused {
13254 probe_in_flight: true,
13255 ..
13256 }
13257 ));
13258 }
13259
13260 #[tokio::test]
13261 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
13262 for candidate in [false, true] {
13263 let registry = Arc::new(Registry::default());
13264 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13265 [("vault".to_string(), true), ("squatter".to_string(), true)],
13266 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13267 );
13268 let conn = ConnectionId::new(77);
13269 let (ctx, mut rx) = route_ctx(conn);
13270 let initial = capability_manifest("squatter", &[], &[]);
13271 if candidate {
13272 registry
13273 .register_candidate_with_control_ops(
13274 initial.clone(),
13275 PROTOCOL_VERSION,
13276 conn,
13277 module_baseline_control_ops(),
13278 )
13279 .unwrap();
13280 handler
13281 .forwarding
13282 .register_candidate_module_connection(
13283 conn,
13284 "squatter".to_string(),
13285 PROTOCOL_VERSION,
13286 manifest_concurrency(&initial),
13287 ctx.egress.clone(),
13288 )
13289 .unwrap();
13290 } else {
13291 hello_via_sink(
13292 &handler,
13293 &ctx,
13294 &mut rx,
13295 hello_frame_with_manifest(initial.clone(), 1),
13296 )
13297 .await;
13298 }
13299 let response = handler
13300 .handle_control_frame(
13301 &ctx,
13302 catalog_update_with_capabilities_frame(
13303 2,
13304 capability_manifest("squatter", &["credentials-provider/v1"], &[])
13305 .capabilities
13306 .unwrap(),
13307 ),
13308 )
13309 .await
13310 .unwrap();
13311 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13312 assert_eq!(
13313 registry
13314 .get_module_by_connection(conn)
13315 .unwrap()
13316 .unwrap()
13317 .manifest,
13318 initial
13319 );
13320 }
13321 }
13322
13323 #[test]
13324 #[cfg(not(feature = "dev-desktop-approval"))]
13325 fn server_describe_default_build_advertises_no_config_capability() {
13326 let assert_default = |handler: &ControlHandler| {
13327 let request = Frame::build(
13328 FrameType::Request,
13329 control_flags(),
13330 0,
13331 0,
13332 79,
13333 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13334 )
13335 .unwrap();
13336 let responses = handler.handle_server_describe(request).unwrap();
13337 assert_eq!(responses.len(), 1);
13338 let response: Value = serde_json::from_slice(&responses[0].body).unwrap();
13339 assert_eq!(response["op"], "server.describe");
13340 let capabilities = response["capabilities"]
13341 .as_array()
13342 .expect("capability array");
13343 assert!(
13344 !capabilities.is_empty(),
13345 "exercise an actual describe response, not an empty fixture"
13346 );
13347 assert!(
13348 capabilities
13349 .iter()
13350 .all(|capability| !capability.as_str().unwrap().starts_with("config.")),
13351 "a default build must not announce configuration authority: {capabilities:?}"
13352 );
13353 };
13354 assert_default(&ControlHandler::new(Arc::new(Registry::default())));
13355 #[cfg(unix)]
13356 {
13357 let (_rig, _owner, _supervisor, handler, _spec) =
13361 crate::config_gate::runtime::RuntimeOwner::unactivated_supervisor_fixture_for_test(
13362 );
13363 assert_default(&handler);
13364 }
13365 }
13366
13367 #[test]
13368 fn server_describe_surfaces_required_capability_verdict_fields() {
13369 let registry = Arc::new(Registry::default());
13370 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13371 [
13372 ("consumer".to_string(), true),
13373 ("provider".to_string(), false),
13374 ],
13375 BTreeMap::new(),
13376 );
13377 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13378 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13379 provides: Vec::new(),
13380 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13381 capability: "credentials-provider/v1".to_string(),
13382 need: subc_protocol::manifest::CapabilityNeed::Required,
13383 }],
13384 must_never_reach: Vec::new(),
13385 });
13386 let hello = Frame::build(
13387 FrameType::Hello,
13388 control_flags(),
13389 0,
13390 0,
13391 78,
13392 serde_json::to_vec(&ModuleHelloBody {
13393 manifest: consumer,
13394 protocol_ver: PROTOCOL_VERSION,
13395 control_ops: None,
13396 launch_nonce: None,
13397 })
13398 .expect("HELLO serializes"),
13399 )
13400 .expect("HELLO frame builds");
13401 handler
13402 .handle_control(ConnectionId::new(78), hello)
13403 .expect("consumer registers");
13404 let describe = Frame::build(
13405 FrameType::Request,
13406 control_flags(),
13407 0,
13408 0,
13409 79,
13410 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
13411 .expect("request serializes"),
13412 )
13413 .expect("describe frame builds");
13414 let response = handler
13415 .handle_server_describe(describe)
13416 .expect("server.describe succeeds");
13417 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
13418 let requirement = &rendered["capability_requirements"][0];
13419 assert_eq!(requirement["consumer"], "consumer");
13420 assert_eq!(requirement["verdict"], "never_provided");
13421 assert_eq!(requirement["episode_seq"], 1);
13422 assert_eq!(requirement["config_satisfiable"], false);
13423 assert_eq!(requirement["runtime_available"], false);
13424 assert!(requirement["detail"]
13425 .as_str()
13426 .expect("detail string")
13427 .contains("credentials-provider/v1"));
13428 }
13429
13430 fn deregister_required_provider(
13436 daemon_shutdown_begun: bool,
13437 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
13438 const CAPABILITY: &str = "project-identity/v1";
13439 let registry = Arc::new(Registry::default());
13440 let daemon_shutdown = DaemonShutdownFlag::default();
13441 let handler = ControlHandler::new(Arc::clone(®istry))
13442 .with_capability_config(
13443 [
13444 ("consumer".to_string(), true),
13445 ("provider".to_string(), true),
13446 ],
13447 BTreeMap::new(),
13448 )
13449 .with_daemon_shutdown_flag(daemon_shutdown.clone());
13450 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13451 consumer.capabilities = Some(CapabilityDeclarations {
13452 provides: Vec::new(),
13453 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13454 capability: CAPABILITY.to_string(),
13455 need: subc_protocol::manifest::CapabilityNeed::Required,
13456 }],
13457 must_never_reach: Vec::new(),
13458 });
13459 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
13460 let provider_conn = ConnectionId::new(182);
13461 for (conn, manifest, corr) in [
13462 (ConnectionId::new(181), consumer, 181),
13463 (provider_conn, provider, 182),
13464 ] {
13465 let hello = handler
13466 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
13467 .expect("module registers");
13468 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13469 }
13470 assert_eq!(
13471 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13472 Some(CapabilityVerdict::Provided),
13473 "the registered provider must satisfy the requirement first"
13474 );
13475 if daemon_shutdown_begun {
13476 daemon_shutdown.set_for_test();
13477 }
13478
13479 let capture = EventCapture::default();
13480 let _subscriber =
13481 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13482 handler
13483 .cleanup_connection(provider_conn)
13484 .expect("provider deregisters");
13485 let events = capture
13486 .events()
13487 .into_iter()
13488 .filter(|event| {
13489 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
13490 })
13491 .collect();
13492 (
13493 events,
13494 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13495 )
13496 }
13497
13498 #[test]
13499 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
13500 let (events, verdict) = deregister_required_provider(true);
13501 assert_eq!(
13502 verdict,
13503 Some(CapabilityVerdict::NeverProvided),
13504 "shutdown must not change the verdict route.open refusals read"
13505 );
13506 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13507 let event = &events[0];
13508 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
13509 assert_eq!(
13510 event.fields.get("cause").map(String::as_str),
13511 Some("\"daemon_shutdown\""),
13512 "{event:?}"
13513 );
13514 assert_eq!(
13515 event.fields.get("verdict").map(String::as_str),
13516 Some("\"never_provided\""),
13517 "{event:?}"
13518 );
13519 }
13520
13521 #[test]
13522 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
13523 let (events, verdict) = deregister_required_provider(false);
13524 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
13525 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13526 let event = &events[0];
13527 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
13528 assert_eq!(event.fields.get("cause"), None, "{event:?}");
13529 assert_eq!(
13530 event.fields.get("verdict").map(String::as_str),
13531 Some("\"never_provided\""),
13532 "{event:?}"
13533 );
13534 }
13535
13536 #[test]
13537 fn catalog_list_omits_capabilities_for_legacy_manifest() {
13538 let registry = Arc::new(Registry::default());
13539 let handler = ControlHandler::new(Arc::clone(®istry));
13540 let hello = handler
13541 .handle_control(
13542 ConnectionId::new(101),
13543 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
13544 )
13545 .expect("legacy HELLO registers");
13546 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13547
13548 let request = Frame::build(
13549 FrameType::Request,
13550 control_flags(),
13551 0,
13552 0,
13553 102,
13554 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
13555 .expect("catalog request serializes"),
13556 )
13557 .expect("catalog request frame builds");
13558 let response = handler
13559 .handle_catalog_list(request, None)
13560 .expect("catalog list succeeds");
13561 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
13562 assert!(
13563 body["modules"][0].get("capabilities").is_none(),
13564 "legacy manifest must retain an absent capabilities field on catalog.list"
13565 );
13566 }
13567
13568 #[test]
13569 fn hello_ack_omits_storage_when_no_storage_config() {
13570 let registry = Arc::new(Registry::default());
13571 let handler = ControlHandler::new(Arc::clone(®istry));
13572 let responses = handler
13573 .handle_control(
13574 ConnectionId::new(1),
13575 hello_frame("aft", PROTOCOL_VERSION, 7),
13576 )
13577 .unwrap();
13578 let ack = parse_ack(&responses[0]);
13579 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
13580 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
13581 }
13582
13583 #[tokio::test]
13584 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
13585 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
13586 let registry = Arc::new(Registry::default());
13587 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
13588 let responses = handler
13589 .handle_control(
13590 ConnectionId::new(1),
13591 hello_frame("aft", PROTOCOL_VERSION, 7),
13592 )
13593 .unwrap();
13594 let ack = parse_ack(&responses[0]);
13595 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
13596
13597 let described = handler
13598 .handle_control_frame(
13599 &route_ctx(ConnectionId::new(2)).0,
13600 Frame::build(
13601 FrameType::Request,
13602 control_flags(),
13603 0,
13604 0,
13605 9,
13606 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13607 )
13608 .unwrap(),
13609 )
13610 .await
13611 .unwrap();
13612 let ClientControlResponse::ServerDescribe { machine_id, .. } =
13613 serde_json::from_slice(&described[0].body).unwrap()
13614 else {
13615 panic!("server.describe answered with another shape");
13616 };
13617 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
13618 }
13619
13620 #[test]
13621 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
13622 let registry = Arc::new(Registry::default());
13625 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
13626 crate::daemon_config::StorageConfig::Sqlite {
13627 data_home: std::path::PathBuf::from("/data"),
13628 },
13629 ));
13630
13631 let responses = handler
13632 .handle_control(
13633 ConnectionId::new(1),
13634 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
13635 )
13636 .unwrap();
13637 let ack = parse_ack(&responses[0]);
13638 assert_eq!(
13639 ack.storage,
13640 Some(serde_json::json!({
13641 "module_id": "alfonso-routing",
13642 "storage_namespace": "default",
13643 "isolation": { "kind": "module" },
13644 "backend": {
13645 "backend": "sqlite",
13646 "path": "/data/cortexkit/alfonso-routing/store.db"
13647 }
13648 })),
13649 "the delivered descriptor is the module's own sqlite store path"
13650 );
13651 }
13652
13653 #[test]
13654 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
13655 let registry = Arc::new(Registry::default());
13656 let handler = ControlHandler::new(Arc::clone(®istry));
13657 let conn = ConnectionId::new(1);
13658 let responses = handler
13659 .handle_control(
13660 conn,
13661 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13662 )
13663 .unwrap();
13664 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
13665 let registration = registry.get_module("aft").unwrap().unwrap();
13666 assert_eq!(registration.control_ops, module_baseline_control_ops());
13667
13668 let frame =
13669 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
13670 assert!(handler
13671 .guard_module_control_op(&frame, "aft", "route.bind")
13672 .unwrap()
13673 .is_none());
13674 let error = handler
13675 .guard_module_control_op(&frame, "aft", "test.synthetic")
13676 .unwrap()
13677 .expect("synthetic ungranted op should be rejected");
13678 assert_eq!(error.header.ty, FrameType::Error);
13679 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
13680 }
13681
13682 #[test]
13683 fn hello_control_ops_some_adds_optional_grants() {
13684 let registry = Arc::new(Registry::default());
13685 let handler = ControlHandler::new(Arc::clone(®istry));
13686 handler
13687 .handle_control(
13688 ConnectionId::new(1),
13689 hello_frame_with_control_ops(
13690 "aft",
13691 PROTOCOL_VERSION,
13692 7,
13693 Some(vec![
13694 "future.synthetic".to_string(),
13695 "route.bind".to_string(),
13696 ]),
13697 ),
13698 )
13699 .unwrap();
13700 let registration = registry.get_module("aft").unwrap().unwrap();
13701 assert_eq!(
13702 registration.control_ops,
13703 vec![
13704 "route.bind".to_string(),
13705 "route.status".to_string(),
13706 "future.synthetic".to_string(),
13707 ]
13708 );
13709 let frame =
13710 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
13711 assert!(handler
13712 .guard_module_control_op(&frame, "aft", "future.synthetic")
13713 .unwrap()
13714 .is_none());
13715 }
13716
13717 #[tokio::test]
13718 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
13719 let registry = Arc::new(Registry::default());
13720 let forwarding = Arc::new(ForwardingTable::default());
13721 let handler =
13722 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13723 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
13724 hello_via_sink(
13725 &handler,
13726 &module_ctx,
13727 &mut module_rx,
13728 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13729 )
13730 .await;
13731
13732 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
13733 let responses = handler
13734 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
13735 .await
13736 .unwrap();
13737 assert_eq!(responses.len(), 1);
13738 assert_eq!(responses[0].header.ty, FrameType::Error);
13739 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
13740 assert!(module_rx.try_recv().is_err());
13741 }
13742
13743 #[tokio::test]
13744 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
13745 let registry = Arc::new(Registry::default());
13746 let forwarding = Arc::new(ForwardingTable::default());
13747 let handler =
13748 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13749 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
13750 hello_via_sink(
13751 &handler,
13752 &module_ctx,
13753 &mut module_rx,
13754 hello_frame_with_control_ops(
13755 "aft",
13756 PROTOCOL_VERSION,
13757 7,
13758 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13759 ),
13760 )
13761 .await;
13762
13763 let project_root = unique_project_root("demux");
13764 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
13765 let route_handler = handler.clone();
13766 let route_task = tokio::spawn(async move {
13767 route_handler
13768 .handle_control_frame(
13769 &route_client_ctx,
13770 route_open_frame(100, "aft", project_root),
13771 )
13772 .await
13773 .unwrap()
13774 });
13775 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13776 .await
13777 .unwrap()
13778 .unwrap();
13779 assert!(matches!(
13780 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
13781 ModuleControlRequest::RouteBind { .. }
13782 ));
13783
13784 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
13785 let health_handler = handler.clone();
13786 let health_task = tokio::spawn(async move {
13787 health_handler
13788 .handle_control_frame(
13789 &health_client_ctx,
13790 supervisor_health_probe_frame(101, "aft"),
13791 )
13792 .await
13793 .unwrap()
13794 });
13795 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13796 .await
13797 .unwrap()
13798 .unwrap();
13799 assert_eq!(
13800 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13801 ModuleControlRequest::HealthCheck {}
13802 );
13803
13804 handler
13805 .handle_control_frame(
13806 &module_ctx,
13807 health_response(health_frame.header.corr, HealthStatus::Degraded),
13808 )
13809 .await
13810 .unwrap();
13811 let health_response = health_task.await.unwrap();
13812 assert_eq!(health_response.len(), 1);
13813 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13814 ClientControlResponse::SupervisorHealthProbe {
13815 module_id,
13816 status,
13817 detail,
13818 metrics,
13819 } => {
13820 assert_eq!(module_id, "aft");
13821 assert_eq!(status, HealthStatus::Degraded);
13822 assert_eq!(detail.as_deref(), Some("warming"));
13823 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
13824 }
13825 other => panic!("unexpected health response: {other:?}"),
13826 }
13827
13828 handler
13829 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13830 .await
13831 .unwrap();
13832 let route_response = route_task.await.unwrap();
13833 assert!(route_response.is_empty());
13834 let published = route_client_rx.recv().await.unwrap();
13835 assert!(matches!(
13836 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13837 ClientControlResponse::RouteOpen { .. }
13838 ));
13839 }
13840
13841 async fn relay_route_open(
13846 handler: &ControlHandler,
13847 client_connection: ConnectionId,
13848 client_egress: &FrameSink,
13849 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13850 corr: u64,
13851 module_id: &str,
13852 project_root_label: &str,
13853 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
13854 let ctx = RouteCtx {
13855 connection_id: client_connection,
13856 egress: client_egress.clone(),
13857 };
13858 let handler = handler.clone();
13859 let project_root = unique_project_root(project_root_label);
13860 let module_id = module_id.to_string();
13861 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
13862 let task = tokio::spawn(async move {
13863 let _guard = tracing::dispatcher::set_default(&dispatch);
13864 handler
13865 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
13866 .await
13867 .unwrap()
13868 });
13869 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13870 .await
13871 .expect("module receives the relayed route.bind")
13872 .expect("module egress is open");
13873 (task, bind.frame)
13874 }
13875
13876 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
13877 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
13878 ModuleControlRequest::RouteBind {
13879 route_channel,
13880 epoch,
13881 ..
13882 } => (route_channel, epoch),
13883 other => panic!("expected a route.bind request, got {other:?}"),
13884 }
13885 }
13886
13887 fn published_route(frame: &Frame) -> (u16, u32) {
13888 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
13889 ClientControlResponse::RouteOpen {
13890 route_channel,
13891 route_epoch,
13892 ..
13893 } => (route_channel, route_epoch),
13894 other => panic!("expected a route.open response, got {other:?}"),
13895 }
13896 }
13897
13898 #[tokio::test]
13918 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
13919 let registry = Arc::new(Registry::default());
13920 let forwarding = Arc::new(ForwardingTable::default());
13921 let handler =
13922 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13923
13924 let module_connection = ConnectionId::new(30);
13925 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13926 hello_via_sink(
13927 &handler,
13928 &module_ctx,
13929 &mut module_rx,
13930 hello_frame("aft", PROTOCOL_VERSION, 7),
13931 )
13932 .await;
13933
13934 let dying_client = ConnectionId::new(31);
13935 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
13936
13937 let (first_task, first_bind) = relay_route_open(
13940 &handler,
13941 dying_client,
13942 &dying_ctx.egress,
13943 &mut module_rx,
13944 100,
13945 "aft",
13946 "closing-first",
13947 )
13948 .await;
13949 handler
13950 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
13951 .await
13952 .unwrap();
13953 assert!(first_task.await.unwrap().is_empty());
13954 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
13955
13956 let (second_task, second_bind) = relay_route_open(
13958 &handler,
13959 dying_client,
13960 &dying_ctx.egress,
13961 &mut module_rx,
13962 101,
13963 "aft",
13964 "closing-second",
13965 )
13966 .await;
13967 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
13968
13969 assert!(forwarding
13972 .escalate_client_delivery_failure(
13973 dying_client,
13974 first_channel,
13975 first_epoch,
13976 CloseReason::new(
13977 "module_to_client_delivery_failed",
13978 "client egress refused a module frame",
13979 ),
13980 crate::forwarding::UndeliveredFrame {
13981 module_id: None,
13982 sink: &dying_ctx.egress,
13983 },
13984 )
13985 .unwrap());
13986 assert!(!dying_ctx.egress.is_closed());
13987
13988 let ack = handler
13990 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
13991 .await;
13992 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13993 if module_loop_error.is_some() {
13994 handler.cleanup_connection(module_connection).unwrap();
13998 }
13999 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14003 .await
14004 .ok()
14005 .flatten();
14006
14007 assert!(
14009 registry
14010 .get_module_by_connection(module_connection)
14011 .unwrap()
14012 .is_some(),
14013 "one client's closing connection ended the shared module connection: \
14014 {module_loop_error:?}"
14015 );
14016 let cotenant = ConnectionId::new(32);
14018 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14019 let (cotenant_task, cotenant_bind) = relay_route_open(
14020 &handler,
14021 cotenant,
14022 &cotenant_ctx.egress,
14023 &mut module_rx,
14024 102,
14025 "aft",
14026 "closing-cotenant",
14027 )
14028 .await;
14029 handler
14030 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
14031 .await
14032 .unwrap();
14033 assert!(cotenant_task.await.unwrap().is_empty());
14034 let (cotenant_channel, cotenant_epoch) =
14035 published_route(&cotenant_rx.recv().await.unwrap());
14036 assert!(matches!(
14037 forwarding
14038 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14039 .unwrap(),
14040 DataRoute::Client(DataRouteState::Bound(_))
14041 ));
14042
14043 let goodbye = post_ack_module_frame
14046 .expect("module receives a GOODBYE for the abandoned route channel");
14047 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14048 assert_eq!(goodbye.header.channel, abandoned_channel);
14049 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14050
14051 assert!(dying_rx.try_recv().is_err());
14055 let second_response = second_task.await.unwrap();
14056 assert_eq!(second_response.len(), 1);
14057 assert_eq!(
14058 parse_error(&second_response[0])["code"],
14059 "target_unavailable"
14060 );
14061 }
14062
14063 async fn catalog_snapshot(
14066 handler: &ControlHandler,
14067 client_ctx: &RouteCtx,
14068 corr: u64,
14069 ) -> (u64, Vec<(String, bool)>) {
14070 let body =
14071 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
14072 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
14073 let reply = handler
14074 .handle_control_frame(client_ctx, request)
14075 .await
14076 .unwrap()
14077 .pop()
14078 .unwrap();
14079 let ClientControlResponse::CatalogList {
14080 generation,
14081 modules,
14082 ..
14083 } = serde_json::from_slice(&reply.body).unwrap()
14084 else {
14085 panic!("not a catalog.list reply");
14086 };
14087 (
14088 generation,
14089 modules
14090 .into_iter()
14091 .map(|entry| (entry.module_id, entry.ready))
14092 .collect(),
14093 )
14094 }
14095
14096 fn hello_held_before_forwarding() -> (
14102 ControlHandler,
14103 Arc<std::sync::Barrier>,
14104 std::thread::JoinHandle<Vec<Frame>>,
14105 RouteCtx,
14106 mpsc::Receiver<crate::router::OutboundFrame>,
14107 ) {
14108 let hold = Arc::new(std::sync::Barrier::new(2));
14109 let handler = ControlHandler::with_forwarding(
14110 Arc::new(Registry::default()),
14111 Arc::new(ForwardingTable::default()),
14112 )
14113 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14114 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14115 let hello = {
14116 let handler = handler.clone();
14117 let module_ctx = module_ctx.clone();
14118 std::thread::spawn(move || {
14119 handler
14120 .handle_hello(
14121 module_ctx.connection_id,
14122 Some(module_ctx.egress.clone()),
14123 hello_frame("aft", PROTOCOL_VERSION, 7),
14124 )
14125 .unwrap()
14126 })
14127 };
14128 (handler, hold, hello, module_ctx, module_rx)
14129 }
14130
14131 fn lookup_during_hello_publication<T: Send + 'static>(
14137 registry: &Registry,
14138 hold: &std::sync::Barrier,
14139 lookup: impl FnOnce() -> T + Send + 'static,
14140 ) -> T {
14141 let attempted = registry.observe_next_lock_attempt();
14142 let reader = std::thread::spawn(lookup);
14143 if attempted.recv().unwrap() {
14144 let result = reader.join().unwrap();
14145 hold.wait();
14146 result
14147 } else {
14148 hold.wait();
14149 reader.join().unwrap()
14150 }
14151 }
14152
14153 #[test]
14154 fn catalog_read_after_hello_ack_waits_for_publication() {
14155 let registry = Arc::new(Registry::default());
14156 let hold = Arc::new(std::sync::Barrier::new(2));
14157 let mut handler = ControlHandler::with_forwarding(
14158 Arc::clone(®istry),
14159 Arc::new(ForwardingTable::default()),
14160 );
14161 handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
14162 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14163 let hello = {
14164 let handler = handler.clone();
14165 std::thread::spawn(move || {
14166 handler
14167 .handle_hello(
14168 module_ctx.connection_id,
14169 Some(module_ctx.egress),
14170 hello_frame("aft", PROTOCOL_VERSION, 7),
14171 )
14172 .unwrap()
14173 })
14174 };
14175 hold.wait();
14176 assert_eq!(
14177 module_rx.try_recv().unwrap().frame.header.ty,
14178 FrameType::HelloAck
14179 );
14180 let replies = lookup_during_hello_publication(®istry, &hold, move || {
14181 let request = Frame::build(
14184 FrameType::Request,
14185 control_flags(),
14186 0,
14187 0,
14188 300,
14189 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
14190 )
14191 .unwrap();
14192 handler.handle_catalog_list(request, None).unwrap()
14193 });
14194 assert!(hello.join().unwrap().is_empty());
14195 let ClientControlResponse::CatalogList {
14196 generation,
14197 modules,
14198 ..
14199 } = serde_json::from_slice(&replies[0].body).unwrap()
14200 else {
14201 panic!("not a catalog.list reply");
14202 };
14203 assert_eq!(
14204 modules
14205 .iter()
14206 .map(|entry| entry.module_id.as_str())
14207 .collect::<Vec<_>>(),
14208 vec!["aft"]
14209 );
14210 assert_eq!(generation, 1);
14211 }
14212
14213 #[tokio::test]
14222 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
14223 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
14224 let client = ConnectionId::new(41);
14225 let (client_ctx, mut client_rx) = route_ctx(client);
14226
14227 hold.wait();
14228 assert!(handler.registry.get_module("aft").unwrap().is_some());
14230 assert!(!handler
14231 .forwarding
14232 .has_live_module_connection("aft")
14233 .unwrap());
14234 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
14235 assert!(
14236 listed.is_empty(),
14237 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
14238 );
14239 let refused = handler
14240 .handle_control_frame(
14241 &client_ctx,
14242 route_open_frame(301, "aft", unique_project_root("listed-window")),
14243 )
14244 .await
14245 .unwrap();
14246 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
14247 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
14248 let duplicate = handler
14249 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
14250 .await
14251 .unwrap();
14252 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14253
14254 hold.wait();
14255 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14256 let ack = module_rx.try_recv().unwrap().frame;
14257 assert_eq!(ack.header.ty, FrameType::HelloAck);
14258 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14259 assert_eq!(listed, vec![("aft".to_string(), true)]);
14260
14261 let (open_task, bind) = relay_route_open(
14262 &handler,
14263 client,
14264 &client_ctx.egress,
14265 &mut module_rx,
14266 303,
14267 "aft",
14268 "listed-after",
14269 )
14270 .await;
14271 assert!(handler
14272 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14273 .await
14274 .unwrap()
14275 .is_empty());
14276 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
14277 published_route(&client_rx.try_recv().unwrap().frame);
14278 }
14279
14280 #[tokio::test]
14287 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
14288 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
14289 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14290 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14291
14292 hold.wait();
14293 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
14294 assert!(listed.is_empty());
14295 assert_eq!(
14296 during, before,
14297 "the generation moved while the module was not listed"
14298 );
14299
14300 hold.wait();
14301 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14302 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14303 assert_eq!(listed, vec![("aft".to_string(), true)]);
14304 assert_eq!(
14305 after,
14306 before + 1,
14307 "the module appeared without exactly one generation bump"
14308 );
14309 }
14310
14311 #[tokio::test]
14316 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
14317 let registry = Arc::new(Registry::default());
14318 let handler = ControlHandler::with_forwarding(
14319 Arc::clone(®istry),
14320 Arc::new(ForwardingTable::default()),
14321 );
14322 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14323 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14324 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14325 drop(module_rx);
14326
14327 let replies = handler
14328 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
14329 .await
14330 .unwrap();
14331 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
14332 assert!(registry.get_module("aft").unwrap().is_none());
14333 assert!(registry
14334 .get_module_by_connection(module_ctx.connection_id)
14335 .unwrap()
14336 .is_none());
14337 assert_eq!(
14338 catalog_snapshot(&handler, &client_ctx, 301).await,
14339 (before, Vec::new())
14340 );
14341 }
14342
14343 async fn route_open_held_after_bind_send() -> (
14347 ControlHandler,
14348 Arc<tokio::sync::Barrier>,
14349 RouteCtx,
14350 mpsc::Receiver<crate::router::OutboundFrame>,
14351 ) {
14352 let hold = Arc::new(tokio::sync::Barrier::new(2));
14353 let handler = ControlHandler::with_forwarding(
14354 Arc::new(Registry::default()),
14355 Arc::new(ForwardingTable::default()),
14356 )
14357 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
14358 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14359 hello_via_sink(
14360 &handler,
14361 &module_ctx,
14362 &mut module_rx,
14363 hello_frame("aft", PROTOCOL_VERSION, 7),
14364 )
14365 .await;
14366 (handler, hold, module_ctx, module_rx)
14367 }
14368
14369 #[tokio::test]
14376 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
14377 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14378 let client = ConnectionId::new(41);
14379 let (client_ctx, mut client_rx) = route_ctx(client);
14380
14381 let (open_task, bind) = relay_route_open(
14382 &handler,
14383 client,
14384 &client_ctx.egress,
14385 &mut module_rx,
14386 200,
14387 "aft",
14388 "acked-before-mark",
14389 )
14390 .await;
14391 let (module_channel, module_epoch) = route_bind_channel(&bind);
14392 hold.wait().await;
14394
14395 assert!(handler
14397 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14398 .await
14399 .unwrap()
14400 .is_empty());
14401 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
14402
14403 hold.wait().await;
14404 assert!(
14405 open_task.await.unwrap().is_empty(),
14406 "route.open answers accepted"
14407 );
14408
14409 match module_rx.try_recv() {
14412 Err(mpsc::error::TryRecvError::Empty) => {}
14413 Ok(outbound) => panic!(
14414 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
14415 outbound.frame.header.ty,
14416 outbound.frame.header.channel,
14417 outbound.frame.header.epoch
14418 ),
14419 Err(err) => panic!("module egress closed: {err}"),
14420 }
14421
14422 let router = Router::with_control_handler(Arc::new(handler.clone()));
14425 let request = Frame::build(
14426 FrameType::Request,
14427 Flags::new(true, Priority::Interactive, false),
14428 client_channel,
14429 client_epoch,
14430 201,
14431 b"first request".to_vec(),
14432 )
14433 .unwrap();
14434 router
14435 .route_for_connection(&client_ctx, request)
14436 .await
14437 .unwrap();
14438 let delivered = module_rx.try_recv().unwrap().frame;
14439 assert_eq!(delivered.header.ty, FrameType::Request);
14440 assert_eq!(
14441 (delivered.header.channel, delivered.header.epoch),
14442 (module_channel, module_epoch)
14443 );
14444 assert_eq!(delivered.body, b"first request");
14445 }
14446
14447 #[tokio::test]
14453 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
14454 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14455 let client = ConnectionId::new(42);
14456 let (client_ctx, _client_rx) = route_ctx(client);
14457
14458 let (open_task, bind) = relay_route_open(
14459 &handler,
14460 client,
14461 &client_ctx.egress,
14462 &mut module_rx,
14463 300,
14464 "aft",
14465 "released-before-mark",
14466 )
14467 .await;
14468 let (module_channel, module_epoch) = route_bind_channel(&bind);
14469 hold.wait().await;
14470
14471 handler.cleanup_connection(client).unwrap();
14472 assert!(
14473 module_rx.try_recv().is_err(),
14474 "client teardown must not GOODBYE a bind route.open is still queueing"
14475 );
14476
14477 hold.wait().await;
14478 let response = open_task.await.unwrap();
14479 assert_eq!(response.len(), 1);
14480 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14481
14482 let goodbye = module_rx
14483 .try_recv()
14484 .expect("module is told to drop the binding the daemon abandoned")
14485 .frame;
14486 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14487 assert_eq!(
14488 (goodbye.header.channel, goodbye.header.epoch),
14489 (module_channel, module_epoch)
14490 );
14491 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14492 }
14493
14494 #[tokio::test]
14499 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
14500 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14501 let client = ConnectionId::new(43);
14502 let (client_ctx, client_rx) = route_ctx(client);
14503
14504 let (open_task, bind) = relay_route_open(
14505 &handler,
14506 client,
14507 &client_ctx.egress,
14508 &mut module_rx,
14509 400,
14510 "aft",
14511 "gone-client-before-mark",
14512 )
14513 .await;
14514 let (module_channel, module_epoch) = route_bind_channel(&bind);
14515 hold.wait().await;
14516
14517 drop(client_rx);
14518 assert!(client_ctx.egress.is_closed());
14519 handler
14520 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14521 .await
14522 .unwrap();
14523
14524 hold.wait().await;
14525 let response = open_task.await.unwrap();
14526 assert_eq!(response.len(), 1);
14527 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14528
14529 let goodbye = module_rx
14530 .try_recv()
14531 .expect("module is told to drop the binding it acked")
14532 .frame;
14533 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14534 assert_eq!(
14535 (goodbye.header.channel, goodbye.header.epoch),
14536 (module_channel, module_epoch)
14537 );
14538 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14539 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14540 }
14541
14542 #[test]
14549 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
14550 let handler = ControlHandler::default();
14551 let module_connection = ConnectionId::new(30);
14552 let client_connection = ConnectionId::new(31);
14553
14554 handler
14555 .refuse_to_end_module_connection_for_a_client(
14556 module_connection,
14557 77,
14558 ForwardingError::ConnectionClosing {
14559 connection_id: client_connection,
14560 },
14561 )
14562 .expect("a closing client must never end the module connection");
14563
14564 assert!(matches!(
14565 handler.refuse_to_end_module_connection_for_a_client(
14566 module_connection,
14567 78,
14568 ForwardingError::ConnectionClosing {
14569 connection_id: module_connection,
14570 },
14571 ),
14572 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
14573 connection_id
14574 })) if connection_id == module_connection
14575 ));
14576 assert!(matches!(
14577 handler.refuse_to_end_module_connection_for_a_client(
14578 module_connection,
14579 79,
14580 ForwardingError::Poisoned,
14581 ),
14582 Err(RouterError::Forwarding(ForwardingError::Poisoned))
14583 ));
14584 assert!(matches!(
14585 handler.refuse_to_end_module_connection_for_a_client(
14586 module_connection,
14587 80,
14588 ForwardingError::StaleModuleEndpoint,
14589 ),
14590 Err(RouterError::Forwarding(
14591 ForwardingError::StaleModuleEndpoint
14592 ))
14593 ));
14594 }
14595
14596 #[tokio::test]
14609 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
14610 let registry = Arc::new(Registry::default());
14611 let forwarding = Arc::new(ForwardingTable::default());
14612 let supervisor = SupervisorHandle::new();
14613 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14614 let handler =
14615 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14616 .with_supervisor(supervisor);
14617
14618 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14619 hello_via_sink(
14620 &handler,
14621 &target_ctx,
14622 &mut target_rx,
14623 hello_frame("target", PROTOCOL_VERSION, 1),
14624 )
14625 .await;
14626
14627 let wrong_nonce = handler
14631 .handle_control_frame(
14632 &route_ctx(ConnectionId::new(91)).0,
14633 route_open_frame_with_admission_facts(
14634 20,
14635 "target",
14636 unique_project_root("admission-facts"),
14637 Some(subc_control::ConsumerIdentity {
14638 module_id: "fed".to_string(),
14639 launch_nonce: "not-the-real-nonce".to_string(),
14640 }),
14641 None,
14642 ),
14643 )
14644 .await
14645 .unwrap();
14646 assert_eq!(
14647 parse_error(&wrong_nonce[0])["code"],
14648 "bad_consumer_identity",
14649 "a mismatched launch nonce must be refused, not stamped Reserved"
14650 );
14651
14652 let never_spawned = handler
14656 .handle_control_frame(
14657 &route_ctx(ConnectionId::new(92)).0,
14658 route_open_frame_with_admission_facts(
14659 21,
14660 "target",
14661 unique_project_root("admission-facts"),
14662 Some(subc_control::ConsumerIdentity {
14663 module_id: "never-spawned".to_string(),
14664 launch_nonce: "any-nonce".to_string(),
14665 }),
14666 None,
14667 ),
14668 )
14669 .await
14670 .unwrap();
14671 assert_eq!(
14672 parse_error(&never_spawned[0])["code"],
14673 "bad_consumer_identity",
14674 "an unspawned module_id must be refused rather than accepted for lack of a record"
14675 );
14676 }
14677
14678 #[tokio::test]
14701 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
14702 let registry = Arc::new(Registry::default());
14703 let forwarding = Arc::new(ForwardingTable::default());
14704 let supervisor = SupervisorHandle::new();
14705 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14706 let handler =
14707 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14708 .with_supervisor(supervisor);
14709
14710 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
14711 hello_via_sink(
14712 &handler,
14713 &target_ctx,
14714 &mut target_rx,
14715 hello_frame("target", PROTOCOL_VERSION, 1),
14716 )
14717 .await;
14718
14719 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
14720 let route_handler = handler.clone();
14721 let route_task = tokio::spawn(async move {
14722 route_handler
14723 .handle_control_frame(
14724 &client_ctx,
14725 route_open_frame_with_admission_facts(
14726 30,
14727 "target",
14728 unique_project_root("admission-facts"),
14729 Some(subc_control::ConsumerIdentity {
14730 module_id: "fed".to_string(),
14731 launch_nonce: "fed-nonce".to_string(),
14732 }),
14733 None,
14734 ),
14735 )
14736 .await
14737 .unwrap()
14738 });
14739
14740 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14747 .await
14748 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
14749 .expect("module control channel closed before route.bind");
14750 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14751 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14752 panic!("expected route.bind")
14753 };
14754 assert_eq!(
14755 principal,
14756 Some(Principal::Reserved {
14757 module_id: "fed".to_string()
14758 }),
14759 "a correctly attested consumer must be stamped Reserved for its own id"
14760 );
14761
14762 handler
14763 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14764 .await
14765 .unwrap();
14766 assert!(route_task.await.unwrap().is_empty());
14767 assert!(
14768 matches!(
14769 serde_json::from_slice::<ClientControlResponse>(
14770 &client_rx.recv().await.unwrap().body
14771 )
14772 .unwrap(),
14773 ClientControlResponse::RouteOpen { .. }
14774 ),
14775 "the route must actually open, not merely avoid an error"
14776 );
14777 }
14778
14779 #[tokio::test(start_paused = true)]
14780 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
14781 let registry = Arc::new(Registry::default());
14782 let forwarding = Arc::new(ForwardingTable::default());
14783 let supervisor = SupervisorHandle::new();
14784 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14785 let handler =
14786 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14787 .with_supervisor(supervisor);
14788
14789 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
14790 hello_via_sink(
14791 &handler,
14792 &target_ctx,
14793 &mut target_rx,
14794 hello_frame("target", PROTOCOL_VERSION, 1),
14795 )
14796 .await;
14797
14798 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
14799 let direct_handler = handler.clone();
14800 let direct_open = tokio::spawn(async move {
14801 direct_handler
14802 .handle_control_frame(
14803 &direct_ctx,
14804 route_open_frame(2, "target", unique_project_root("route-census-direct")),
14805 )
14806 .await
14807 .unwrap()
14808 });
14809 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14810 .await
14811 .expect("no direct route.bind within 5s")
14812 .expect("target control channel closed before direct route.bind");
14813 handler
14814 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
14815 .await
14816 .unwrap();
14817 assert!(direct_open.await.unwrap().is_empty());
14818 let _ = direct_rx.recv().await.unwrap();
14819
14820 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
14821 let reserved_handler = handler.clone();
14822 let reserved_open = tokio::spawn(async move {
14823 reserved_handler
14824 .handle_control_frame(
14825 &reserved_ctx,
14826 route_open_frame_with_admission_facts(
14827 3,
14828 "target",
14829 unique_project_root("admission-facts"),
14830 Some(ConsumerIdentity {
14831 module_id: "fed".to_string(),
14832 launch_nonce: "fed-nonce".to_string(),
14833 }),
14834 None,
14835 ),
14836 )
14837 .await
14838 .unwrap()
14839 });
14840 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14841 .await
14842 .expect("no reserved route.bind within 5s")
14843 .expect("target control channel closed before reserved route.bind");
14844 handler
14845 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
14846 .await
14847 .unwrap();
14848 assert!(reserved_open.await.unwrap().is_empty());
14849 let _ = reserved_rx.recv().await.unwrap();
14850
14851 forwarding
14852 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
14853 .unwrap();
14854 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
14855 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
14856 module_id: Some("target".to_string()),
14857 })
14858 .unwrap();
14859 let census_frame =
14860 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
14861 let response = handler
14862 .handle_control_frame(&census_ctx, census_frame)
14863 .await
14864 .unwrap()
14865 .pop()
14866 .unwrap();
14867 let actual: Value = serde_json::from_slice(&response.body).unwrap();
14868 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
14869 assert!(matches!(
14870 decoded,
14871 ClientControlResponse::SupervisorRoutes { .. }
14872 ));
14873 let routes = actual["modules"][0]["routes"].as_array().unwrap();
14874 assert_eq!(routes.len(), 2);
14875 assert!(routes.iter().all(|route| route["draining"] == true));
14876 assert!(
14879 routes.iter().all(|route| route["drain_reason"] == "reload"),
14880 "draining routes must name the drain's reason: {routes:?}"
14881 );
14882 assert!(routes.iter().any(|route| {
14883 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
14884 }));
14885 assert!(routes.iter().any(|route| {
14886 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
14887 }));
14888
14889 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14890 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
14891 if std::env::var_os("UPDATE_GOLDEN").is_some() {
14892 std::fs::write(
14893 &golden_path,
14894 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
14895 )
14896 .unwrap();
14897 }
14898 let expected: Value =
14899 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
14900 assert_eq!(actual, expected);
14901 }
14902
14903 async fn query_live_roots(
14904 handler: &ControlHandler,
14905 module_ctx: &RouteCtx,
14906 ) -> ModuleControlResponseToModule {
14907 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
14908 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
14909 let response = handler
14910 .handle_control_frame(module_ctx, frame)
14911 .await
14912 .unwrap()
14913 .pop()
14914 .unwrap();
14915 serde_json::from_slice(&response.body).unwrap()
14916 }
14917
14918 #[tokio::test(start_paused = true)]
14919 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
14920 let registry = Arc::new(Registry::default());
14921 let forwarding = Arc::new(ForwardingTable::default());
14922 let handler = ControlHandler::with_forwarding(registry, forwarding);
14923 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
14924 hello_via_sink(
14925 &handler,
14926 &target_ctx,
14927 &mut target_rx,
14928 hello_frame("target", PROTOCOL_VERSION, 1),
14929 )
14930 .await;
14931 let root = unique_project_root("live-roots-known");
14932 let path = ProjectRootId::from_path_allowing_missing(root.path())
14933 .unwrap()
14934 .as_path()
14935 .to_path_buf();
14936 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
14937 let open_handler = handler.clone();
14938 let opened = tokio::spawn(async move {
14939 open_handler
14940 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
14941 .await
14942 .unwrap()
14943 });
14944 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14945 .await
14946 .unwrap()
14947 .unwrap();
14948 handler
14949 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
14950 .await
14951 .unwrap();
14952 assert!(opened.await.unwrap().is_empty());
14953 let _ = client_rx.recv().await.unwrap();
14954
14955 let root = unique_project_root("live-roots-pending");
14956 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
14957 .unwrap()
14958 .as_path()
14959 .to_path_buf();
14960 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
14961 let open_handler = handler.clone();
14962 let pending = tokio::spawn(async move {
14963 open_handler
14964 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
14965 .await
14966 .unwrap()
14967 });
14968 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14969 .await
14970 .unwrap()
14971 .unwrap();
14972 let actual = query_live_roots(&handler, &target_ctx).await;
14973 let ModuleControlResponseToModule::LiveRoots {
14974 roots,
14975 unknown_root_bindings,
14976 total_bindings,
14977 } = actual
14978 else {
14979 panic!("expected live roots")
14980 };
14981 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
14982 assert_eq!(unknown_root_bindings, 0);
14983 assert_eq!(
14984 roots.len(),
14985 2,
14986 "root-known arm must retain each canonical root"
14987 );
14988 assert_eq!(
14989 total_bindings,
14990 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14991 );
14992 let counts = roots
14993 .iter()
14994 .map(|root| (root.project_root.clone(), root.bound, root.pending))
14995 .collect::<Vec<_>>();
14996 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
14997 expected.sort_by(|a, b| a.0.cmp(&b.0));
14998 assert_eq!(
14999 counts, expected,
15000 "roots must sort by path and count pending separately"
15001 );
15002 handler
15003 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
15004 .await
15005 .unwrap();
15006 assert!(pending.await.unwrap().is_empty());
15007 }
15008
15009 #[tokio::test(start_paused = true)]
15010 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
15011 let forwarding = Arc::new(ForwardingTable::default());
15012 let handler =
15013 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
15014 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
15015 hello_via_sink(
15016 &handler,
15017 &target_ctx,
15018 &mut target_rx,
15019 hello_frame("target", PROTOCOL_VERSION, 1),
15020 )
15021 .await;
15022 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
15023 let pending = forwarding
15024 .begin_route_bind_relay_for_test(
15025 client_ctx.connection_id,
15026 client_ctx.egress.clone(),
15027 2,
15028 "target",
15029 )
15030 .unwrap();
15031 forwarding
15032 .complete_pending_relay(
15033 target_ctx.connection_id,
15034 pending.corr,
15035 RouteBindRelayOutcome::Accepted,
15036 )
15037 .unwrap();
15038 let actual = query_live_roots(&handler, &target_ctx).await;
15039 let ModuleControlResponseToModule::LiveRoots {
15040 roots,
15041 unknown_root_bindings,
15042 total_bindings,
15043 } = actual
15044 else {
15045 panic!("expected live roots")
15046 };
15047 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
15048 assert_eq!(
15049 unknown_root_bindings, 1,
15050 "unknown-root arm must not read as no bindings"
15051 );
15052 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
15053 assert_eq!(
15054 total_bindings,
15055 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15056 );
15057 }
15058
15059 #[tokio::test(start_paused = true)]
15067 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
15068 let forwarding = Arc::new(ForwardingTable::default());
15069 let handler =
15070 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
15071 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
15072 let replies = handler
15073 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
15074 .await
15075 .unwrap();
15076 let queued_by_hello = module_rx.len();
15077
15078 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
15079 let open_handler = handler.clone();
15080 let open = tokio::spawn(async move {
15081 open_handler
15082 .handle_control_frame(
15083 &client_ctx,
15084 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
15085 )
15086 .await
15087 .unwrap()
15088 });
15089 let mut spins = 0;
15091 while module_rx.len() == queued_by_hello {
15092 spins += 1;
15093 assert!(spins < 10_000, "route.open never queued a route.bind");
15094 tokio::task::yield_now().await;
15095 }
15096
15097 for reply in replies {
15099 module_ctx.egress.send(reply).await.unwrap();
15100 }
15101
15102 let first = module_rx.recv().await.unwrap().frame;
15103 assert_eq!(
15104 first.header.ty,
15105 FrameType::HelloAck,
15106 "the first frame a registering module reads must be its HELLO_ACK"
15107 );
15108 assert_eq!(first.header.corr, 7);
15109 let second = module_rx.recv().await.unwrap().frame;
15110 assert_eq!(second.header.ty, FrameType::Request);
15111 assert!(
15112 matches!(
15113 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
15114 ModuleControlRequest::RouteBind { .. }
15115 ),
15116 "the route.bind follows the ack"
15117 );
15118 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
15119
15120 handler
15121 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
15122 .await
15123 .unwrap();
15124 assert!(open.await.unwrap().is_empty());
15125 let _ = client_rx.recv().await.unwrap();
15126 }
15127
15128 #[tokio::test(start_paused = true)]
15129 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
15130 let handler = ControlHandler::with_forwarding(
15131 Arc::new(Registry::default()),
15132 Arc::new(ForwardingTable::default()),
15133 );
15134 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
15135 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
15136 hello_via_sink(
15137 &handler,
15138 &first_ctx,
15139 &mut first_rx,
15140 hello_frame("first", PROTOCOL_VERSION, 1),
15141 )
15142 .await;
15143 hello_via_sink(
15144 &handler,
15145 &second_ctx,
15146 &mut second_rx,
15147 hello_frame("second", PROTOCOL_VERSION, 2),
15148 )
15149 .await;
15150 let root = unique_project_root("second-only");
15151 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
15152 let cloned = handler.clone();
15153 let open = tokio::spawn(async move {
15154 cloned
15155 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
15156 .await
15157 .unwrap()
15158 });
15159 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
15160 .await
15161 .unwrap()
15162 .unwrap();
15163 let first = query_live_roots(&handler, &first_ctx).await;
15164 let second = query_live_roots(&handler, &second_ctx).await;
15165 assert!(
15166 matches!(
15167 first,
15168 ModuleControlResponseToModule::LiveRoots {
15169 total_bindings: 0,
15170 ..
15171 }
15172 ),
15173 "cross-module scope must not expose another module's roots"
15174 );
15175 assert!(
15176 matches!(
15177 second,
15178 ModuleControlResponseToModule::LiveRoots {
15179 total_bindings: 1,
15180 ..
15181 }
15182 ),
15183 "second module must see its pending route"
15184 );
15185 handler
15186 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
15187 .await
15188 .unwrap();
15189 assert!(open.await.unwrap().is_empty());
15190 }
15191
15192 #[tokio::test(start_paused = true)]
15193 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
15194 let handler = ControlHandler::with_forwarding(
15195 Arc::new(Registry::default()),
15196 Arc::new(ForwardingTable::default()),
15197 );
15198 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
15199 hello_via_sink(
15200 &handler,
15201 &target_ctx,
15202 &mut target_rx,
15203 hello_frame("target", PROTOCOL_VERSION, 1),
15204 )
15205 .await;
15206 let actual = query_live_roots(&handler, &target_ctx).await;
15207 let ModuleControlResponseToModule::LiveRoots {
15208 roots,
15209 unknown_root_bindings,
15210 total_bindings,
15211 } = actual
15212 else {
15213 panic!("expected live roots")
15214 };
15215 assert!(roots.is_empty());
15216 assert_eq!(unknown_root_bindings, 0);
15217 assert_eq!(total_bindings, 0);
15218 assert_eq!(
15219 total_bindings,
15220 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15221 );
15222 }
15223
15224 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
15231 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
15232 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
15233 let text = std::fs::read_to_string(&path)
15234 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
15235 let vectors: Vec<(String, Value)> = text
15236 .lines()
15237 .filter(|line| !line.trim().is_empty())
15238 .map(|line| {
15239 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
15240 let id = entry["corpus_id"]
15241 .as_str()
15242 .expect("every vector carries a corpus_id")
15243 .to_string();
15244 (id, entry["package"].clone())
15245 })
15246 .collect();
15247 assert_eq!(
15251 vectors.len(),
15252 3,
15253 "vendored fed corpus changed size; re-sync from subc-federation"
15254 );
15255
15256 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
15271 let richest = vectors
15272 .iter()
15273 .filter_map(|(_, package)| package.as_object())
15274 .map(|object| {
15275 object
15276 .keys()
15277 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
15278 .count()
15279 })
15280 .max()
15281 .unwrap_or(0);
15282 assert!(
15283 richest >= 2,
15284 "vendored corpus no longer carries a package with unmodelled fields, \
15285 so the relay test can no longer distinguish a verbatim relay from a lossy one"
15286 );
15287
15288 vectors
15289 }
15290
15291 #[tokio::test]
15299 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
15300 for (corpus_id, package) in fed_admission_facts_vectors() {
15301 let registry = Arc::new(Registry::default());
15302 let forwarding = Arc::new(ForwardingTable::default());
15303 let supervisor = SupervisorHandle::new();
15304 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15305 let handler =
15306 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15307 .with_supervisor(supervisor)
15308 .with_admission_facts_config(
15309 Some("fed".to_string()),
15310 Some(vec!["target".to_string()]),
15311 );
15312
15313 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
15314 hello_via_sink(
15315 &handler,
15316 &target_ctx,
15317 &mut target_rx,
15318 hello_frame("target", PROTOCOL_VERSION, 1),
15319 )
15320 .await;
15321
15322 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
15323 let route_handler = handler.clone();
15324 let expected = package.clone();
15325 let route_task = tokio::spawn(async move {
15326 route_handler
15327 .handle_control_frame(
15328 &client_ctx,
15329 route_open_frame_with_admission_facts(
15330 20,
15331 "target",
15332 unique_project_root("admission-facts"),
15333 Some(subc_control::ConsumerIdentity {
15334 module_id: "fed".to_string(),
15335 launch_nonce: "fed-nonce".to_string(),
15336 }),
15337 Some(package),
15338 ),
15339 )
15340 .await
15341 .unwrap()
15342 });
15343
15344 let bind_frame = target_rx.recv().await.unwrap();
15345 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15346 let ModuleControlRequest::RouteBind {
15347 admission_facts, ..
15348 } = bind
15349 else {
15350 panic!("{corpus_id}: expected route.bind")
15351 };
15352 assert_eq!(
15353 admission_facts,
15354 Some(expected),
15355 "{corpus_id}: relay must not add, drop or reshape any field"
15356 );
15357
15358 handler
15359 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15360 .await
15361 .unwrap();
15362 route_task.await.unwrap();
15363 }
15364 }
15365
15366 #[tokio::test]
15367 async fn route_bind_never_relays_the_openers_build_class() {
15368 let registry = Arc::new(Registry::default());
15369 let forwarding = Arc::new(ForwardingTable::default());
15370 let handler =
15371 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15372 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
15373 hello_via_sink(
15374 &handler,
15375 &target_ctx,
15376 &mut target_rx,
15377 hello_frame("target", PROTOCOL_VERSION, 1),
15378 )
15379 .await;
15380
15381 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
15382 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
15383 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15384 busy_refusals: None,
15385 target: RouteTarget::ToolProvider {
15386 module_id: "target".to_string(),
15387 },
15388 identity,
15389 consumer_identity: None,
15390 consumer_capabilities: None,
15391 role_versions: None,
15392 admission_facts: None,
15393 scope: None,
15394 })
15395 .unwrap();
15396 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
15397
15398 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
15399 let route_handler = handler.clone();
15400 let route_task = tokio::spawn(async move {
15401 route_handler
15402 .handle_control_frame(&client_ctx, open)
15403 .await
15404 .unwrap()
15405 });
15406 let bind_frame = target_rx.recv().await.unwrap();
15407 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15408 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
15409 panic!("expected route.bind")
15410 };
15411 assert_eq!(
15412 identity.build_class, None,
15413 "route.bind carried the build class"
15414 );
15415 assert!(
15416 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
15417 "the route.bind bytes still name build_class"
15418 );
15419 handler
15420 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15421 .await
15422 .unwrap();
15423 route_task.await.unwrap();
15424 }
15425
15426 #[tokio::test]
15427 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
15428 let registry = Arc::new(Registry::default());
15429 let forwarding = Arc::new(ForwardingTable::default());
15430 let supervisor = SupervisorHandle::new();
15431 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15432 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
15433 let handler =
15434 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15435 .with_supervisor(supervisor)
15436 .with_admission_facts_config(
15437 Some("fed".to_string()),
15438 Some(vec!["target".to_string()]),
15439 );
15440
15441 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
15442 hello_via_sink(
15443 &handler,
15444 &target_ctx,
15445 &mut target_rx,
15446 hello_frame("target", PROTOCOL_VERSION, 1),
15447 )
15448 .await;
15449 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
15450 hello_via_sink(
15451 &handler,
15452 &other_ctx,
15453 &mut other_rx,
15454 hello_frame("other", PROTOCOL_VERSION, 2),
15455 )
15456 .await;
15457
15458 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
15459 let expected_facts = facts.clone();
15460 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
15461 let route_handler = handler.clone();
15462 let route_task = tokio::spawn(async move {
15463 route_handler
15464 .handle_control_frame(
15465 &client_ctx,
15466 route_open_frame_with_admission_facts(
15467 10,
15468 "target",
15469 unique_project_root("admission-facts"),
15470 Some(subc_control::ConsumerIdentity {
15471 module_id: "fed".to_string(),
15472 launch_nonce: "fed-nonce".to_string(),
15473 }),
15474 Some(facts.clone()),
15475 ),
15476 )
15477 .await
15478 .unwrap()
15479 });
15480 let bind_frame = target_rx.recv().await.unwrap();
15481 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15482 let ModuleControlRequest::RouteBind {
15483 admission_facts, ..
15484 } = bind
15485 else {
15486 panic!("expected route.bind")
15487 };
15488 assert_eq!(admission_facts, Some(expected_facts));
15489 handler
15490 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15491 .await
15492 .unwrap();
15493 assert!(route_task.await.unwrap().is_empty());
15494 assert!(matches!(
15495 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
15496 .unwrap(),
15497 ClientControlResponse::RouteOpen { .. }
15498 ));
15499
15500 let direct = handler
15501 .handle_control_frame(
15502 &route_ctx(ConnectionId::new(73)).0,
15503 route_open_frame_with_admission_facts(
15504 11,
15505 "target",
15506 unique_project_root("admission-facts"),
15507 None,
15508 Some(json!({"x": 1})),
15509 ),
15510 )
15511 .await
15512 .unwrap();
15513 assert_eq!(
15514 parse_error(&direct[0])["code"],
15515 "admission_facts_not_permitted"
15516 );
15517
15518 let different_reserved = handler
15519 .handle_control_frame(
15520 &route_ctx(ConnectionId::new(77)).0,
15521 route_open_frame_with_admission_facts(
15522 15,
15523 "target",
15524 unique_project_root("admission-facts"),
15525 Some(subc_control::ConsumerIdentity {
15526 module_id: "other".to_string(),
15527 launch_nonce: "other-nonce".to_string(),
15528 }),
15529 Some(json!({"x": 1})),
15530 ),
15531 )
15532 .await
15533 .unwrap();
15534 assert_eq!(
15535 parse_error(&different_reserved[0])["code"],
15536 "admission_facts_not_permitted"
15537 );
15538
15539 let other_target = handler
15540 .handle_control_frame(
15541 &route_ctx(ConnectionId::new(74)).0,
15542 route_open_frame_with_admission_facts(
15543 12,
15544 "other",
15545 unique_project_root("admission-facts"),
15546 Some(subc_control::ConsumerIdentity {
15547 module_id: "fed".to_string(),
15548 launch_nonce: "fed-nonce".to_string(),
15549 }),
15550 Some(json!({"x": 1})),
15551 ),
15552 )
15553 .await
15554 .unwrap();
15555 assert_eq!(
15556 parse_error(&other_target[0])["code"],
15557 "admission_facts_target_not_allowed"
15558 );
15559
15560 let nonexistent = handler
15561 .handle_control_frame(
15562 &route_ctx(ConnectionId::new(75)).0,
15563 route_open_frame_with_admission_facts(
15564 13,
15565 "missing",
15566 unique_project_root("admission-facts"),
15567 None,
15568 Some(json!({"x": 1})),
15569 ),
15570 )
15571 .await
15572 .unwrap();
15573 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
15574
15575 let described = handler
15576 .handle_control_frame(
15577 &route_ctx(ConnectionId::new(76)).0,
15578 Frame::build(
15579 FrameType::Request,
15580 control_flags(),
15581 0,
15582 0,
15583 14,
15584 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15585 )
15586 .unwrap(),
15587 )
15588 .await
15589 .unwrap();
15590 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15591 serde_json::from_slice(&described[0].body).unwrap()
15592 else {
15593 panic!("expected server.describe response")
15594 };
15595 assert!(capabilities
15596 .iter()
15597 .any(|cap| cap == "admission_facts_relay_v1"));
15598 }
15599
15600 #[tokio::test]
15601 async fn admission_facts_without_configured_carrier_are_rejected() {
15602 let registry = Arc::new(Registry::default());
15603 let forwarding = Arc::new(ForwardingTable::default());
15604 let handler = ControlHandler::with_forwarding(registry, forwarding);
15605 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
15606 hello_via_sink(
15607 &handler,
15608 &target_ctx,
15609 &mut target_rx,
15610 hello_frame("target", PROTOCOL_VERSION, 1),
15611 )
15612 .await;
15613
15614 let responses = handler
15615 .handle_control_frame(
15616 &route_ctx(ConnectionId::new(79)).0,
15617 route_open_frame_with_admission_facts(
15618 16,
15619 "target",
15620 unique_project_root("admission-facts"),
15621 None,
15622 Some(json!({"x": 1})),
15623 ),
15624 )
15625 .await
15626 .unwrap();
15627 assert_eq!(
15628 parse_error(&responses[0])["code"],
15629 "admission_facts_not_permitted"
15630 );
15631 }
15632
15633 #[tokio::test]
15634 async fn route_open_relays_consumer_capabilities_verbatim() {
15635 let registry = Arc::new(Registry::default());
15636 let forwarding = Arc::new(ForwardingTable::default());
15637 let handler =
15638 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15639 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
15640 hello_via_sink(
15641 &handler,
15642 &module_ctx,
15643 &mut module_rx,
15644 hello_frame("aft", PROTOCOL_VERSION, 7),
15645 )
15646 .await;
15647
15648 let expected = vec!["elicitation".to_string(), "roots".to_string()];
15649 let expected_for_request = expected.clone();
15650 let project_root = unique_project_root("consumer-capabilities-present");
15651 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
15652 let route_handler = handler.clone();
15653 let route_task = tokio::spawn(async move {
15654 route_handler
15655 .handle_control_frame(
15656 &client_ctx,
15657 route_open_frame_with_consumer_capabilities(
15658 401,
15659 "aft",
15660 project_root,
15661 Some(expected_for_request),
15662 ),
15663 )
15664 .await
15665 .unwrap()
15666 });
15667 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15668 .await
15669 .unwrap()
15670 .unwrap();
15671 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15672 let ModuleControlRequest::RouteBind {
15673 consumer_capabilities,
15674 ..
15675 } = bind
15676 else {
15677 panic!("expected route.bind request, got {bind:?}");
15678 };
15679 assert_eq!(consumer_capabilities, Some(expected.clone()));
15680
15681 handler
15682 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15683 .await
15684 .unwrap();
15685 let route_response = route_task.await.unwrap();
15686 assert!(route_response.is_empty());
15687 let published = client_rx.recv().await.unwrap();
15688 assert!(matches!(
15689 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15690 ClientControlResponse::RouteOpen { .. }
15691 ));
15692 }
15693
15694 #[tokio::test]
15695 async fn route_open_without_consumer_capabilities_relays_none() {
15696 let registry = Arc::new(Registry::default());
15697 let forwarding = Arc::new(ForwardingTable::default());
15698 let handler =
15699 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15700 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
15701 hello_via_sink(
15702 &handler,
15703 &module_ctx,
15704 &mut module_rx,
15705 hello_frame("aft", PROTOCOL_VERSION, 7),
15706 )
15707 .await;
15708
15709 let project_root = unique_project_root("consumer-capabilities-absent");
15710 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
15711 let route_handler = handler.clone();
15712 let route_task = tokio::spawn(async move {
15713 route_handler
15714 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
15715 .await
15716 .unwrap()
15717 });
15718 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15719 .await
15720 .unwrap()
15721 .unwrap();
15722 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15723 let ModuleControlRequest::RouteBind {
15724 consumer_capabilities,
15725 ..
15726 } = bind
15727 else {
15728 panic!("expected route.bind request, got {bind:?}");
15729 };
15730 assert_eq!(consumer_capabilities, None);
15731
15732 handler
15733 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15734 .await
15735 .unwrap();
15736 let route_response = route_task.await.unwrap();
15737 assert!(route_response.is_empty());
15738 let published = client_rx.recv().await.unwrap();
15739 assert!(matches!(
15740 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15741 ClientControlResponse::RouteOpen { .. }
15742 ));
15743 }
15744
15745 async fn bind_role_versions_for(
15749 sent: Option<BTreeMap<String, String>>,
15750 connection: u64,
15751 ) -> Option<BTreeMap<String, String>> {
15752 let registry = Arc::new(Registry::default());
15753 let forwarding = Arc::new(ForwardingTable::default());
15754 let handler =
15755 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15756 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
15757 hello_via_sink(
15758 &handler,
15759 &module_ctx,
15760 &mut module_rx,
15761 hello_frame("aft", PROTOCOL_VERSION, 7),
15762 )
15763 .await;
15764 let project_root = unique_project_root("role-versions");
15765 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
15766 let route_handler = handler.clone();
15767 let route_task = tokio::spawn(async move {
15768 route_handler
15769 .handle_control_frame(
15770 &client_ctx,
15771 route_open_frame_with_role_versions(403, "aft", project_root, sent),
15772 )
15773 .await
15774 .unwrap()
15775 });
15776 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15777 .await
15778 .expect("a well-formed route.open reaches the module as a bind")
15779 .unwrap();
15780 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15781 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
15782 panic!("expected route.bind request, got {bind:?}");
15783 };
15784 handler
15785 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15786 .await
15787 .unwrap();
15788 assert!(route_task.await.unwrap().is_empty());
15789 let published = client_rx.recv().await.unwrap();
15790 assert!(matches!(
15791 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15792 ClientControlResponse::RouteOpen { .. }
15793 ));
15794 role_versions
15795 }
15796
15797 #[tokio::test]
15798 async fn route_open_relays_role_versions_verbatim() {
15799 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
15800 assert_eq!(
15801 bind_role_versions_for(Some(sent.clone()), 141).await,
15802 Some(sent)
15803 );
15804 }
15805
15806 #[tokio::test]
15809 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
15810 assert_eq!(bind_role_versions_for(None, 143).await, None);
15811 assert_eq!(
15812 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
15813 None
15814 );
15815 }
15816
15817 #[tokio::test]
15818 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
15819 let registry = Arc::new(Registry::default());
15820 let forwarding = Arc::new(ForwardingTable::default());
15821 let handler =
15822 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15823 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
15824 hello_via_sink(
15825 &handler,
15826 &module_ctx,
15827 &mut module_rx,
15828 hello_frame("aft", PROTOCOL_VERSION, 7),
15829 )
15830 .await;
15831
15832 let nine: BTreeMap<String, String> = (0..9)
15833 .map(|index| (format!("role-{index}"), "v1".to_string()))
15834 .collect();
15835 for (label, malformed) in [
15836 (
15837 "invalid role name",
15838 role_versions(&[("Tool_Provider", "v1")]),
15839 ),
15840 ("invalid version", role_versions(&[("tool-provider", "v0")])),
15841 ("nine entries", nine),
15842 ] {
15843 let responses = tokio::time::timeout(
15846 Duration::from_secs(1),
15847 handler.handle_control_frame(
15848 &route_ctx(ConnectionId::new(148)).0,
15849 route_open_frame_with_role_versions(
15850 404,
15851 "aft",
15852 unique_project_root("role-versions-malformed"),
15853 Some(malformed),
15854 ),
15855 ),
15856 )
15857 .await
15858 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
15859 .unwrap();
15860 assert_eq!(responses.len(), 1, "{label}");
15861 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
15862 let error = parse_error(&responses[0]);
15863 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
15864 assert_eq!(
15865 error["detail"]["field"], "role_versions",
15866 "{label}: {error}"
15867 );
15868 assert!(
15869 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
15870 "{label}: a malformed declaration is terminal"
15871 );
15872 assert!(
15873 module_rx.try_recv().is_err(),
15874 "{label}: the module must never see a bind"
15875 );
15876 }
15877 }
15878
15879 #[tokio::test]
15883 async fn route_role_versions_capability_is_advertised() {
15884 let handler = ControlHandler::new(Arc::new(Registry::default()));
15885 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
15886 let ack = hello_via_sink(
15887 &handler,
15888 &ctx,
15889 &mut rx,
15890 hello_frame("m", PROTOCOL_VERSION, 1),
15891 )
15892 .await;
15893 let ack = parse_ack(&ack);
15894 assert!(
15895 ack.subc_capabilities
15896 .iter()
15897 .any(|c| c == "route-role-versions/v1"),
15898 "{:?}",
15899 ack.subc_capabilities
15900 );
15901
15902 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15903 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15904 let reply = handler
15905 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
15906 .await
15907 .unwrap()
15908 .pop()
15909 .unwrap();
15910 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15911 serde_json::from_slice(&reply.body).unwrap()
15912 else {
15913 panic!("not a server.describe reply");
15914 };
15915 assert!(
15916 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
15917 "{capabilities:?}"
15918 );
15919 }
15920
15921 #[tokio::test]
15925 async fn catalog_events_capability_is_advertised() {
15926 let handler = ControlHandler::new(Arc::new(Registry::default()));
15927 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15928 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15929 let reply = handler
15930 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
15931 .await
15932 .unwrap()
15933 .pop()
15934 .unwrap();
15935 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15936 serde_json::from_slice(&reply.body).unwrap()
15937 else {
15938 panic!("not a server.describe reply");
15939 };
15940 assert!(
15941 capabilities
15942 .iter()
15943 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
15944 "{capabilities:?}"
15945 );
15946 }
15947
15948 #[tokio::test]
15949 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
15950 let handler = ControlHandler::new(Arc::new(Registry::default()));
15951 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
15952 let ack = hello_via_sink(
15953 &handler,
15954 &module,
15955 &mut module_rx,
15956 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
15957 )
15958 .await;
15959 let ack = parse_ack(&ack);
15960 assert!(
15961 ack.subc_capabilities
15962 .iter()
15963 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15964 "{:?}",
15965 ack.subc_capabilities
15966 );
15967 assert!(ack
15968 .subc_capabilities
15969 .iter()
15970 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15971
15972 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
15973 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15974 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15975 let reply = handler
15976 .handle_control_frame(&client, frame)
15977 .await
15978 .unwrap()
15979 .pop()
15980 .unwrap();
15981 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15982 serde_json::from_slice(&reply.body).unwrap()
15983 else {
15984 panic!("not a server.describe reply");
15985 };
15986 assert!(
15987 capabilities
15988 .iter()
15989 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15990 "{capabilities:?}"
15991 );
15992 assert!(capabilities
15993 .iter()
15994 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15995 }
15996
15997 #[tokio::test]
15998 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
15999 let handler = ControlHandler::new(Arc::new(Registry::default()));
16000 let (module, mut rx) = route_ctx(ConnectionId::new(156));
16001 let ack = hello_via_sink(
16002 &handler,
16003 &module,
16004 &mut rx,
16005 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
16006 )
16007 .await;
16008 assert!(parse_ack(&ack)
16009 .subc_capabilities
16010 .iter()
16011 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
16012 assert!(parse_ack(&ack)
16013 .subc_capabilities
16014 .iter()
16015 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
16016 let (client, _rx) = route_ctx(ConnectionId::new(157));
16017 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16018 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16019 let reply = handler
16020 .handle_control_frame(&client, frame)
16021 .await
16022 .unwrap()
16023 .pop()
16024 .unwrap();
16025 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16026 serde_json::from_slice(&reply.body).unwrap()
16027 else {
16028 panic!("server.describe reply")
16029 };
16030 assert!(capabilities
16031 .iter()
16032 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
16033 assert!(capabilities
16034 .iter()
16035 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
16036 }
16037
16038 #[tokio::test]
16039 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
16040 let handler = ControlHandler::new(Arc::new(Registry::default()));
16041 let (module, mut rx) = route_ctx(ConnectionId::new(158));
16042 let ack = hello_via_sink(
16043 &handler,
16044 &module,
16045 &mut rx,
16046 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
16047 )
16048 .await;
16049 assert!(parse_ack(&ack)
16050 .subc_capabilities
16051 .iter()
16052 .any(|cap| cap == "scope-workdir/v1"));
16053 let (client, _rx) = route_ctx(ConnectionId::new(159));
16054 let frame = Frame::build(
16055 FrameType::Request,
16056 control_flags(),
16057 0,
16058 0,
16059 5,
16060 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16061 )
16062 .unwrap();
16063 let reply = handler
16064 .handle_control_frame(&client, frame)
16065 .await
16066 .unwrap()
16067 .remove(0);
16068 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16069 serde_json::from_slice(&reply.body).unwrap()
16070 else {
16071 panic!("server.describe reply")
16072 };
16073 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
16074 }
16075
16076 #[tokio::test]
16077 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
16078 let handler = ControlHandler::new(Arc::new(Registry::default()));
16079 let (module, mut rx) = route_ctx(ConnectionId::new(160));
16080 let ack = hello_via_sink(
16081 &handler,
16082 &module,
16083 &mut rx,
16084 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
16085 )
16086 .await;
16087 assert!(parse_ack(&ack)
16088 .subc_capabilities
16089 .iter()
16090 .any(|cap| cap == "worker-runs/v1"));
16091 let (client, _rx) = route_ctx(ConnectionId::new(161));
16092 let frame = Frame::build(
16093 FrameType::Request,
16094 control_flags(),
16095 0,
16096 0,
16097 5,
16098 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16099 )
16100 .unwrap();
16101 let reply = handler
16102 .handle_control_frame(&client, frame)
16103 .await
16104 .unwrap()
16105 .remove(0);
16106 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16107 serde_json::from_slice(&reply.body).unwrap()
16108 else {
16109 panic!("server.describe reply")
16110 };
16111 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
16112 }
16113
16114 #[tokio::test]
16115 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
16116 let handler = ControlHandler::new(Arc::new(Registry::default()));
16117 let (module, mut rx) = route_ctx(ConnectionId::new(154));
16118 let ack = hello_via_sink(
16119 &handler,
16120 &module,
16121 &mut rx,
16122 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
16123 )
16124 .await;
16125 assert!(parse_ack(&ack)
16126 .subc_capabilities
16127 .iter()
16128 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16129 let (client, _rx) = route_ctx(ConnectionId::new(155));
16130 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16131 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16132 let reply = handler
16133 .handle_control_frame(&client, frame)
16134 .await
16135 .unwrap()
16136 .pop()
16137 .unwrap();
16138 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16139 serde_json::from_slice(&reply.body).unwrap()
16140 else {
16141 panic!("not a server.describe reply")
16142 };
16143 assert!(capabilities
16144 .iter()
16145 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16146 }
16147
16148 #[tokio::test]
16149 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
16150 let registry = Arc::new(Registry::default());
16151 let forwarding = Arc::new(ForwardingTable::default());
16152 let handler =
16153 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16154 .with_health_probe_timeout(Duration::from_secs(5));
16155 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
16156 hello_via_sink(
16157 &handler,
16158 &module_ctx,
16159 &mut module_rx,
16160 non_routable_hello_frame_with_control_ops(
16161 "mcp",
16162 300,
16163 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16164 ),
16165 )
16166 .await;
16167 assert!(registry
16168 .get_module("mcp")
16169 .unwrap()
16170 .unwrap()
16171 .manifest
16172 .provides
16173 .is_empty());
16174
16175 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
16176 let route_response = handler
16177 .handle_control_frame(
16178 &route_client_ctx,
16179 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
16180 )
16181 .await
16182 .unwrap();
16183 assert_eq!(route_response[0].header.ty, FrameType::Error);
16184 assert_eq!(
16185 parse_error(&route_response[0])["code"],
16186 "target_unavailable"
16187 );
16188 assert!(parse_error(&route_response[0])["message"]
16189 .as_str()
16190 .unwrap()
16191 .contains("does not provide the requested target"));
16192 assert!(module_rx.try_recv().is_err());
16193
16194 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
16195 let health_handler = handler.clone();
16196 let health_task = tokio::spawn(async move {
16197 health_handler
16198 .handle_control_frame(
16199 &health_client_ctx,
16200 supervisor_health_probe_frame(302, "mcp"),
16201 )
16202 .await
16203 .unwrap()
16204 });
16205 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16206 .await
16207 .unwrap()
16208 .unwrap();
16209 assert_eq!(
16210 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
16211 ModuleControlRequest::HealthCheck {}
16212 );
16213 handler
16214 .handle_control_frame(
16215 &module_ctx,
16216 health_response(health_frame.header.corr, HealthStatus::Ok),
16217 )
16218 .await
16219 .unwrap();
16220 let health_response = health_task.await.unwrap();
16221 assert_eq!(health_response[0].header.ty, FrameType::Response);
16222 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
16223 ClientControlResponse::SupervisorHealthProbe {
16224 module_id, status, ..
16225 } => {
16226 assert_eq!(module_id, "mcp");
16227 assert_eq!(status, HealthStatus::Ok);
16228 }
16229 other => panic!("unexpected health response: {other:?}"),
16230 }
16231
16232 forwarding
16237 .cleanup_connection(module_ctx.connection_id)
16238 .unwrap();
16239 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
16240 let cleanup_response = tokio::time::timeout(
16241 Duration::from_millis(200),
16242 handler.handle_control_frame(
16243 &cleanup_probe_ctx,
16244 supervisor_health_probe_frame(303, "mcp"),
16245 ),
16246 )
16247 .await
16248 .expect("probe should fail immediately when the control lane is gone")
16249 .unwrap();
16250 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
16251 assert_eq!(
16252 parse_error(&cleanup_response[0])["code"],
16253 "target_unavailable"
16254 );
16255 assert!(parse_error(&cleanup_response[0])["message"]
16256 .as_str()
16257 .unwrap()
16258 .contains("no module connection"));
16259
16260 handler
16261 .cleanup_connection(module_ctx.connection_id)
16262 .unwrap();
16263 }
16264
16265 #[tokio::test]
16266 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
16267 let registry = Arc::new(Registry::default());
16268 let supervisor_handle = SupervisorHandle::new();
16269 let supervisor =
16270 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
16271 .with_handle(supervisor_handle.clone())
16272 .with_connection_file_path(
16273 std::env::temp_dir()
16274 .join(format!("subc-route-open-warming-{}", std::process::id())),
16275 );
16276 let module = supervisor
16277 .supervise_configured(
16278 ModuleSpec {
16279 script: None,
16280 carry_containment: false,
16281 module_id: "warming".to_string(),
16282 program: fake_aft_stub_path(),
16283 args: Vec::new(),
16284 env: Vec::new(),
16285 reserved: false,
16286 reserved_prefixes: Vec::new(),
16287 protocol: ModuleProtocol::Subc,
16288 overlap: Default::default(),
16289 },
16290 true,
16291 )
16292 .unwrap();
16293 assert_eq!(module.state().unwrap(), ModuleState::Running);
16294
16295 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16296 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
16297 let response = handler
16298 .handle_control_frame(
16299 &ctx,
16300 route_open_frame(304, "warming", unique_project_root("warming")),
16301 )
16302 .await
16303 .unwrap();
16304 module.stop().await.unwrap();
16305
16306 assert_eq!(response[0].header.ty, FrameType::Error);
16307 let error = parse_error(&response[0]);
16308 assert_eq!(error["code"], "module_warming");
16309 assert!(error["message"]
16310 .as_str()
16311 .unwrap()
16312 .contains("state=running, enabled=true, live=false"));
16313 }
16314
16315 #[test]
16316 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
16317 let handler = ControlHandler::new(Arc::new(Registry::default()));
16318 let capture = EventCapture::default();
16319 let _subscriber =
16320 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16321 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
16322 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
16323 let pending = (0..limit).collect::<Vec<_>>();
16324 let response = handler
16325 .route_open_capacity_refusal(
16326 &ctx,
16327 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
16328 "busy",
16329 pending.len(),
16330 limit,
16331 )
16332 .unwrap();
16333 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16334 let event = capture
16335 .events()
16336 .into_iter()
16337 .find(|event| {
16338 event.target == "control"
16339 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
16340 })
16341 .expect("connection admission refusal event");
16342 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16343 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16344 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
16347 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
16348 }
16349
16350 #[test]
16351 fn route_open_target_cap_logs_admission_reason_and_capacity() {
16352 let handler = ControlHandler::new(Arc::new(Registry::default()));
16353 let capture = EventCapture::default();
16354 let _subscriber =
16355 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16356 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
16357 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
16358 let guards = (0..limit)
16359 .map(|_| {
16360 handler
16361 .route_bind_concurrency
16362 .try_admit("busy", limit)
16363 .unwrap()
16364 })
16365 .collect::<Vec<_>>();
16366 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
16367 Err(in_flight) => in_flight,
16368 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
16369 };
16370 let response = handler
16371 .route_open_target_capacity_refusal(
16372 &ctx,
16373 &RouteOpenCaller::unknown(),
16374 &route_open_frame(397, "busy", unique_project_root("target-cap")),
16375 "busy",
16376 in_flight,
16377 )
16378 .unwrap();
16379 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16380 let event = capture
16381 .events()
16382 .into_iter()
16383 .find(|event| {
16384 event.target == "control"
16385 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
16386 })
16387 .expect("target admission refusal event");
16388 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16389 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16390 drop(guards);
16391 }
16392
16393 #[cfg(unix)]
16394 #[tokio::test(start_paused = true)]
16395 async fn supervisor_list_shows_a_failed_modules_next_retry() {
16396 let registry = Arc::new(Registry::default());
16397 let handle = SupervisorHandle::new();
16398 let supervisor =
16399 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
16400 .with_handle(handle.clone());
16401 let module = supervisor
16402 .spawn(crate::ModuleSpec {
16403 script: None,
16404 carry_containment: false,
16405 module_id: "retry-list".into(),
16406 program: PathBuf::from("/bin/sh"),
16407 args: vec!["-c".into(), "exit 17".into()],
16408 env: vec![],
16409 reserved: false,
16410 reserved_prefixes: vec![],
16411 protocol: ModuleProtocol::None,
16412 overlap: Default::default(),
16413 })
16414 .unwrap();
16415 let deadline = std::time::Instant::now() + Duration::from_secs(10);
16416 while module.state().unwrap() != crate::ModuleState::Failed {
16417 assert!(
16418 std::time::Instant::now() < deadline,
16419 "waiting for budget failure"
16420 );
16421 tokio::task::yield_now().await;
16422 }
16423 let handler = ControlHandler::new(registry).with_supervisor(handle);
16424 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16425 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16426 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16427 assert_eq!(body["modules"][0]["state"], "failed");
16428 let retry = body["modules"][0]["next_retry_at_ms"]
16429 .as_u64()
16430 .expect("scheduled retry on the wire");
16431 let now = std::time::SystemTime::now()
16432 .duration_since(std::time::UNIX_EPOCH)
16433 .unwrap()
16434 .as_millis() as u64;
16435 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
16436 module.set_enabled(false).await.unwrap();
16437 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16438 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16439 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16440 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
16441 }
16442
16443 #[tokio::test]
16444 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
16445 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
16446 let supervisor =
16447 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
16448 let module = supervisor
16449 .spawn(crate::ModuleSpec {
16450 script: None,
16451 carry_containment: false,
16452 module_id: "pid-read-budget".into(),
16453 program: std::env::current_exe().unwrap(),
16454 args: vec![
16455 "--exact".into(),
16456 "control::tests::pid_resource_read_fixture".into(),
16457 ],
16458 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16459 .into_iter()
16460 .map(|key| (key.into(), dir.display().to_string()))
16461 .chain(std::iter::once((
16462 "SUBC_SUBTREE_PID_FIXTURE".into(),
16463 "1".into(),
16464 )))
16465 .collect(),
16466 reserved: false,
16467 reserved_prefixes: Vec::new(),
16468 protocol: ModuleProtocol::None,
16469 overlap: Default::default(),
16470 })
16471 .unwrap();
16472 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16473 while !matches!(
16474 module.child_resource_usage(),
16475 subc_control::ChildResourceUsage::Measured(_)
16476 ) {
16477 assert!(
16478 tokio::time::Instant::now() < ready_deadline,
16479 "fixture process was not measurable"
16480 );
16481 tokio::time::sleep(Duration::from_millis(10)).await;
16482 }
16483 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
16484 let reading = {
16485 let _guard = crate::subtree_resources::fault_read_guard().await;
16486 crate::subtree_resources::read_process(module.clone()).await
16487 };
16488 stop_resource_fixture(&module, &dir).await;
16489 assert!(
16490 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
16491 "{reading:?}"
16492 );
16493 }
16494
16495 #[test]
16496 fn pid_resource_read_fixture() {
16497 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
16498 let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
16499 let deadline = std::time::Instant::now() + Duration::from_secs(60);
16500 while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
16501 std::thread::sleep(Duration::from_millis(5));
16502 }
16503 std::fs::write(dir.join("module-stopped"), "").unwrap();
16504 }
16505 }
16506
16507 async fn stop_resource_fixture(
16508 module: &crate::supervise::SupervisedModule,
16509 dir: &std::path::Path,
16510 ) {
16511 let stopping = module.stop();
16512 tokio::pin!(stopping);
16513 tokio::select! {
16514 result = &mut stopping => result.unwrap(),
16515 () = async {
16516 while module.state().unwrap() != crate::supervise::ModuleState::Draining {
16517 tokio::time::sleep(Duration::from_millis(5)).await;
16518 }
16519 std::fs::write(dir.join("stop-module"), "").unwrap();
16520 } => stopping.await.unwrap(),
16521 }
16522 #[cfg(windows)]
16523 assert!(
16524 dir.join("module-stopped").exists(),
16525 "the resource fixture must exit cooperatively, not wait for the drain kill"
16526 );
16527 assert_eq!(
16528 module.state().unwrap(),
16529 crate::supervise::ModuleState::Stopped
16530 );
16531 }
16532
16533 #[tokio::test]
16534 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
16535 let registry = Arc::new(Registry::default());
16536 let handle = SupervisorHandle::new();
16537 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16538 .with_handle(handle.clone());
16539 supervisor
16540 .supervise_configured(
16541 crate::ModuleSpec {
16542 script: None,
16543 module_id: "disabled-resource-reading".into(),
16544 program: std::env::current_exe().unwrap(),
16545 args: Vec::new(),
16546 env: Vec::new(),
16547 reserved: false,
16548 reserved_prefixes: Vec::new(),
16549 protocol: ModuleProtocol::None,
16550 overlap: Default::default(),
16551 carry_containment: false,
16552 },
16553 false,
16554 )
16555 .unwrap();
16556 let handler = ControlHandler::new(registry).with_supervisor(handle);
16557 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16558 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16559 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16560 assert!(body["modules"][0].get("tree").is_none());
16561 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
16562 }
16563
16564 #[tokio::test]
16565 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
16566 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
16567 let registry = Arc::new(Registry::default());
16568 let handle = SupervisorHandle::new();
16569 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16570 .with_handle(handle.clone());
16571 let module = supervisor
16572 .spawn(crate::ModuleSpec {
16573 script: None,
16574 carry_containment: false,
16575 module_id: "list-process-priority".into(),
16576 program: std::env::current_exe().unwrap(),
16577 args: vec![
16578 "--exact".into(),
16579 "control::tests::pid_resource_read_fixture".into(),
16580 ],
16581 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16582 .into_iter()
16583 .map(|key| (key.into(), dir.display().to_string()))
16584 .chain(std::iter::once((
16585 "SUBC_SUBTREE_PID_FIXTURE".into(),
16586 "1".into(),
16587 )))
16588 .collect(),
16589 reserved: false,
16590 reserved_prefixes: Vec::new(),
16591 protocol: ModuleProtocol::None,
16592 overlap: Default::default(),
16593 })
16594 .unwrap();
16595 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16596 loop {
16597 let image = module.running_image_agreement().await;
16598 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
16599 break;
16600 }
16601 assert!(
16602 tokio::time::Instant::now() < ready_deadline,
16603 "the fixture process never became observable: {image:?}"
16604 );
16605 tokio::time::sleep(Duration::from_millis(10)).await;
16606 }
16607 let guard = crate::subtree_resources::fault_read_guard().await;
16608 let (target, release, _) = crate::subtree_resources::held();
16609 module.set_subtree_target_for_test(target);
16610 let handler = ControlHandler::new(registry).with_supervisor(handle);
16611 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16612 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16613 drop(release);
16614 drop(guard);
16615 stop_resource_fixture(&module, &dir).await;
16616 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16617 assert_eq!(
16618 body["modules"][0]["resources"]["status"], "measured",
16619 "the optional tree timeout must not discard the process reading: {body}"
16620 );
16621 assert!(body["modules"][0].get("tree").is_none());
16622 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
16623 assert_eq!(
16624 body["modules"][0]["pending_reload"]["image"]["status"], "match",
16625 "the tree timeout must not hide a completed image observation"
16626 );
16627 }
16628
16629 #[cfg(any(target_os = "linux", target_os = "macos"))]
16630 #[tokio::test]
16631 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
16632 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
16633 .with_daemon_provenance(
16634 std::process::id(),
16635 0,
16636 Some(std::env::current_exe().unwrap()),
16637 None,
16638 None,
16639 );
16640 let (probe, release, began) = ExecutableIdentityProbe::held();
16641 handler.daemon_provenance.probe = probe;
16642 let observe = |handler: ControlHandler, corr| {
16643 tokio::spawn(async move {
16644 let (ctx, _) = route_ctx(ConnectionId::new(corr));
16645 let frame =
16646 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
16647 .unwrap();
16648 handler
16649 .handle_supervisor_provenance(&ctx, frame, None)
16650 .await
16651 .unwrap()
16652 })
16653 };
16654 let explicit = observe(handler.clone(), 114);
16655 began.await.unwrap();
16656 let concurrent = observe(handler.clone(), 115);
16657 tokio::time::sleep(Duration::from_millis(300)).await;
16660 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16661 let list = tokio::time::timeout(
16662 Duration::from_secs(1),
16663 handler.handle_supervisor_list(frame),
16664 )
16665 .await;
16666 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
16667 drop(release);
16668 let replies = explicit.await.unwrap();
16669 let concurrent_replies = concurrent.await.unwrap();
16670 assert!(
16671 explicit_was_pending,
16672 "explicit provenance must await the full observation"
16673 );
16674 let list = list
16675 .expect("a concurrent explicit probe must not stall list")
16676 .unwrap();
16677 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
16678 assert_eq!(list_body["op"], "supervisor.list");
16679 for replies in [replies, concurrent_replies] {
16680 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16681 assert_eq!(
16682 body["daemon"]["daemon_observed"]["running_image"]["status"],
16683 "match"
16684 );
16685 }
16686 }
16687
16688 #[tokio::test]
16689 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
16690 let registry = Arc::new(Registry::default());
16691 let handle = SupervisorHandle::new();
16692 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16693 .with_handle(handle.clone());
16694 let module = supervisor
16695 .supervise_configured(
16696 crate::ModuleSpec {
16697 script: None,
16698 carry_containment: false,
16699 module_id: "memory-read".into(),
16700 program: PathBuf::from("disabled-module"),
16701 args: Vec::new(),
16702 env: Vec::new(),
16703 reserved: false,
16704 reserved_prefixes: Vec::new(),
16705 protocol: ModuleProtocol::Subc,
16706 overlap: Default::default(),
16707 },
16708 true,
16709 )
16710 .unwrap();
16711 let _guard = crate::subtree_resources::fault_read_guard().await;
16712 let (target, release, began) = crate::subtree_resources::held();
16713 let stalled_target = target.clone();
16716 let stalled = tokio::task::spawn_blocking(move || {
16717 stalled_target.tree(&subc_os::ProcessTable::default())
16718 });
16719 began.await.unwrap();
16720 module.set_subtree_target_for_test(target);
16721 let handler = ControlHandler::new(registry).with_supervisor(handle);
16722 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
16723 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16724 let replies = handler
16725 .handle_client_control_request(
16726 &ctx,
16727 frame,
16728 ClientControlRequest::SupervisorList {},
16729 &mut None,
16730 )
16731 .await
16732 .unwrap();
16733 assert!(
16734 replies.is_empty(),
16735 "list reply must be detached from the connection reader"
16736 );
16737 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16738 let replies = handler
16739 .handle_client_control_request(
16740 &ctx,
16741 frame,
16742 ClientControlRequest::SupervisorSpawnSnapshot {},
16743 &mut None,
16744 )
16745 .await
16746 .unwrap();
16747 assert_eq!(
16748 replies.len(),
16749 1,
16750 "another request must complete while the subtree read stalls"
16751 );
16752 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16753 .await
16754 .unwrap()
16755 .unwrap();
16756 release.send(()).unwrap();
16757 drop(release);
16758 stalled.await.unwrap();
16759 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
16760 assert!(
16761 body["modules"][0].get("tree").is_none(),
16762 "an unknown tree reading must be omitted after the bounded read expires"
16763 );
16764 handler
16768 .resource_tracker
16769 .read(handler.supervisor.list(), false, "released_tree_fixture")
16770 .await;
16771 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16772 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16773 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16774 assert_eq!(
16775 body["modules"][0]["tree"]["source"], "process_tree",
16776 "the next request must export the completed refresh"
16777 );
16778 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
16779 }
16780
16781 #[tokio::test]
16782 async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
16783 use subc_control::{
16784 AgentCommandResources, ChildMemoryKind, ChildResourceReading,
16785 ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
16786 TreeReading, TreeSource,
16787 };
16788 let registry = Arc::new(Registry::default());
16789 let handle = SupervisorHandle::new();
16790 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16791 .with_handle(handle.clone());
16792 supervisor
16793 .supervise_configured(
16794 crate::ModuleSpec {
16795 script: None,
16796 carry_containment: false,
16797 module_id: "budget-worker".into(),
16798 program: PathBuf::from("disabled-module"),
16799 args: Vec::new(),
16800 env: Vec::new(),
16801 reserved: false,
16802 reserved_prefixes: Vec::new(),
16803 protocol: ModuleProtocol::Subc,
16804 overlap: Default::default(),
16805 },
16806 true,
16807 )
16808 .unwrap();
16809 let handler = ControlHandler::new(registry).with_supervisor(handle);
16810 let now = u64::try_from(
16811 std::time::SystemTime::now()
16812 .duration_since(std::time::UNIX_EPOCH)
16813 .unwrap()
16814 .as_millis(),
16815 )
16816 .unwrap();
16817 let measured = now - 1_000;
16818 let hour = CpuLastHour::new(100, 60_000);
16819 let daemon = DaemonResources::new(7, 1, "test")
16820 .with_measured_at_ms(Some(measured))
16821 .with_resources(Some(ChildResourceUsage::Measured(
16822 ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
16823 )))
16824 .with_cpu_last_hour(Some(hour.clone()));
16825 let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
16826 .with_measured_at_ms(Some(measured))
16827 .with_cpu_last_hour(Some(hour.clone()));
16828 let tree = TreeReading::new(
16829 150,
16830 50,
16831 ChildMemoryKind::ResidentSet,
16832 1,
16833 TreeSource::ProcessTree,
16834 )
16835 .with_measured_at_ms(Some(measured));
16836 handler.resource_tracker.seed_committed_for_test(
16837 &handler.supervisor.list(),
16838 crate::resource_totals::Readings {
16839 modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
16840 daemon: Some(daemon),
16841 agent_commands: Some(vec![agents]),
16842 ..Default::default()
16843 },
16844 now,
16845 );
16846 list_diagnostics::scope(async {
16847 tokio::time::sleep(Duration::from_millis(110)).await;
16848 assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
16849 let request =
16850 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16851 let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
16852 let ClientControlResponse::SupervisorList {
16853 modules,
16854 daemon,
16855 agent_commands,
16856 ..
16857 } = serde_json::from_slice(&replies[0].body).unwrap()
16858 else {
16859 panic!("expected a list response")
16860 };
16861 assert!(matches!(
16862 modules[0].resources,
16863 Some(ChildResourceUsage::Unavailable {
16864 reason: ChildResourceUnavailableReason::Unreadable
16865 })
16866 ));
16867 assert_eq!(
16868 (
16869 modules[0].tree.as_ref().unwrap().cpu_ms,
16870 modules[0].tree.as_ref().unwrap().measured_at_ms
16871 ),
16872 (150, Some(measured))
16873 );
16874 let daemon = daemon.unwrap();
16875 let agents = agent_commands.unwrap();
16876 assert_eq!(daemon.measured_at_ms, Some(measured));
16877 assert_eq!(
16878 (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
16879 (1, 50, Some(measured))
16880 );
16881 for hour in [
16882 modules[0].cpu_last_hour.as_ref(),
16883 daemon.cpu_last_hour.as_ref(),
16884 agents[0].cpu_last_hour.as_ref(),
16885 ] {
16886 let hour = hour.unwrap();
16887 assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
16888 }
16889 })
16890 .await;
16891 }
16892
16893 #[tokio::test]
16894 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
16895 let registry = Arc::new(Registry::default());
16896 let handle = SupervisorHandle::new();
16897 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16898 .with_handle(handle.clone());
16899 supervisor
16900 .supervise_configured(
16901 crate::ModuleSpec {
16902 script: None,
16903 carry_containment: false,
16904 module_id: "drift".into(),
16905 program: PathBuf::from("disabled-module"),
16906 args: Vec::new(),
16907 env: Vec::new(),
16908 reserved: false,
16909 reserved_prefixes: Vec::new(),
16910 protocol: ModuleProtocol::Subc,
16911 overlap: Default::default(),
16912 },
16913 false,
16914 )
16915 .unwrap();
16916 let handler = ControlHandler::new(registry).with_supervisor(handle);
16917 for configured in [None, Some(false), Some(true)] {
16918 handler.capability_evaluator.configure(
16919 configured.map(|enabled| ("drift".to_string(), enabled)),
16920 BTreeMap::new(),
16921 );
16922 let frame =
16923 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16924 let replies = handler
16925 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
16926 .await
16927 .unwrap();
16928 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16929 assert!(body.get("configured_enabled").is_none());
16930 assert_eq!(body["op"], "supervisor.ack");
16931 let frame =
16932 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16933 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16934 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16935 assert_eq!(
16936 body["modules"][0].get("configured_enabled"),
16937 configured.map(Value::Bool).as_ref()
16938 );
16939 assert!(body["modules"][0].get("tree").is_none());
16940 assert!(body["modules"][0].get("subtree").is_none());
16941 }
16942 }
16943
16944 #[tokio::test(flavor = "current_thread")]
16945 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
16946 let handler = ControlHandler::new(Arc::new(Registry::default()));
16947 let capture = EventCapture::default();
16948 let _subscriber =
16949 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16950 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
16951 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16952
16953 let _ = handler
16954 .handle_client_control_request(
16955 &ctx,
16956 frame,
16957 ClientControlRequest::SupervisorSetEnabled {
16958 module_id: "broca".to_string(),
16959 enabled: false,
16960 },
16961 &mut None,
16962 )
16963 .await
16964 .unwrap();
16965
16966 let events = capture
16967 .events()
16968 .into_iter()
16969 .filter(|event| {
16970 event.target == "control"
16971 && event.fields.get("message").map(String::as_str)
16972 == Some("supervisor request received")
16973 })
16974 .collect::<Vec<_>>();
16975 assert_eq!(events.len(), 1, "one supervisor request log line");
16976 let event = &events[0];
16977 assert_eq!(event.level, tracing::Level::INFO);
16978 assert_eq!(
16979 event.fields.get("op"),
16980 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
16981 );
16982 assert_eq!(
16983 event.fields.get("module_id"),
16984 Some(&"\"broca\"".to_string())
16985 );
16986 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
16987 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
16988 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
16989 }
16990
16991 #[tokio::test(flavor = "current_thread")]
16992 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
16993 let registry = Arc::new(Registry::default());
16994 let module_connection = ConnectionId::new(303);
16995 registry
16996 .register_with_control_ops(
16997 manifest("aft", PROTOCOL_VERSION),
16998 PROTOCOL_VERSION,
16999 module_connection,
17000 module_baseline_control_ops(),
17001 )
17002 .unwrap();
17003 let handler = ControlHandler::new(registry);
17004 let capture = EventCapture::default();
17005 let _subscriber =
17006 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17007 let (ctx, _rx) = route_ctx(module_connection);
17008 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
17009
17010 let _ = handler
17011 .handle_client_control_request(
17012 &ctx,
17013 frame,
17014 ClientControlRequest::SupervisorSetEnabled {
17015 module_id: "broca".to_string(),
17016 enabled: false,
17017 },
17018 &mut None,
17019 )
17020 .await
17021 .unwrap();
17022
17023 let callers = capture
17024 .events()
17025 .into_iter()
17026 .filter(|event| {
17027 event.fields.get("message").map(String::as_str)
17028 == Some("supervisor request received")
17029 })
17030 .map(|event| event.fields.get("caller").cloned())
17031 .collect::<Vec<_>>();
17032 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
17033 }
17034
17035 #[tokio::test(flavor = "current_thread")]
17036 async fn supervisor_list_does_not_log_request_received() {
17037 let handler = ControlHandler::new(Arc::new(Registry::default()));
17038 let capture = EventCapture::default();
17039 let _subscriber =
17040 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17041 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
17042 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
17043
17044 handler
17045 .handle_client_control_request(
17046 &ctx,
17047 frame,
17048 ClientControlRequest::SupervisorList {},
17049 &mut None,
17050 )
17051 .await
17052 .unwrap();
17053
17054 assert!(capture.events().into_iter().all(|event| {
17055 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17056 }));
17057 }
17058
17059 #[tokio::test(flavor = "current_thread")]
17060 async fn supervisor_rescan_preview_does_not_log_request_received() {
17061 let handler = ControlHandler::new(Arc::new(Registry::default()));
17062 let capture = EventCapture::default();
17063 let _subscriber =
17064 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17065 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
17066 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
17067
17068 handler
17069 .handle_client_control_request(
17070 &ctx,
17071 frame,
17072 ClientControlRequest::SupervisorRescan { preview: true },
17073 &mut None,
17074 )
17075 .await
17076 .unwrap();
17077
17078 assert!(capture.events().into_iter().all(|event| {
17079 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17080 }));
17081 }
17082
17083 struct ApproveOperator;
17084 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
17085 fn prompt(
17086 &self,
17087 _: &str,
17088 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
17089 ) -> crate::operator_confirm::ProviderResult {
17090 crate::operator_confirm::ProviderResult::Approved
17091 }
17092 }
17093
17094 #[tokio::test]
17095 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
17096 assert_eq!(
17097 ControlHandler::default().privileged_operator_prompt,
17098 PrivilegedOperatorPrompt::Log
17099 );
17100 for (initial, changed, running) in [
17101 ("log", "enforce", PrivilegedOperatorPrompt::Log),
17102 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
17103 ] {
17104 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
17105 let path = dir.join("subc.jsonc");
17106 std::fs::write(
17107 &path,
17108 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
17109 )
17110 .unwrap();
17111 let config = crate::daemon_config::load(&path).unwrap().unwrap();
17112 let registry = Arc::new(Registry::default());
17113 let handle = SupervisorHandle::new();
17114 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
17115 .with_handle(handle.clone());
17116 let handler = ControlHandler::new(registry)
17117 .with_supervisor(handle)
17118 .with_privileged_operator_prompt(config.privileged_operator_prompt)
17119 .with_supervisor_rescan(supervisor, &path, config.port);
17120 handler.forwarding.operator_confirms().configure(
17121 Arc::new(ApproveOperator),
17122 crate::operator_confirm::OperatorLimits::default(),
17123 );
17124 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
17125
17126 for (disk_mode, needs_restart) in [
17127 (Some(initial), false),
17128 (Some(changed), true),
17129 (Some(changed), true),
17130 (None, running == PrivilegedOperatorPrompt::Enforce),
17131 (Some(initial), false),
17132 ] {
17133 let mut doc = json!({"version":1});
17134 if let Some(mode) = disk_mode {
17135 doc["privileged_operator_prompt"] = json!(mode);
17136 }
17137 std::fs::write(&path, doc.to_string()).unwrap();
17138 for preview in [true, false] {
17139 let request = ClientControlRequest::SupervisorRescan { preview };
17140 let frame = Frame::build(
17141 FrameType::Request,
17142 control_flags(),
17143 0,
17144 0,
17145 1,
17146 serde_json::to_vec(&request).unwrap(),
17147 )
17148 .unwrap();
17149 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17150 assert_eq!(frames.len(), 1);
17151 let ClientControlResponse::SupervisorRescan { result } =
17152 serde_json::from_slice(&frames[0].body).unwrap()
17153 else {
17154 panic!("expected a rescan response");
17155 };
17156 let expected = if needs_restart {
17157 vec!["privileged_operator_prompt"]
17158 } else {
17159 Vec::new()
17160 };
17161 assert_eq!(result.restart_required, expected);
17162 assert_eq!(handler.privileged_operator_prompt, running);
17163 assert_eq!(
17164 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
17165 running
17166 );
17167 }
17168 }
17169 }
17170 }
17171
17172 #[tokio::test]
17173 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
17174 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
17175 let path = dir.join("subc.jsonc");
17176 let registry = Arc::new(Registry::default());
17177 let handle = SupervisorHandle::new();
17178 let supervisor =
17179 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
17180 let handler = ControlHandler::new(registry)
17181 .with_supervisor(handle)
17182 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
17183 .with_supervisor_rescan(supervisor, &path, None);
17184 handler.forwarding.operator_confirms().configure(
17185 Arc::new(ApproveOperator),
17186 crate::operator_confirm::OperatorLimits::default(),
17187 );
17188 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
17189 for preview in [true, false] {
17190 std::fs::write(
17191 &path,
17192 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
17193 )
17194 .unwrap();
17195 let request = ClientControlRequest::SupervisorRescan { preview };
17196 let frame = Frame::build(
17197 FrameType::Request,
17198 control_flags(),
17199 0,
17200 0,
17201 1,
17202 serde_json::to_vec(&request).unwrap(),
17203 )
17204 .unwrap();
17205 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17206 assert_eq!(frames.len(), 1);
17207 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
17208 assert_eq!(
17209 handler.privileged_operator_prompt,
17210 PrivilegedOperatorPrompt::Enforce
17211 );
17212 }
17213 }
17214
17215 #[tokio::test]
17220 async fn route_open_refusal_names_the_check_that_refused() {
17221 let handler = ControlHandler::new(Arc::new(Registry::default()));
17222 let capture = EventCapture::default();
17223 let _subscriber =
17224 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17225 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17226 let response = handler
17227 .handle_control_frame(
17228 &ctx,
17229 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
17230 )
17231 .await
17232 .unwrap();
17233
17234 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17235 let event = capture
17236 .events()
17237 .into_iter()
17238 .find(|event| {
17239 event.target == "control"
17240 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17241 })
17242 .expect("route.open refusal event");
17243 assert_eq!(
17244 event.fields.get("reason"),
17245 Some(&"\"not_registered\"".to_string())
17246 );
17247 }
17248
17249 #[tokio::test]
17250 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
17251 let registry = Arc::new(Registry::default());
17252 let supervisor_handle = SupervisorHandle::new();
17253 let supervisor =
17254 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17255 .with_handle(supervisor_handle.clone())
17256 .with_connection_file_path(std::env::temp_dir().join(format!(
17257 "subc-route-open-refusal-info-{}",
17258 std::process::id()
17259 )));
17260 let module = supervisor
17261 .supervise_configured(
17262 ModuleSpec {
17263 script: None,
17264 carry_containment: false,
17265 module_id: "warming".to_string(),
17266 program: fake_aft_stub_path(),
17267 args: Vec::new(),
17268 env: Vec::new(),
17269 reserved: false,
17270 reserved_prefixes: Vec::new(),
17271 protocol: ModuleProtocol::Subc,
17272 overlap: Default::default(),
17273 },
17274 true,
17275 )
17276 .unwrap();
17277 assert_eq!(module.state().unwrap(), ModuleState::Running);
17278
17279 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17280 assert!(handler
17281 .counters()
17282 .snapshot()
17283 .get("route_open_refused_by_code")
17284 .is_none());
17285 let capture = EventCapture::default();
17286 let _subscriber =
17287 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17288 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
17289 let response = handler
17290 .handle_control_frame(
17291 &ctx,
17292 route_open_frame(394, "warming", unique_project_root("refusal-info")),
17293 )
17294 .await
17295 .unwrap();
17296 module.stop().await.unwrap();
17297
17298 assert_eq!(parse_error(&response[0])["code"], "module_warming");
17299 let event = capture
17300 .events()
17301 .into_iter()
17302 .find(|event| {
17303 event.target == "control"
17304 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
17305 })
17306 .expect("route.open refusal event");
17307 assert_eq!(
17308 event.fields.get("module_id"),
17309 Some(&"\"warming\"".to_string())
17310 );
17311 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
17312 assert_eq!(
17313 event.fields.get("reason"),
17314 Some(&"\"supervised_not_registered\"".to_string())
17315 );
17316 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
17317 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
17318 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
17319 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
17320 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
17321 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
17322 assert_eq!(
17323 handler.counters().snapshot()["route_open_refused_by_code"],
17324 json!({ "module_warming": 1 })
17325 );
17326 }
17327
17328 const OUTAGE_START: &str = "route.open refusing module: not serving";
17329 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
17330
17331 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
17332 capture
17333 .events()
17334 .into_iter()
17335 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
17336 .collect()
17337 }
17338
17339 fn supervise_stub(
17340 registry: &Arc<Registry>,
17341 module_id: &str,
17342 enabled: bool,
17343 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
17344 let supervisor_handle = SupervisorHandle::new();
17345 let supervisor =
17346 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
17347 .with_handle(supervisor_handle.clone())
17348 .with_connection_file_path(std::env::temp_dir().join(format!(
17349 "subc-route-outage-{module_id}-{}",
17350 std::process::id()
17351 )));
17352 let module = supervisor
17353 .supervise_configured(
17354 ModuleSpec {
17355 script: None,
17356 carry_containment: false,
17357 module_id: module_id.to_string(),
17358 program: fake_aft_stub_path(),
17359 args: Vec::new(),
17360 env: Vec::new(),
17361 reserved: false,
17362 reserved_prefixes: Vec::new(),
17363 protocol: ModuleProtocol::Subc,
17364 overlap: Default::default(),
17365 },
17366 enabled,
17367 )
17368 .unwrap();
17369 (supervisor_handle, module)
17370 }
17371
17372 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
17373 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
17374 module_id: module_id.to_string(),
17375 drain_timeout_ms: Some(50),
17376 })
17377 .unwrap();
17378 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
17379 }
17380
17381 #[test]
17385 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
17386 let registry = Arc::new(Registry::default());
17387 let forwarding = Arc::new(ForwardingTable::default());
17388 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17389 let second = ControlHandler::with_forwarding(registry, forwarding);
17390 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
17391 }
17392
17393 #[tokio::test(flavor = "current_thread")]
17397 async fn route_open_unknown_module_refusals_add_no_outage_state() {
17398 let handler = ControlHandler::new(Arc::new(Registry::default()));
17399 let capture = EventCapture::default();
17400 let _subscriber =
17401 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17402 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
17403 for corr in 0..8 {
17404 let response = handler
17405 .handle_control_frame(
17406 &ctx,
17407 route_open_frame(
17408 380 + corr,
17409 &format!("nobody-{corr}"),
17410 unique_project_root("outage-unknown"),
17411 ),
17412 )
17413 .await
17414 .unwrap();
17415 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17416 }
17417
17418 assert_eq!(handler.route_outages.tracked_module_count(), 0);
17419 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
17420 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
17421 }
17422
17423 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17428 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
17429 let registry = Arc::new(Registry::default());
17430 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
17431 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17432 let capture = EventCapture::default();
17433 let _subscriber =
17434 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17435 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
17436 handler.route_outages.record_accepted("outage-restart");
17439
17440 let response = handler
17441 .handle_control_frame(
17442 &ctx,
17443 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
17444 )
17445 .await
17446 .unwrap();
17447 assert_eq!(response[0].header.ty, FrameType::Error);
17448 let starts = outage_lines(&capture, OUTAGE_START);
17449 assert_eq!(starts.len(), 1, "{starts:?}");
17450 assert_eq!(starts[0].level, tracing::Level::WARN);
17451 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
17452 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
17453 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
17454 handler.route_outages.record_accepted("outage-restart");
17455 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
17456
17457 let restart = handler
17458 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
17459 .await
17460 .unwrap();
17461 assert_eq!(
17462 restart[0].header.ty,
17463 FrameType::Response,
17464 "{:?}",
17465 parse_error(&restart[0])
17466 );
17467 handler
17468 .handle_control_frame(
17469 &ctx,
17470 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
17471 )
17472 .await
17473 .unwrap();
17474 module.stop().await.unwrap();
17475
17476 let starts = outage_lines(&capture, OUTAGE_START);
17477 assert_eq!(starts.len(), 2, "{starts:?}");
17478 assert_eq!(starts[1].level, tracing::Level::INFO);
17479 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
17480 }
17481
17482 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17486 async fn failed_operator_restart_leaves_no_operator_mark() {
17487 let registry = Arc::new(Registry::default());
17488 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
17489 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17490 let capture = EventCapture::default();
17491 let _subscriber =
17492 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17493 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
17494 handler.route_outages.record_accepted("outage-disabled");
17495
17496 let restart = handler
17497 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
17498 .await
17499 .unwrap();
17500 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
17501 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
17502
17503 handler
17504 .handle_control_frame(
17505 &ctx,
17506 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
17507 )
17508 .await
17509 .unwrap();
17510 let starts = outage_lines(&capture, OUTAGE_START);
17511 assert_eq!(starts.len(), 1, "{starts:?}");
17512 assert_eq!(starts[0].level, tracing::Level::WARN);
17513 }
17514
17515 #[tokio::test(flavor = "current_thread")]
17516 async fn route_open_unknown_module_escapes_target_module_id() {
17517 let handler = ControlHandler::new(Arc::new(Registry::default()));
17518 let capture = EventCapture::default();
17519 let _subscriber =
17520 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17521 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
17522 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17523 let response = handler
17524 .handle_control_frame(
17525 &ctx,
17526 route_open_frame(
17527 395,
17528 hostile_module_id,
17529 unique_project_root("hostile-target-module-id"),
17530 ),
17531 )
17532 .await
17533 .unwrap();
17534
17535 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17536 let event = capture
17537 .events()
17538 .into_iter()
17539 .find(|event| {
17540 event.target == "control"
17541 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17542 })
17543 .expect("route.open unknown-module refusal event");
17544 let logged = event.fields.get("module_id").expect("module_id field");
17545 assert!(!logged.bytes().any(|byte| byte < 0x20));
17546 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17547 }
17548
17549 #[tokio::test(flavor = "current_thread")]
17552 async fn route_open_refusal_carries_daemon_origin() {
17553 let handler = ControlHandler::new(Arc::new(Registry::default()));
17554 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17555 let response = handler
17556 .handle_control_frame(
17557 &ctx,
17558 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
17559 )
17560 .await
17561 .unwrap();
17562
17563 assert_eq!(response.len(), 1);
17564 assert_eq!(response[0].header.ty, FrameType::Error);
17565 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17566 assert!(
17567 response[0].header.flags.is_daemon_origin(),
17568 "route.open refusal must carry daemon origin: {:?}",
17569 response[0].header.flags
17570 );
17571 }
17572
17573 #[tokio::test(flavor = "current_thread")]
17577 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
17578 let registry = Arc::new(Registry::default());
17579 let forwarding = Arc::new(ForwardingTable::default());
17580 let handler =
17581 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17582 let module_connection = ConnectionId::new(95);
17583 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17584 hello_via_sink(
17585 &handler,
17586 &module_ctx,
17587 &mut module_rx,
17588 hello_frame("aft", PROTOCOL_VERSION, 395),
17589 )
17590 .await;
17591 forwarding
17592 .begin_module_drain("aft", RouteCloseReason::Restart)
17593 .unwrap()
17594 .expect("the module endpoint drains");
17595
17596 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
17597 let response = handler
17598 .handle_control_frame(
17599 &client_ctx,
17600 route_open_frame(396, "aft", unique_project_root("drain-origin")),
17601 )
17602 .await
17603 .unwrap();
17604
17605 assert_eq!(response.len(), 1);
17606 assert_eq!(response[0].header.ty, FrameType::Error);
17607 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17608 assert!(
17609 response[0].header.flags.is_daemon_origin(),
17610 "drain module_reloading must carry daemon origin: {:?}",
17611 response[0].header.flags
17612 );
17613 }
17614
17615 #[tokio::test(flavor = "current_thread")]
17619 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
17620 let registry = Arc::new(Registry::default());
17621 let forwarding = Arc::new(ForwardingTable::default());
17622 let handler =
17623 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17624 let module_connection = ConnectionId::new(95);
17625 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17626 hello_via_sink(
17627 &handler,
17628 &module_ctx,
17629 &mut module_rx,
17630 hello_frame("aft", PROTOCOL_VERSION, 395),
17631 )
17632 .await;
17633
17634 let client_connection = ConnectionId::new(96);
17635 let (client_ctx, _client_rx) = route_ctx(client_connection);
17636 let (route_task, bind) = relay_route_open(
17637 &handler,
17638 client_connection,
17639 &client_ctx.egress,
17640 &mut module_rx,
17641 396,
17642 "aft",
17643 "spoofed-reloading",
17644 )
17645 .await;
17646 let rejection = Frame::build(
17649 FrameType::Error,
17650 crate::router::daemon_error_flags(),
17651 0,
17652 0,
17653 bind.header.corr,
17654 serde_json::to_vec(&ErrorBody::new(
17655 "module_reloading",
17656 "module refused route.bind",
17657 ))
17658 .unwrap(),
17659 )
17660 .unwrap();
17661 handler
17662 .handle_control_frame(&module_ctx, rejection)
17663 .await
17664 .unwrap();
17665
17666 let response = route_task.await.unwrap();
17667 assert_eq!(response.len(), 1);
17668 assert_eq!(response[0].header.ty, FrameType::Error);
17669 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17670 assert!(
17671 !response[0].header.flags.is_daemon_origin(),
17672 "a module's own rejection must not carry daemon origin: {:?}",
17673 response[0].header.flags
17674 );
17675 }
17676
17677 #[tokio::test(flavor = "current_thread")]
17678 async fn route_open_module_rejection_uses_daemon_counter_key() {
17679 let registry = Arc::new(Registry::default());
17680 let forwarding = Arc::new(ForwardingTable::default());
17681 let handler =
17682 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17683 let module_connection = ConnectionId::new(95);
17684 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17685 hello_via_sink(
17686 &handler,
17687 &module_ctx,
17688 &mut module_rx,
17689 hello_frame("aft", PROTOCOL_VERSION, 395),
17690 )
17691 .await;
17692
17693 let client_connection = ConnectionId::new(96);
17694 let (client_ctx, _client_rx) = route_ctx(client_connection);
17695 let capture = EventCapture::default();
17696 static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
17703 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
17704 let _subscriber =
17705 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17706 let (route_task, bind) = relay_route_open(
17707 &handler,
17708 client_connection,
17709 &client_ctx.egress,
17710 &mut module_rx,
17711 396,
17712 "aft",
17713 "hostile-module-code",
17714 )
17715 .await;
17716 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
17717 let rejection = Frame::build(
17718 FrameType::Error,
17719 control_flags(),
17720 0,
17721 0,
17722 bind.header.corr,
17723 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
17724 )
17725 .unwrap();
17726 handler
17727 .handle_control_frame(&module_ctx, rejection)
17728 .await
17729 .unwrap();
17730
17731 let response = route_task.await.unwrap();
17732 assert_eq!(parse_error(&response[0])["code"], hostile_code);
17733 let counters = handler.counters().snapshot();
17734 assert_eq!(
17735 counters["route_open_refused_by_code"],
17736 json!({ "module_rejected": 1 })
17737 );
17738 assert!(counters["route_open_refused_by_code"]
17739 .get(hostile_code)
17740 .is_none());
17741
17742 let event = capture
17743 .events()
17744 .into_iter()
17745 .find(|event| {
17746 event.target == "control"
17747 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
17748 })
17749 .expect("route.open module-rejection refusal event");
17750 let logged = event.fields.get("module_code").expect("module_code field");
17751 assert!(!logged.bytes().any(|byte| byte < 0x20));
17752 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17753 }
17754
17755 #[tokio::test]
17756 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
17757 let registry = Arc::new(Registry::default());
17758 let supervisor_handle = SupervisorHandle::new();
17759 let missing_program = std::env::temp_dir().join(format!(
17760 "subc-route-open-missing-program-{}",
17761 std::process::id()
17762 ));
17763 let supervisor =
17764 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17765 .with_handle(supervisor_handle.clone());
17766 let module = supervisor
17767 .supervise_configured(
17768 ModuleSpec {
17769 script: None,
17770 carry_containment: false,
17771 module_id: "failed".to_string(),
17772 program: missing_program,
17773 args: Vec::new(),
17774 env: Vec::new(),
17775 reserved: false,
17776 reserved_prefixes: Vec::new(),
17777 protocol: ModuleProtocol::Subc,
17778 overlap: Default::default(),
17779 },
17780 true,
17781 )
17782 .unwrap();
17783 assert_eq!(module.state().unwrap(), ModuleState::Failed);
17784
17785 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17786 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
17787 let response = handler
17788 .handle_control_frame(
17789 &ctx,
17790 route_open_frame(305, "failed", unique_project_root("failed")),
17791 )
17792 .await
17793 .unwrap();
17794
17795 assert_eq!(response[0].header.ty, FrameType::Error);
17796 let error = parse_error(&response[0]);
17797 assert_eq!(error["code"], "target_unavailable");
17798 assert!(error["message"]
17799 .as_str()
17800 .unwrap()
17801 .contains("state=failed, enabled=true, live=false"));
17802 }
17803
17804 #[tokio::test]
17805 async fn route_open_role_mismatch_remains_target_unavailable() {
17806 let registry = Arc::new(Registry::default());
17807 let handler = ControlHandler::new(Arc::clone(®istry));
17808 handler
17809 .handle_control(
17810 ConnectionId::new(41),
17811 non_routable_hello_frame_with_control_ops("health-only", 306, None),
17812 )
17813 .unwrap();
17814
17815 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
17816 let response = handler
17817 .handle_control_frame(
17818 &ctx,
17819 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
17820 )
17821 .await
17822 .unwrap();
17823
17824 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17825 assert!(parse_error(&response[0])["message"]
17826 .as_str()
17827 .unwrap()
17828 .contains("does not provide the requested target"));
17829 }
17830
17831 #[tokio::test]
17832 async fn route_open_inactive_registration_remains_target_unavailable() {
17833 let registry = Arc::new(Registry::default());
17834 let handler = ControlHandler::new(Arc::clone(®istry));
17835 handler
17836 .handle_control(
17837 ConnectionId::new(43),
17838 hello_frame("inactive", PROTOCOL_VERSION, 308),
17839 )
17840 .unwrap();
17841 assert!(registry
17842 .set_module_state_for_test("inactive", ChannelState::Closed)
17843 .unwrap());
17844
17845 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
17846 let response = handler
17847 .handle_control_frame(
17848 &ctx,
17849 route_open_frame(309, "inactive", unique_project_root("inactive")),
17850 )
17851 .await
17852 .unwrap();
17853
17854 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17855 assert!(parse_error(&response[0])["message"]
17856 .as_str()
17857 .unwrap()
17858 .contains("is not active"));
17859 }
17860
17861 #[tokio::test]
17862 async fn late_health_reply_is_recorded_through_the_module_response_path() {
17863 let registry = Arc::new(Registry::default());
17864 let forwarding = Arc::new(ForwardingTable::default());
17865 let supervisor_handle = SupervisorHandle::new();
17866 let supervisor =
17867 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
17868 .with_forwarding(Arc::clone(&forwarding))
17869 .with_handle(supervisor_handle.clone());
17870 let module = supervisor
17871 .supervise_configured(
17872 crate::ModuleSpec {
17873 script: None,
17874 carry_containment: false,
17875 module_id: "late-health-response".to_string(),
17876 program: PathBuf::from("disabled-module"),
17877 args: Vec::new(),
17878 env: Vec::new(),
17879 reserved: false,
17880 reserved_prefixes: Vec::new(),
17881 protocol: ModuleProtocol::Subc,
17882 overlap: Default::default(),
17883 },
17884 false,
17885 )
17886 .unwrap();
17887 let handler =
17888 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17889 .with_supervisor(supervisor_handle);
17890 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
17891 handler
17892 .handle_control_frame(
17893 &module_ctx,
17894 hello_frame_with_control_ops(
17895 "late-health-response",
17896 PROTOCOL_VERSION,
17897 7,
17898 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17899 ),
17900 )
17901 .await
17902 .unwrap();
17903 let probe_started_at = Instant::now() - Duration::from_millis(80);
17904 let pending = forwarding
17905 .begin_health_probe_rpc_for(
17906 "late-health-response",
17907 MODULE_CONTROL_OP_HEALTH_CHECK,
17908 probe_started_at,
17909 Instant::now() - Duration::from_millis(1),
17910 )
17911 .unwrap();
17912 assert!(forwarding
17913 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
17914 .unwrap());
17915
17916 let responses = handler
17917 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
17918 .await
17919 .unwrap();
17920
17921 assert!(responses.is_empty());
17922 let health = module.status().unwrap().health;
17923 assert_eq!(health.late_answer_count, 1);
17924 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
17925 }
17926
17927 #[tokio::test]
17928 async fn health_probe_timeout_and_module_death_are_typed() {
17929 let registry = Arc::new(Registry::default());
17930 let forwarding = Arc::new(ForwardingTable::default());
17931 let handler =
17932 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17933 .with_health_probe_timeout(Duration::from_millis(50));
17934 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
17935 hello_via_sink(
17936 &handler,
17937 &module_ctx,
17938 &mut module_rx,
17939 hello_frame_with_control_ops(
17940 "aft",
17941 PROTOCOL_VERSION,
17942 7,
17943 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17944 ),
17945 )
17946 .await;
17947
17948 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
17949 let responses = handler
17950 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
17951 .await
17952 .unwrap();
17953 assert_eq!(responses[0].header.ty, FrameType::Error);
17954 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
17955 let _ = module_rx.try_recv();
17956
17957 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
17958 let health_handler = handler.clone();
17959 let death_task = tokio::spawn(async move {
17960 health_handler
17961 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
17962 .await
17963 .unwrap()
17964 });
17965 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
17966 .await
17967 .unwrap()
17968 .unwrap();
17969 handler
17970 .cleanup_connection(module_ctx.connection_id)
17971 .unwrap();
17972 let responses = death_task.await.unwrap();
17973 assert_eq!(responses[0].header.ty, FrameType::Error);
17974 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
17975 }
17976
17977 #[test]
17978 fn hello_requires_exact_protocol_version() {
17979 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
17980 let registry = Arc::new(Registry::default());
17981 let handler = ControlHandler::new(Arc::clone(®istry));
17982 let responses = handler
17983 .handle_control(
17984 ConnectionId::new(connection),
17985 hello_frame("aft", offered, 9),
17986 )
17987 .unwrap();
17988
17989 assert_eq!(responses.len(), 1);
17990 assert_eq!(responses[0].header.ty, FrameType::Error);
17991 let error = parse_error(&responses[0]);
17992 assert_eq!(error["code"], "version_unsupported");
17993 assert!(registry.get_module("aft").unwrap().is_none());
17994 assert_eq!(registry.active_registration_count().unwrap(), 0);
17995 }
17996 }
17997
17998 #[test]
17999 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
18000 let registry = Arc::new(Registry::default());
18001 let forwarding = Arc::new(ForwardingTable::default());
18002 let handler =
18003 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18004 let module_connection = ConnectionId::new(301);
18005 let registration = registry
18006 .register_with_control_ops(
18007 manifest("aft-push", PROTOCOL_VERSION),
18008 PROTOCOL_VERSION,
18009 module_connection,
18010 module_baseline_control_ops(),
18011 )
18012 .unwrap();
18013 let (module_tx, _module_rx) = mpsc::channel(8);
18014 let endpoint = forwarding
18015 .register_module_connection(
18016 module_connection,
18017 "aft-push".to_string(),
18018 PROTOCOL_VERSION,
18019 manifest_concurrency(®istration.manifest),
18020 FrameSink::new(module_tx),
18021 )
18022 .unwrap();
18023
18024 let unknown = Frame::build(
18026 FrameType::Push,
18027 control_flags(),
18028 0,
18029 0,
18030 5,
18031 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
18032 )
18033 .unwrap();
18034 let out = handler.handle_status_update(endpoint, unknown).unwrap();
18035 assert!(
18036 out.is_empty(),
18037 "unknown push op must be ignored, got {out:?}"
18038 );
18039
18040 let malformed = Frame::build(
18042 FrameType::Push,
18043 control_flags(),
18044 0,
18045 0,
18046 6,
18047 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
18048 )
18049 .unwrap();
18050 let out = handler.handle_status_update(endpoint, malformed).unwrap();
18051 assert_eq!(out.len(), 1);
18052 assert_eq!(out[0].header.ty, FrameType::Error);
18053 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
18054 }
18055
18056 #[test]
18057 fn hello_rejected_when_connection_already_owns_client_routes() {
18058 let registry = Arc::new(Registry::default());
18059 let forwarding = Arc::new(ForwardingTable::default());
18060 let handler =
18061 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18062 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
18064 let client_connection = ConnectionId::new(202);
18065
18066 let responses = handler
18069 .handle_control(
18070 client_connection,
18071 hello_frame("aft-second", PROTOCOL_VERSION, 9),
18072 )
18073 .unwrap();
18074 assert_eq!(responses[0].header.ty, FrameType::Error);
18075 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
18076 assert!(registry.get_module("aft-second").unwrap().is_none());
18077 }
18078
18079 #[tokio::test]
18080 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
18081 let registry = Arc::new(Registry::default());
18082 let forwarding = Arc::new(ForwardingTable::default());
18083 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
18084 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
18085 hello_via_sink(
18086 &handler,
18087 &module_ctx,
18088 &mut module_rx,
18089 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
18090 )
18091 .await;
18092 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
18093 let pending = forwarding
18094 .begin_route_bind_relay_for_test(
18095 client_ctx.connection_id,
18096 client_ctx.egress.clone(),
18097 2,
18098 "alpha",
18099 )
18100 .unwrap();
18101 forwarding
18102 .complete_pending_relay(
18103 module_ctx.connection_id,
18104 pending.corr,
18105 RouteBindRelayOutcome::Accepted,
18106 )
18107 .unwrap();
18108 client_rx.try_recv().unwrap();
18109 for module_id in ["beta", "alpha"] {
18110 let replies = handler
18111 .handle_control_frame(
18112 &module_ctx,
18113 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
18114 )
18115 .await
18116 .unwrap();
18117 assert_eq!(replies.len(), 1, "second HELLO must be refused");
18118 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
18119 }
18120 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
18121 assert!(registry.get_module("beta").unwrap().is_none());
18122 assert!(matches!(
18123 forwarding
18124 .lookup_data_route(
18125 client_ctx.connection_id,
18126 pending.client_channel,
18127 pending.client_epoch,
18128 )
18129 .unwrap(),
18130 DataRoute::Client(DataRouteState::Bound(_))
18131 ));
18132 assert!(handler
18133 .hello_launch_nonces
18134 .lock()
18135 .unwrap()
18136 .presented(module_ctx.connection_id, Some("alpha-nonce")));
18137 assert!(module_rx.try_recv().is_err());
18138 }
18139
18140 #[test]
18141 fn reserved_module_hello_requires_matching_launch_nonce() {
18142 let registry = Arc::new(Registry::default());
18143 let supervisor = SupervisorHandle::new();
18144 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
18147 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18148
18149 let no_nonce = handler
18151 .handle_control(
18152 ConnectionId::new(1),
18153 hello_frame("vault", PROTOCOL_VERSION, 1),
18154 )
18155 .unwrap();
18156 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
18157 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
18158 assert!(registry.get_module("vault").unwrap().is_none());
18159
18160 let wrong = handler
18162 .handle_control(
18163 ConnectionId::new(2),
18164 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
18165 )
18166 .unwrap();
18167 assert_eq!(wrong[0].header.ty, FrameType::Error);
18168 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
18169 assert!(registry.get_module("vault").unwrap().is_none());
18170
18171 let ok = handler
18173 .handle_control(
18174 ConnectionId::new(3),
18175 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
18176 )
18177 .unwrap();
18178 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
18179 assert!(registry.get_module("vault").unwrap().is_some());
18180 }
18181
18182 #[test]
18183 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
18184 let registry = Arc::new(Registry::default());
18185 let supervisor = SupervisorHandle::new();
18186 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18187 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18188 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18189
18190 let squat = handler
18191 .handle_control(
18192 ConnectionId::new(1),
18193 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
18194 )
18195 .unwrap();
18196 assert_eq!(squat[0].header.ty, FrameType::Error);
18197 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
18198 assert!(parse_error(&squat[0])["message"]
18199 .as_str()
18200 .unwrap()
18201 .contains("fed:"));
18202
18203 let accepted_peer = handler
18204 .handle_control(
18205 ConnectionId::new(2),
18206 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
18207 )
18208 .unwrap();
18209 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
18210
18211 let accepted_short = handler
18212 .handle_control(
18213 ConnectionId::new(3),
18214 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
18215 )
18216 .unwrap();
18217 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
18218
18219 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
18220 let response = handler
18221 .handle_control(
18222 ConnectionId::new(conn),
18223 hello_frame(module_id, PROTOCOL_VERSION, conn),
18224 )
18225 .unwrap();
18226 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
18227 }
18228 }
18229
18230 #[test]
18231 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
18232 let registry = Arc::new(Registry::default());
18233 let supervisor = SupervisorHandle::new();
18234 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18235 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18236 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
18237 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18238
18239 let owner_nonce = handler
18240 .handle_control(
18241 ConnectionId::new(1),
18242 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
18243 )
18244 .unwrap();
18245 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
18246 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
18247 assert!(registry.get_module("fed:special").unwrap().is_none());
18248
18249 let exact_nonce = handler
18250 .handle_control(
18251 ConnectionId::new(2),
18252 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
18253 )
18254 .unwrap();
18255 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
18256 assert!(registry.get_module("fed:special").unwrap().is_some());
18257 }
18258
18259 #[test]
18260 fn non_reserved_module_ignores_launch_nonce() {
18261 let registry = Arc::new(Registry::default());
18262 let handler = ControlHandler::new(Arc::clone(®istry));
18265 let no_nonce = handler
18266 .handle_control(
18267 ConnectionId::new(1),
18268 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
18269 )
18270 .unwrap();
18271 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
18272 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
18273
18274 let echoed_nonce = handler
18275 .handle_control(
18276 ConnectionId::new(2),
18277 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
18278 )
18279 .unwrap();
18280 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
18281 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
18282 }
18283
18284 #[test]
18285 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
18286 let handler = ControlHandler::default();
18287 let conn = ConnectionId::new(1);
18288 let malformed = Frame::build(
18289 FrameType::Hello,
18290 control_flags(),
18291 0,
18292 0,
18293 3,
18294 b"{not json".to_vec(),
18295 )
18296 .unwrap();
18297
18298 let error = handler.handle_control(conn, malformed).unwrap();
18299 assert_eq!(error[0].header.ty, FrameType::Error);
18300 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
18301
18302 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18303 let pong = handler.handle_control(conn, ping).unwrap();
18304 assert_eq!(pong[0].header.ty, FrameType::Pong);
18305 assert_eq!(pong[0].header.corr, 4);
18306 }
18307
18308 #[test]
18309 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
18310 let registry = Arc::new(Registry::default());
18311 let handler = ControlHandler::new(Arc::clone(®istry));
18312
18313 handler
18314 .handle_control(
18315 ConnectionId::new(1),
18316 hello_frame("aft", PROTOCOL_VERSION, 1),
18317 )
18318 .unwrap();
18319 let duplicate = handler
18320 .handle_control(
18321 ConnectionId::new(2),
18322 hello_frame("aft", PROTOCOL_VERSION, 2),
18323 )
18324 .unwrap();
18325
18326 assert_eq!(duplicate[0].header.ty, FrameType::Error);
18327 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18328 let registration = registry.get_module("aft").unwrap().unwrap();
18329 assert_eq!(registration.connection_id, ConnectionId::new(1));
18330 }
18331
18332 #[test]
18333 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
18334 let registry = Arc::new(Registry::default());
18335 let forwarding = Arc::new(ForwardingTable::default());
18336 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
18337 let handler =
18338 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18339 .with_process_liveness(process_liveness);
18340 let (ctx, route_channel, route_epoch) =
18341 bind_liveness_route(®istry, &forwarding, "aft-dead");
18342 let responses = handler
18343 .handle_route_poll(
18344 &ctx,
18345 route_poll_frame(41, PollKind::Liveness, route_channel),
18346 route_channel,
18347 route_epoch,
18348 PollKind::Liveness,
18349 )
18350 .unwrap();
18351
18352 assert_eq!(responses.len(), 1);
18353 assert_eq!(responses[0].header.ty, FrameType::Response);
18354 assert_route_poll_liveness(&responses[0], false);
18355 }
18356
18357 #[test]
18358 fn liveness_poll_without_process_source_uses_bound_route() {
18359 let registry = Arc::new(Registry::default());
18360 let forwarding = Arc::new(ForwardingTable::default());
18361 let handler =
18362 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18363 let (ctx, route_channel, route_epoch) =
18364 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
18365 let responses = handler
18366 .handle_route_poll(
18367 &ctx,
18368 route_poll_frame(42, PollKind::Liveness, route_channel),
18369 route_channel,
18370 route_epoch,
18371 PollKind::Liveness,
18372 )
18373 .unwrap();
18374
18375 assert_route_poll_liveness(&responses[0], true);
18376 }
18377
18378 #[test]
18379 fn liveness_poll_untracked_process_source_uses_bound_route() {
18380 let registry = Arc::new(Registry::default());
18381 let forwarding = Arc::new(ForwardingTable::default());
18382 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
18383 let handler =
18384 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18385 .with_process_liveness(process_liveness);
18386 let (ctx, route_channel, route_epoch) =
18387 bind_liveness_route(®istry, &forwarding, "aft-untracked");
18388 let responses = handler
18389 .handle_route_poll(
18390 &ctx,
18391 route_poll_frame(43, PollKind::Liveness, route_channel),
18392 route_channel,
18393 route_epoch,
18394 PollKind::Liveness,
18395 )
18396 .unwrap();
18397
18398 assert_route_poll_liveness(&responses[0], true);
18399 }
18400
18401 #[tokio::test]
18402 async fn unknown_op_returns_unknown_control_op() {
18403 let handler = ControlHandler::default();
18404 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
18405 let request = Frame::build(
18406 FrameType::Request,
18407 control_flags(),
18408 0,
18409 0,
18410 55,
18411 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
18412 )
18413 .unwrap();
18414
18415 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18416
18417 assert_eq!(response.len(), 1);
18418 assert_eq!(response[0].header.ty, FrameType::Error);
18419 assert_eq!(response[0].header.corr, 55);
18420 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
18421 }
18422
18423 #[tokio::test]
18424 async fn supervisor_provenance_rejects_unknown_exact_module() {
18425 let handler = ControlHandler::default();
18426 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
18427 let request = Frame::build(
18428 FrameType::Request,
18429 control_flags(),
18430 0,
18431 0,
18432 57,
18433 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
18434 )
18435 .unwrap();
18436
18437 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18438
18439 assert_eq!(response.len(), 1);
18440 assert_eq!(response[0].header.ty, FrameType::Error);
18441 assert_eq!(response[0].header.corr, 57);
18442 let error = parse_error(&response[0]);
18443 assert_eq!(error["code"], "unknown_module");
18444 assert_eq!(error["message"], "module_id 'missing' is not supervised");
18445 }
18446
18447 #[test]
18448 fn provenance_probe_override_keeps_handler_tests_deterministic() {
18449 let expected = subc_control::RunningImageAgreement::Unavailable {
18450 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18451 };
18452 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
18453 assert_eq!(handler.provenance_probe_override, Some(expected));
18454 }
18455
18456 #[test]
18457 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
18458 let verdict = reload_verdict(
18459 std::path::Path::new("/bin/new"),
18460 Some(std::path::Path::new("/bin/old")),
18461 subc_control::RunningImageAgreement::Unavailable {
18462 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18463 },
18464 );
18465 assert!(matches!(
18466 verdict.path,
18467 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
18468 if configured == std::path::Path::new("/bin/new")
18469 && spawned_from == std::path::Path::new("/bin/old")
18470 ));
18471 }
18472
18473 #[test]
18474 fn launch_spec_without_running_digest_is_unavailable_not_match() {
18475 use subc_control::{
18476 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
18477 };
18478 let configured = crate::supervise::ModuleSpec {
18479 script: None,
18480 carry_containment: false,
18481 module_id: "legacy".into(),
18482 program: "/module".into(),
18483 args: Vec::new(),
18484 env: Vec::new(),
18485 reserved: false,
18486 reserved_prefixes: Vec::new(),
18487 protocol: subc_control::ModuleProtocol::Subc,
18488 overlap: Default::default(),
18489 };
18490 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
18491 assert_eq!(
18492 launch_spec_agreement(&configured, false, Some(&digest)),
18493 Agreement::Unavailable {
18494 reason: Reason::NotRunning
18495 }
18496 );
18497 assert_eq!(
18498 launch_spec_agreement(&configured, true, None),
18499 Agreement::Unavailable {
18500 reason: Reason::SpawnedSpecUnavailable
18501 }
18502 );
18503 assert_eq!(
18504 launch_spec_agreement(&configured, true, Some(&digest)),
18505 Agreement::Match
18506 );
18507 }
18508
18509 #[cfg(windows)]
18510 #[test]
18511 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
18512 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
18513 let configured = dir.join("module.exe");
18514 std::fs::write(&configured, b"executable path fixture").unwrap();
18515 let spawned = std::fs::canonicalize(&configured).unwrap();
18516 assert_ne!(
18517 configured, spawned,
18518 "fixture must exercise different path spellings"
18519 );
18520 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
18521 let lower_case = configured.to_string_lossy().to_lowercase();
18522 for spelling in [
18523 &configured,
18524 Path::new(&forward_slashes),
18525 Path::new(&lower_case),
18526 ] {
18527 let verdict = reload_verdict(
18528 spelling,
18529 Some(&spawned),
18530 subc_control::RunningImageAgreement::Unavailable {
18531 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18532 },
18533 );
18534 assert_eq!(
18535 verdict.path,
18536 ReloadPathAgreement::Match,
18537 "configured {spelling:?}, spawned {spawned:?}"
18538 );
18539 }
18540 }
18541
18542 #[cfg(windows)]
18543 #[test]
18544 fn windows_reload_path_keeps_different_existing_program_pending() {
18545 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
18546 let old = dir.join("old.exe");
18547 let new = dir.join("new.exe");
18548 std::fs::write(&old, b"same executable content").unwrap();
18549 std::fs::copy(&old, &new).unwrap();
18550 let spawned = std::fs::canonicalize(&old).unwrap();
18551 let image = subc_control::RunningImageAgreement::Unavailable {
18552 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18553 };
18554 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
18555 assert!(
18556 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
18557 if configured == new && spawned_from == spawned)
18558 );
18559 let missing = dir.join("missing.exe");
18560 assert!(matches!(
18561 reload_verdict(&missing, Some(&spawned), image).path,
18562 ReloadPathAgreement::Mismatch { .. }
18563 ));
18564 }
18565
18566 #[test]
18567 fn reload_verdict_detects_replaced_image_at_same_path() {
18568 let image = subc_control::RunningImageAgreement::Mismatch {
18569 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
18570 digest: "old".into(),
18571 },
18572 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
18573 digest: "new".into(),
18574 },
18575 };
18576 let verdict = reload_verdict(
18577 std::path::Path::new("/bin/same"),
18578 Some(std::path::Path::new("/bin/same")),
18579 image.clone(),
18580 );
18581 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18582 assert_eq!(verdict.image, image);
18583 }
18584
18585 #[test]
18586 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
18587 let image = subc_control::RunningImageAgreement::Unavailable {
18588 reason: subc_control::RunningImageUnavailableReason::NotRunning,
18589 };
18590 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
18591 assert_eq!(
18592 verdict.path,
18593 subc_control::ReloadPathAgreement::Unavailable {
18594 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
18595 }
18596 );
18597 assert_eq!(verdict.image, image);
18598
18599 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
18600 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18601 };
18602 let verdict = reload_verdict(
18603 std::path::Path::new("/bin/same"),
18604 Some(std::path::Path::new("/bin/same")),
18605 unconfirmed.clone(),
18606 );
18607 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18608 assert_eq!(verdict.image, unconfirmed);
18609 }
18610
18611 #[test]
18612 fn reload_verdict_preserves_each_image_unavailability_reason() {
18613 use subc_control::RunningImageUnavailableReason as Reason;
18614
18615 for reason in [
18616 Reason::NotRunning,
18617 Reason::UnsupportedPlatform,
18618 Reason::RunningExecutableUnreadable,
18619 Reason::SpawnedPathUnreadable,
18620 Reason::HashFailed,
18621 Reason::ProcessIdentityUnconfirmed,
18622 Reason::Unknown("future_probe_reason".to_string()),
18623 ] {
18624 let image = subc_control::RunningImageAgreement::Unavailable {
18625 reason: reason.clone(),
18626 };
18627 let verdict = reload_verdict(
18628 std::path::Path::new("/bin/same"),
18629 Some(std::path::Path::new("/bin/same")),
18630 image.clone(),
18631 );
18632 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18633 assert_eq!(verdict.image, image, "{reason:?}");
18634 }
18635 }
18636
18637 #[tokio::test]
18638 async fn malformed_control_bodies_return_invalid_control_body() {
18639 let handler = ControlHandler::default();
18640 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
18641
18642 for (corr, body) in [
18643 (56, br#"{"route_channel":1}"#.as_slice()),
18644 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
18645 (
18646 58,
18647 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
18648 ),
18649 ] {
18650 let request = Frame::build(
18651 FrameType::Request,
18652 control_flags(),
18653 0,
18654 0,
18655 corr,
18656 body.to_vec(),
18657 )
18658 .unwrap();
18659 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18660
18661 assert_eq!(response.len(), 1);
18662 assert_eq!(response[0].header.ty, FrameType::Error);
18663 assert_eq!(response[0].header.corr, corr);
18664 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
18665 }
18666 }
18667
18668 #[tokio::test]
18669 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
18670 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18671 let registry = Arc::new(Registry::default());
18672 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18673 let router = Router::with_control_handler(Arc::clone(&control));
18674 let connection = router.begin_connection();
18675 let (ctx, mut rx) = route_ctx(connection.id());
18676
18677 router
18678 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
18679 .await
18680 .unwrap();
18681 let response = rx.recv().await.unwrap();
18682 let ack = parse_ack(&response);
18683 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18684 let channel = 1;
18685
18686 let goodbye =
18687 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
18688 router.route_for_connection(&ctx, goodbye).await.unwrap();
18689 assert!(rx.try_recv().is_err());
18690 assert!(registry.get_module("aft").unwrap().is_none());
18691
18692 router
18693 .route_for_connection(&ctx, channel_request(channel, 13))
18694 .await
18695 .unwrap();
18696 let error_frame = rx.recv().await.unwrap();
18697 assert_eq!(error_frame.header.ty, FrameType::Error);
18698 assert_eq!(error_frame.header.channel, channel);
18699 let captured = crate::router::test_log::captured_logs(&logs);
18700 assert_eq!(
18701 captured
18702 .lines()
18703 .filter(|line| {
18704 line.contains("module_id=aft")
18705 && line.contains("reason=explicit_goodbye")
18706 && line.contains("module registration ended")
18707 })
18708 .count(),
18709 1,
18710 "unexpected GOODBYE registry log: {captured}"
18711 );
18712 }
18713
18714 #[tokio::test]
18715 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
18716 let registry = Arc::new(Registry::default());
18717 let handler = ControlHandler::new(registry).with_capability_config(
18718 [("prov".to_string(), true), ("cons".to_string(), true)],
18719 BTreeMap::new(),
18720 );
18721 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
18722 register_capability_manifest(
18723 &handler,
18724 &provider_ctx,
18725 &mut provider_rx,
18726 capability_manifest("prov", &["thing/v1"], &[]),
18727 1,
18728 )
18729 .await;
18730 let mut consumer = capability_manifest("cons", &[], &[]);
18731 consumer.capabilities.as_mut().unwrap().requires.push(
18732 subc_protocol::manifest::CapabilityRequirement {
18733 capability: "thing/v1".to_string(),
18734 need: subc_protocol::manifest::CapabilityNeed::Required,
18735 },
18736 );
18737 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
18738 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
18739 assert_eq!(
18740 handler.capability_evaluator.verdict("cons", "thing/v1"),
18741 Some(CapabilityVerdict::Provided)
18742 );
18743 let (mut client_rx, _) = open_route_for_capability_test(
18744 &handler,
18745 &provider_ctx,
18746 &mut provider_rx,
18747 703,
18748 3,
18749 "prov",
18750 None,
18751 )
18752 .await;
18753 let goodbye =
18754 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18755 handler
18756 .handle_control_frame(&provider_ctx, goodbye)
18757 .await
18758 .unwrap();
18759 assert_eq!(
18760 handler.capability_evaluator.verdict("cons", "thing/v1"),
18761 Some(CapabilityVerdict::NeverProvided)
18762 );
18763 let closed = client_rx
18764 .try_recv()
18765 .expect("GOODBYE pushes route.closed before route GOODBYE");
18766 assert!(
18767 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
18768 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
18769 );
18770 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
18771 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
18772 }
18773
18774 struct RouteClosedFixture {
18775 handler: ControlHandler,
18776 #[cfg(unix)]
18777 supervisor: Supervisor,
18778 module_ctx: RouteCtx,
18779 #[cfg(unix)]
18780 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18781 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18782 route: Arc<crate::forwarding::RouteBinding>,
18783 }
18784
18785 async fn route_closed_fixture() -> RouteClosedFixture {
18786 let registry = Arc::new(Registry::default());
18787 let handler = ControlHandler::new(Arc::clone(®istry));
18788 #[cfg(unix)]
18789 let supervisor = Supervisor::new(registry, RestartPolicy::default())
18790 .with_forwarding(Arc::clone(&handler.forwarding));
18791 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
18792 register_capability_manifest(
18793 &handler,
18794 &module_ctx,
18795 &mut module_rx,
18796 capability_manifest("prov", &[], &[]),
18797 1,
18798 )
18799 .await;
18800 let (client_rx, bind) = open_route_for_capability_test(
18801 &handler,
18802 &module_ctx,
18803 &mut module_rx,
18804 702,
18805 2,
18806 "prov",
18807 None,
18808 )
18809 .await;
18810 let ModuleControlRequest::RouteBind {
18811 route_channel: channel,
18812 epoch: route_epoch,
18813 ..
18814 } = bind
18815 else {
18816 panic!("expected route.bind");
18817 };
18818 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
18819 handler
18820 .forwarding
18821 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
18822 .unwrap()
18823 else {
18824 panic!("the acknowledged route must be bound");
18825 };
18826 RouteClosedFixture {
18827 handler,
18828 #[cfg(unix)]
18829 supervisor,
18830 module_ctx,
18831 #[cfg(unix)]
18832 module_rx,
18833 client_rx,
18834 route,
18835 }
18836 }
18837
18838 impl RouteClosedFixture {
18839 #[cfg(unix)]
18840 async fn cut(&mut self) {
18841 let notices = async {
18842 let mut draining = self.module_rx.recv().await.unwrap();
18843 assert!(matches!(
18844 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
18845 &draining.body
18846 )
18847 .unwrap(),
18848 subc_protocol::session::ModuleControlCommand::Draining {
18849 reason: RouteCloseReason::Disable,
18850 ..
18851 }
18852 ));
18853 draining.flushed.take().unwrap().send(()).unwrap();
18854 let mut closing = self.client_rx.recv().await.unwrap();
18855 assert!(matches!(
18856 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
18857 ClientControlPush::RouteClosing {
18858 reason: RouteCloseReason::Restart,
18859 ..
18860 }
18861 ));
18862 closing.flushed.take().unwrap().send(()).unwrap();
18863 };
18864 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
18865 result.unwrap();
18866 }
18867
18868 fn disconnect(&mut self) -> ClientControlPush {
18869 self.handler
18870 .cleanup_connection(self.module_ctx.connection_id)
18871 .unwrap();
18872 let closed = self.client_rx.try_recv().unwrap();
18873 assert_eq!(closed.header.ty, FrameType::Push);
18874 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
18875 assert!(matches!(
18876 &push,
18877 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
18878 if channels == &[self.route.client_channel]
18879 ));
18880 assert_eq!(
18881 self.client_rx.try_recv().unwrap().header.ty,
18882 FrameType::Goodbye
18883 );
18884 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
18885 push
18886 }
18887 }
18888
18889 #[cfg(unix)]
18890 #[tokio::test(start_paused = true)]
18891 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
18892 let mut fixture = route_closed_fixture().await;
18893 fixture.cut().await;
18894 assert!(matches!(
18895 fixture.disconnect(),
18896 ClientControlPush::RouteClosed {
18897 reason: RouteCloseReason::Restart,
18898 drained: true,
18899 ..
18900 }
18901 ));
18902 }
18903
18904 #[cfg(unix)]
18905 #[tokio::test(start_paused = true)]
18906 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
18907 let mut fixture = route_closed_fixture().await;
18908 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
18909 let started = tokio::time::Instant::now();
18910 fixture.cut().await;
18911 assert!(started.elapsed() >= Duration::from_secs(2));
18912 assert_eq!(fixture.route.flow.in_flight(), 1);
18913 assert!(matches!(
18914 fixture.disconnect(),
18915 ClientControlPush::RouteClosed {
18916 reason: RouteCloseReason::Restart,
18917 drained: false,
18918 ..
18919 }
18920 ));
18921 }
18922
18923 #[tokio::test]
18924 async fn module_connection_crash_reports_crash_and_not_drained() {
18925 let mut fixture = route_closed_fixture().await;
18926 assert!(matches!(
18930 fixture.disconnect(),
18931 ClientControlPush::RouteClosed {
18932 reason: RouteCloseReason::Crash,
18933 drained: false,
18934 ..
18935 }
18936 ));
18937 }
18938
18939 #[tokio::test]
18940 async fn dropping_router_connection_releases_registration() {
18941 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18942 let registry = Arc::new(Registry::default());
18943 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18944 let router = Router::with_control_handler(Arc::clone(&control));
18945 let connection = router.begin_connection();
18946 let connection_id = connection.id();
18947 let (ctx, mut rx) = route_ctx(connection_id);
18948
18949 router
18950 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
18951 .await
18952 .unwrap();
18953 let response = rx.recv().await.unwrap();
18954 let ack = parse_ack(&response);
18955 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18956 assert!(registry.get_module("aft").unwrap().is_some());
18957
18958 drop(connection);
18959
18960 assert!(registry.get_module("aft").unwrap().is_none());
18961 assert_eq!(registry.active_registration_count().unwrap(), 0);
18962
18963 control
18964 .cleanup_connection(ConnectionId::new(u64::MAX))
18965 .unwrap();
18966 let captured = crate::router::test_log::captured_logs(&logs);
18967 println!("captured registry lifecycle logs:\n{captured}");
18968 let events: Vec<_> = captured
18969 .lines()
18970 .filter(|line| {
18971 line.contains("module registered module_id=aft ")
18972 || line.contains("module registration ended")
18973 })
18974 .collect();
18975 assert_eq!(
18976 events.len(),
18977 2,
18978 "unexpected registry lifecycle logs: {captured}"
18979 );
18980 assert!(events[0].contains("module registered module_id=aft "));
18981 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
18982 assert!(events[1].contains(&format!(
18983 "module_id=aft connection_id={}",
18984 connection_id.get()
18985 )));
18986 assert!(events[1].contains("reason=connection_closed"));
18987 assert!(events[1].contains("module registration ended"));
18988 assert_eq!(
18989 captured
18990 .lines()
18991 .filter(|line| line.contains("module registered module_id=aft "))
18992 .count(),
18993 1,
18994 "legacy registration admission line must appear once: {captured}"
18995 );
18996 }
18997
18998 #[tokio::test]
18999 async fn hello_registration_keeps_legacy_module_registered_line_once() {
19000 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19001 let registry = Arc::new(Registry::default());
19002 let control = ControlHandler::new(Arc::clone(®istry));
19003 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
19004 hello_via_sink(
19005 &control,
19006 &ctx,
19007 &mut rx,
19008 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
19009 )
19010 .await;
19011
19012 let captured = crate::router::test_log::captured_logs(&logs);
19013 let admissions: Vec<_> = captured
19014 .lines()
19015 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
19016 .collect();
19017 assert_eq!(
19018 admissions.len(),
19019 1,
19020 "legacy admission line must remain exactly once: {captured}"
19021 );
19022 assert!(admissions[0].contains("routable_provider=true"));
19023 assert!(admissions[0].contains("connection_id=777"));
19024 }
19025
19026 fn capability_manifest(
19027 module_id: &str,
19028 provides: &[&str],
19029 must_never_reach: &[&str],
19030 ) -> ModuleManifest {
19031 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
19032 manifest.capabilities = Some(CapabilityDeclarations {
19033 provides: provides
19034 .iter()
19035 .map(|capability| (*capability).to_string())
19036 .collect(),
19037 requires: Vec::new(),
19038 must_never_reach: must_never_reach
19039 .iter()
19040 .map(|capability| (*capability).to_string())
19041 .collect(),
19042 });
19043 manifest
19044 }
19045
19046 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
19047 Frame::build(
19048 FrameType::Hello,
19049 control_flags(),
19050 0,
19051 0,
19052 corr,
19053 serde_json::to_vec(&ModuleHelloBody {
19054 protocol_ver: manifest.protocol_ver,
19055 manifest,
19056 control_ops: None,
19057 launch_nonce: None,
19058 })
19059 .expect("capability test HELLO serializes"),
19060 )
19061 .expect("capability test HELLO frame builds")
19062 }
19063
19064 fn catalog_update_with_capabilities_frame(
19065 corr: u64,
19066 capabilities: CapabilityDeclarations,
19067 ) -> Frame {
19068 Frame::build(
19069 FrameType::Request,
19070 control_flags(),
19071 0,
19072 0,
19073 corr,
19074 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
19075 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
19076 capabilities: Some(capabilities),
19077 ready: None,
19078 })
19079 .expect("capability catalog.update serializes"),
19080 )
19081 .expect("capability catalog.update frame builds")
19082 }
19083
19084 async fn register_capability_manifest(
19085 handler: &ControlHandler,
19086 ctx: &RouteCtx,
19087 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19088 manifest: ModuleManifest,
19089 corr: u64,
19090 ) {
19091 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
19092 }
19093
19094 async fn open_route_for_capability_test(
19095 handler: &ControlHandler,
19096 target_ctx: &RouteCtx,
19097 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19098 client_connection_id: u64,
19099 corr: u64,
19100 target_module_id: &str,
19101 consumer_identity: Option<ConsumerIdentity>,
19102 ) -> (
19103 mpsc::Receiver<crate::router::OutboundFrame>,
19104 ModuleControlRequest,
19105 ) {
19106 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
19107 let route_handler = handler.clone();
19108 let target_module_id = target_module_id.to_string();
19109 let route_task = tokio::spawn(async move {
19110 route_handler
19111 .handle_control_frame(
19112 &client_ctx,
19113 route_open_frame_with_admission_facts(
19114 corr,
19115 &target_module_id,
19116 unique_project_root("admission-facts"),
19117 consumer_identity,
19118 None,
19119 ),
19120 )
19121 .await
19122 .expect("capability test route.open succeeds")
19123 });
19124 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
19125 .await
19126 .expect("capability test route.open must reach route.bind")
19127 .expect("target control receiver stays open");
19128 let bind_request: ModuleControlRequest =
19129 serde_json::from_slice(&bind.body).expect("route.bind decodes");
19130 handler
19131 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
19132 .await
19133 .expect("capability test route.bind ACK succeeds");
19134 assert!(route_task.await.expect("route.open task joins").is_empty());
19135 let opened = client_rx
19136 .recv()
19137 .await
19138 .expect("successful route.open publishes a response");
19139 assert!(matches!(
19140 serde_json::from_slice::<ClientControlResponse>(&opened.body),
19141 Ok(ClientControlResponse::RouteOpen { .. })
19142 ));
19143 (client_rx, bind_request)
19144 }
19145
19146 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
19147 assert_eq!(frame.header.ty, FrameType::Push);
19148 assert_eq!(frame.header.channel, 0);
19149 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
19150 .expect("route.closed control push decodes");
19151 let ClientControlPush::RouteClosed { channels, .. } = &push else {
19152 panic!("expected route.closed");
19153 };
19154 assert_eq!(channels.len(), 1, "exactly one violating route closed");
19155 let closed_channel = channels[0];
19156 let channels = channels.clone();
19157 assert_eq!(
19158 push,
19159 ClientControlPush::RouteClosed {
19160 module_id: target_module_id.to_string(),
19161 channels,
19162 reason: RouteCloseReason::CapabilityDenied,
19163 drained: false,
19164 abandoned: 0,
19165 excluded_subscriptions: 0,
19166 terminal: Some(false),
19167 }
19168 );
19169 closed_channel
19170 }
19171
19172 #[tokio::test]
19173 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
19174 let registry = Arc::new(Registry::default());
19175 let forwarding = Arc::new(ForwardingTable::default());
19176 let supervisor = SupervisorHandle::new();
19177 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19178 let handler =
19179 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19180 .with_supervisor(supervisor);
19181 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
19182 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
19183 register_capability_manifest(
19184 &handler,
19185 &target_ctx,
19186 &mut target_rx,
19187 capability_manifest("target", &["credentials-provider/v1"], &[]),
19188 1,
19189 )
19190 .await;
19191 register_capability_manifest(
19192 &handler,
19193 &opener_ctx,
19194 &mut opener_rx,
19195 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19196 2,
19197 )
19198 .await;
19199
19200 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
19201 let replies = handler
19202 .handle_control_frame(
19203 &client_ctx,
19204 route_open_frame_with_admission_facts(
19205 3,
19206 "target",
19207 unique_project_root("admission-facts"),
19208 Some(ConsumerIdentity {
19209 module_id: "opener".to_string(),
19210 launch_nonce: "opener-nonce".to_string(),
19211 }),
19212 None,
19213 ),
19214 )
19215 .await
19216 .expect("denied route.open returns a typed frame");
19217 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19218 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19219 assert!(
19220 target_rx.try_recv().is_err(),
19221 "forbidden route.open must not relay route.bind"
19222 );
19223 }
19224
19225 #[tokio::test]
19226 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
19227 let registry = Arc::new(Registry::default());
19228 let forwarding = Arc::new(ForwardingTable::default());
19229 let supervisor = SupervisorHandle::new();
19230 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19231 let handler =
19232 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19233 .with_supervisor(supervisor);
19234 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
19235 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
19236 register_capability_manifest(
19237 &handler,
19238 &target_ctx,
19239 &mut target_rx,
19240 capability_manifest("target", &["credentials-provider/v1"], &[]),
19241 1,
19242 )
19243 .await;
19244 register_capability_manifest(
19245 &handler,
19246 &old_opener_ctx,
19247 &mut old_opener_rx,
19248 capability_manifest("opener", &[], &[]),
19249 2,
19250 )
19251 .await;
19252 let (mut client_rx, _) = open_route_for_capability_test(
19253 &handler,
19254 &target_ctx,
19255 &mut target_rx,
19256 712,
19257 3,
19258 "target",
19259 Some(ConsumerIdentity {
19260 module_id: "opener".to_string(),
19261 launch_nonce: "opener-nonce".to_string(),
19262 }),
19263 )
19264 .await;
19265 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19266
19267 handler
19268 .cleanup_connection(old_opener_ctx.connection_id)
19269 .expect("old opener registration cleans up");
19270 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
19271 register_capability_manifest(
19272 &handler,
19273 &new_opener_ctx,
19274 &mut new_opener_rx,
19275 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19276 4,
19277 )
19278 .await;
19279
19280 let closed_channel = assert_capability_denied_push(
19281 client_rx
19282 .try_recv()
19283 .expect("HELLO deny addition must emit route.closed")
19284 .frame,
19285 "target",
19286 );
19287 let goodbye = client_rx
19288 .try_recv()
19289 .expect("HELLO deny addition must send the client a route GOODBYE")
19290 .frame;
19291 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19292 assert_eq!(goodbye.header.channel, closed_channel);
19293 assert_ne!(goodbye.header.channel, 0);
19294 assert_ne!(goodbye.header.epoch, 0);
19295 assert!(client_rx.try_recv().is_err());
19296 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19297 assert!(matches!(
19298 target_rx.try_recv(),
19299 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19300 ));
19301 }
19302
19303 #[tokio::test]
19304 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
19305 let registry = Arc::new(Registry::default());
19306 let forwarding = Arc::new(ForwardingTable::default());
19307 let supervisor = SupervisorHandle::new();
19308 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19309 let handler =
19310 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19311 .with_supervisor(supervisor);
19312 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
19313 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
19314 register_capability_manifest(
19315 &handler,
19316 &target_ctx,
19317 &mut target_rx,
19318 capability_manifest("target", &[], &[]),
19319 1,
19320 )
19321 .await;
19322 register_capability_manifest(
19323 &handler,
19324 &opener_ctx,
19325 &mut opener_rx,
19326 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19327 2,
19328 )
19329 .await;
19330 let (mut client_rx, _) = open_route_for_capability_test(
19331 &handler,
19332 &target_ctx,
19333 &mut target_rx,
19334 722,
19335 3,
19336 "target",
19337 Some(ConsumerIdentity {
19338 module_id: "opener".to_string(),
19339 launch_nonce: "opener-nonce".to_string(),
19340 }),
19341 )
19342 .await;
19343 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19344
19345 let replies = handler
19346 .handle_control_frame(
19347 &target_ctx,
19348 catalog_update_with_capabilities_frame(
19349 4,
19350 CapabilityDeclarations {
19351 provides: vec!["credentials-provider/v1".to_string()],
19352 requires: Vec::new(),
19353 must_never_reach: Vec::new(),
19354 },
19355 ),
19356 )
19357 .await
19358 .expect("claim catalog.update succeeds");
19359 assert!(matches!(
19360 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
19361 Ok(ModuleControlResponseToModule::CatalogUpdate {})
19362 ));
19363 let closed_channel = assert_capability_denied_push(
19364 client_rx
19365 .try_recv()
19366 .expect("claim addition must emit route.closed")
19367 .frame,
19368 "target",
19369 );
19370 let goodbye = client_rx
19371 .try_recv()
19372 .expect("claim addition must send the client a route GOODBYE")
19373 .frame;
19374 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19375 assert_eq!(goodbye.header.channel, closed_channel);
19376 assert_ne!(goodbye.header.channel, 0);
19377 assert_ne!(goodbye.header.epoch, 0);
19378 assert!(client_rx.try_recv().is_err());
19379 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19380 assert!(matches!(
19381 target_rx.try_recv(),
19382 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19383 ));
19384 }
19385
19386 #[tokio::test]
19387 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
19388 let registry = Arc::new(Registry::default());
19389 let forwarding = Arc::new(ForwardingTable::default());
19390 let supervisor = SupervisorHandle::new();
19391 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19392 let handler =
19393 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19394 .with_supervisor(supervisor);
19395 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
19396 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
19397 register_capability_manifest(
19398 &handler,
19399 &target_ctx,
19400 &mut target_rx,
19401 capability_manifest("target", &["credentials-provider/v1"], &[]),
19402 1,
19403 )
19404 .await;
19405 register_capability_manifest(
19406 &handler,
19407 &opener_ctx,
19408 &mut opener_rx,
19409 capability_manifest("opener", &[], &[]),
19410 2,
19411 )
19412 .await;
19413 let (mut client_rx, _) = open_route_for_capability_test(
19414 &handler,
19415 &target_ctx,
19416 &mut target_rx,
19417 732,
19418 3,
19419 "target",
19420 Some(ConsumerIdentity {
19421 module_id: "opener".to_string(),
19422 launch_nonce: "opener-nonce".to_string(),
19423 }),
19424 )
19425 .await;
19426 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19427
19428 handler
19429 .handle_control_frame(
19430 &target_ctx,
19431 catalog_update_with_capabilities_frame(
19432 4,
19433 CapabilityDeclarations {
19434 provides: Vec::new(),
19435 requires: Vec::new(),
19436 must_never_reach: Vec::new(),
19437 },
19438 ),
19439 )
19440 .await
19441 .expect("claim removal catalog.update succeeds");
19442 assert_eq!(
19443 forwarding.active_binding_count().unwrap(),
19444 1,
19445 "removing an attested target claim must leave the route census unchanged"
19446 );
19447 assert!(
19448 client_rx.try_recv().is_err(),
19449 "claim removal must not emit route.closed or a client route GOODBYE"
19450 );
19451 assert!(
19452 target_rx.try_recv().is_err(),
19453 "claim removal must not send the target a route GOODBYE"
19454 );
19455 }
19456
19457 #[tokio::test]
19460 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
19461 let registry = Arc::new(Registry::default());
19462 let forwarding = Arc::new(ForwardingTable::default());
19463 let supervisor = SupervisorHandle::new();
19464 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19465 let handler =
19466 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19467 .with_supervisor(supervisor);
19468 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
19469 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
19470 register_capability_manifest(
19471 &handler,
19472 &target_ctx,
19473 &mut target_rx,
19474 capability_manifest("target", &["credentials-provider/v1"], &[]),
19475 1,
19476 )
19477 .await;
19478 register_capability_manifest(
19479 &handler,
19480 &opener_ctx,
19481 &mut opener_rx,
19482 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19483 2,
19484 )
19485 .await;
19486
19487 let (_client_rx, bind) = open_route_for_capability_test(
19488 &handler,
19489 &target_ctx,
19490 &mut target_rx,
19491 742,
19492 3,
19493 "target",
19494 None,
19495 )
19496 .await;
19497 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
19498 panic!("direct scope-honesty route must bind");
19499 };
19500 assert_eq!(principal, Some(Principal::Direct));
19501 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19502 }
19503
19504 #[tokio::test]
19507 async fn must_never_reach_self_route_is_capability_forbidden() {
19508 let registry = Arc::new(Registry::default());
19509 let forwarding = Arc::new(ForwardingTable::default());
19510 let supervisor = SupervisorHandle::new();
19511 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
19512 let handler =
19513 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19514 .with_supervisor(supervisor);
19515 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
19516 register_capability_manifest(
19517 &handler,
19518 &self_ctx,
19519 &mut self_rx,
19520 capability_manifest(
19521 "self-provider",
19522 &["credentials-provider/v1"],
19523 &["credentials-provider/v1"],
19524 ),
19525 1,
19526 )
19527 .await;
19528
19529 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
19530 let replies = handler
19531 .handle_control_frame(
19532 &client_ctx,
19533 route_open_frame_with_admission_facts(
19534 2,
19535 "self-provider",
19536 unique_project_root("admission-facts"),
19537 Some(ConsumerIdentity {
19538 module_id: "self-provider".to_string(),
19539 launch_nonce: "self-nonce".to_string(),
19540 }),
19541 None,
19542 ),
19543 )
19544 .await
19545 .expect("self-route refusal returns a typed frame");
19546 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19547 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19548 assert!(
19549 self_rx.try_recv().is_err(),
19550 "self denial must not relay route.bind"
19551 );
19552 }
19553
19554 #[test]
19555 fn unsupported_channel_zero_frame_returns_error() {
19556 let handler = ControlHandler::default();
19557 let request = Frame::build(
19558 FrameType::Request,
19559 control_flags(),
19560 0,
19561 0,
19562 21,
19563 b"opaque".to_vec(),
19564 )
19565 .unwrap();
19566
19567 let response = handler
19568 .handle_control(ConnectionId::new(1), request)
19569 .unwrap();
19570
19571 assert_eq!(response[0].header.ty, FrameType::Error);
19572 assert_eq!(
19573 parse_error(&response[0])["code"],
19574 "unsupported_control_frame"
19575 );
19576 }
19577
19578 mod swap {
19583 use super::*;
19584
19585 const INCUMBENT: ConnectionId = ConnectionId::new(30);
19586 const CANDIDATE: ConnectionId = ConnectionId::new(40);
19587
19588 struct Swap {
19589 registry: Arc<Registry>,
19590 forwarding: Arc<ForwardingTable>,
19591 handler: ControlHandler,
19592 incumbent_ctx: RouteCtx,
19593 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19594 candidate_ctx: RouteCtx,
19595 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19596 }
19597
19598 async fn swap_with_incumbent() -> Swap {
19599 let registry = Arc::new(Registry::default());
19600 let forwarding = Arc::new(ForwardingTable::default());
19601 let handler =
19602 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
19603 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
19604 hello_via_sink(
19605 &handler,
19606 &incumbent_ctx,
19607 &mut incumbent_rx,
19608 hello_frame("aft", PROTOCOL_VERSION, 7),
19609 )
19610 .await;
19611 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
19612 Swap {
19613 registry,
19614 forwarding,
19615 handler,
19616 incumbent_ctx,
19617 incumbent_rx,
19618 candidate_ctx,
19619 candidate_rx,
19620 }
19621 }
19622
19623 fn register_candidate(swap: &Swap, ready: Option<bool>) {
19624 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
19625 candidate_manifest.ready = ready;
19626 let registration = swap
19627 .registry
19628 .register_candidate_with_control_ops(
19629 candidate_manifest,
19630 PROTOCOL_VERSION,
19631 CANDIDATE,
19632 module_baseline_control_ops(),
19633 )
19634 .unwrap();
19635 swap.forwarding
19636 .register_candidate_module_connection(
19637 CANDIDATE,
19638 "aft".to_string(),
19639 PROTOCOL_VERSION,
19640 manifest_concurrency(®istration.manifest),
19641 swap.candidate_ctx.egress.clone(),
19642 )
19643 .unwrap();
19644 }
19645
19646 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
19647 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
19648 swap.registry.promote_candidate("aft").unwrap().unwrap();
19649 cutover.incumbent.unwrap()
19650 }
19651
19652 #[tokio::test]
19661 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
19662 let mut swap = swap_with_incumbent().await;
19663 let hold = Arc::new(std::sync::Barrier::new(2));
19664 let handler = swap
19665 .handler
19666 .clone()
19667 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
19668 handler
19669 .supervisor
19670 .open_swap("aft", "candidate-nonce".to_string());
19671 let hello = {
19672 let handler = handler.clone();
19673 let candidate_ctx = swap.candidate_ctx.clone();
19674 std::thread::spawn(move || {
19675 handler
19676 .handle_hello(
19677 candidate_ctx.connection_id,
19678 Some(candidate_ctx.egress.clone()),
19679 hello_frame_with_nonce(
19680 "aft",
19681 PROTOCOL_VERSION,
19682 8,
19683 Some("candidate-nonce"),
19684 ),
19685 )
19686 .unwrap()
19687 })
19688 };
19689
19690 hold.wait();
19691 assert!(swap
19694 .registry
19695 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
19696 .unwrap()
19697 .is_some());
19698 assert!(swap
19699 .registry
19700 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19701 .unwrap()
19702 .is_none());
19703 assert!(swap
19704 .forwarding
19705 .module_endpoint_for_connection(CANDIDATE)
19706 .unwrap()
19707 .is_none());
19708
19709 hold.wait();
19710 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
19711 assert_eq!(
19712 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19713 FrameType::HelloAck
19714 );
19715 let published = swap
19716 .registry
19717 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19718 .unwrap()
19719 .expect("candidate is published once its HELLO completes");
19720 assert_eq!(published.connection_id, CANDIDATE);
19721 assert!(swap
19722 .forwarding
19723 .module_endpoint_for_connection(CANDIDATE)
19724 .unwrap()
19725 .is_some());
19726 }
19727
19728 #[tokio::test]
19729 async fn candidate_lookup_after_hello_ack_waits_for_publication() {
19730 let mut swap = swap_with_incumbent().await;
19731 let hold = Arc::new(std::sync::Barrier::new(2));
19732 swap.handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
19733 swap.handler
19734 .supervisor
19735 .open_swap("aft", "candidate-nonce".to_string());
19736 let hello = {
19737 let handler = swap.handler.clone();
19738 let candidate_ctx = swap.candidate_ctx.clone();
19739 std::thread::spawn(move || {
19740 handler
19741 .handle_hello(
19742 CANDIDATE,
19743 Some(candidate_ctx.egress),
19744 hello_frame_with_nonce(
19745 "aft",
19746 PROTOCOL_VERSION,
19747 8,
19748 Some("candidate-nonce"),
19749 ),
19750 )
19751 .unwrap()
19752 })
19753 };
19754 hold.wait();
19755 assert_eq!(
19756 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19757 FrameType::HelloAck
19758 );
19759 let reader_registry = Arc::clone(&swap.registry);
19760 let published = lookup_during_hello_publication(&swap.registry, &hold, move || {
19761 reader_registry
19762 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19763 .unwrap()
19764 });
19765 assert!(hello.join().unwrap().is_empty());
19766 assert_eq!(
19767 published
19768 .expect("ACK makes candidate publication visible")
19769 .connection_id,
19770 CANDIDATE
19771 );
19772 assert_eq!(
19773 swap.registry.generation().unwrap(),
19774 1,
19775 "candidate is not a second catalog entry"
19776 );
19777 }
19778
19779 #[tokio::test]
19780 async fn audit4_ungated_removed_candidate_publication_rolls_back_admission() {
19781 let mut swap = swap_with_incumbent().await;
19782 let hold = Arc::new(std::sync::Barrier::new(2));
19783 let handler = swap
19784 .handler
19785 .clone()
19786 .with_hello_registry_to_forwarding_hold(hold.clone());
19787 handler
19788 .supervisor
19789 .open_swap("aft", "candidate-nonce".into());
19790 let hello = {
19791 let handler = handler.clone();
19792 let ctx = swap.candidate_ctx.clone();
19793 std::thread::spawn(move || {
19794 handler
19795 .handle_hello(
19796 CANDIDATE,
19797 Some(ctx.egress),
19798 hello_frame_with_nonce(
19799 "aft",
19800 PROTOCOL_VERSION,
19801 8,
19802 Some("candidate-nonce"),
19803 ),
19804 )
19805 .unwrap()
19806 })
19807 };
19808 hold.wait();
19809 assert_eq!(
19810 swap.registry
19811 .deregister_connection(CANDIDATE)
19812 .unwrap()
19813 .len(),
19814 1
19815 );
19816 hold.wait();
19817 assert!(hello.join().unwrap().is_empty());
19818 assert_eq!(
19819 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19820 FrameType::HelloAck
19821 );
19822 assert!(swap.registry.get_candidate("aft").unwrap().is_none());
19823 assert_eq!(
19824 handler
19825 .supervisor
19826 .swap_hello_admission("aft", Some("candidate-nonce")),
19827 SwapHelloAdmission::Candidate,
19828 "failed publication must roll back candidate_admitted for an ungated nonce swap"
19829 );
19830 handler.supervisor.close_swap("aft");
19831 }
19832
19833 fn keyed_total(counters: &Value, key: &str) -> u64 {
19834 counters[key]
19835 .as_object()
19836 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
19837 .unwrap_or(0)
19838 }
19839
19840 #[tokio::test]
19841 async fn replacement_logs_old_end_and_new_admission_once() {
19842 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19843 let mut swap = swap_with_incumbent().await;
19844 swap.handler
19845 .supervisor
19846 .open_swap("aft", "candidate-nonce".to_string());
19847 hello_via_sink(
19848 &swap.handler,
19849 &swap.candidate_ctx,
19850 &mut swap.candidate_rx,
19851 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
19852 )
19853 .await;
19854 cutover(&swap);
19855 swap.handler.cleanup_connection(INCUMBENT).unwrap();
19856
19857 let captured = crate::router::test_log::captured_logs(&logs);
19858 println!("captured replacement registry logs:\n{captured}");
19859 let new_admissions: Vec<_> = captured
19860 .lines()
19861 .filter(|line| {
19862 line.contains("connection_id=40")
19863 && line.contains("swap candidate registered; not routable until cutover")
19864 })
19865 .collect();
19866 assert_eq!(
19867 new_admissions.len(),
19868 1,
19869 "unexpected admission logs: {captured}"
19870 );
19871 assert!(new_admissions[0].contains("module_id=aft"));
19872 assert!(new_admissions[0].contains("connection_id=40"));
19873 assert!(
19874 new_admissions[0].contains("swap candidate registered; not routable until cutover")
19875 );
19876 assert!(new_admissions[0].contains("ready=true"));
19877
19878 let old_admissions: Vec<_> = captured
19879 .lines()
19880 .filter(|line| {
19881 line.contains("module registered module_id=aft ")
19882 && line.contains("connection_id=30")
19883 })
19884 .collect();
19885 assert_eq!(
19886 old_admissions.len(),
19887 1,
19888 "unexpected incumbent admission: {captured}"
19889 );
19890
19891 let old_ends: Vec<_> = captured
19892 .lines()
19893 .filter(|line| {
19894 line.contains("connection_id=30") && line.contains("module registration ended")
19895 })
19896 .collect();
19897 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
19898 assert!(old_ends[0].contains("module_id=aft"));
19899 assert!(old_ends[0].contains("connection_id=30"));
19900 assert!(old_ends[0].contains("reason=replaced"));
19901 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
19902 assert_eq!(
19903 captured
19904 .lines()
19905 .filter(|line| line.contains("module registration promoted"))
19906 .count(),
19907 1,
19908 "unexpected promotion logs: {captured}"
19909 );
19910 }
19911
19912 #[tokio::test]
19918 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
19919 let mut swap = swap_with_incumbent().await;
19920 let handler = swap.handler.clone();
19921
19922 let cotenant = ConnectionId::new(31);
19924 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
19925 let (cotenant_task, cotenant_bind) = relay_route_open(
19926 &handler,
19927 cotenant,
19928 &cotenant_ctx.egress,
19929 &mut swap.incumbent_rx,
19930 100,
19931 "aft",
19932 "swap-cotenant",
19933 )
19934 .await;
19935 handler
19936 .handle_control_frame(
19937 &swap.incumbent_ctx,
19938 route_bind_ack(cotenant_bind.header.corr),
19939 )
19940 .await
19941 .unwrap();
19942 assert!(cotenant_task.await.unwrap().is_empty());
19943 let (cotenant_channel, cotenant_epoch) =
19944 published_route(&cotenant_rx.recv().await.unwrap());
19945
19946 let caller = ConnectionId::new(32);
19948 let (caller_ctx, mut caller_rx) = route_ctx(caller);
19949 let (caller_task, caller_bind) = relay_route_open(
19950 &handler,
19951 caller,
19952 &caller_ctx.egress,
19953 &mut swap.incumbent_rx,
19954 101,
19955 "aft",
19956 "swap-caller",
19957 )
19958 .await;
19959 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
19960
19961 register_candidate(&swap, None);
19962 cutover(&swap);
19963
19964 let ack = handler
19966 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
19967 .await;
19968 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
19969 if module_loop_error.is_some() {
19970 handler.cleanup_connection(INCUMBENT).unwrap();
19973 }
19974
19975 assert!(
19977 cotenant_rx.try_recv().is_err(),
19978 "the co-tenant route on the incumbent was torn down by one late ack: \
19979 {module_loop_error:?}"
19980 );
19981 assert!(matches!(
19982 swap.forwarding
19983 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
19984 .unwrap(),
19985 DataRoute::Client(DataRouteState::Bound(_))
19986 ));
19987 assert_eq!(module_loop_error, None);
19988 assert!(swap
19989 .registry
19990 .get_module_by_connection(INCUMBENT)
19991 .unwrap()
19992 .is_some());
19993
19994 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
19996 .await
19997 .expect("the incumbent is told to drop the abandoned binding")
19998 .unwrap()
19999 .frame;
20000 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
20001 assert_eq!(goodbye.header.channel, abandoned_channel);
20002 assert_eq!(goodbye.header.epoch, abandoned_epoch);
20003 assert!(swap.incumbent_rx.try_recv().is_err());
20004
20005 let response = caller_task.await.unwrap();
20007 assert_eq!(response.len(), 1);
20008 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
20009 assert!(caller_rx.try_recv().is_err());
20010
20011 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
20015 let counters = handler.counters().snapshot();
20016 assert_eq!(
20017 keyed_total(&counters, "route_open_accepted_by_principal"),
20018 1
20019 );
20020 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
20021 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
20022 }
20023
20024 #[tokio::test]
20028 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
20029 let mut swap = swap_with_incumbent().await;
20030 register_candidate(&swap, None);
20031 let incumbent = cutover(&swap);
20032 swap.forwarding
20033 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
20034 .unwrap()
20035 .expect("the incumbent is still registered");
20036
20037 let client = ConnectionId::new(33);
20038 let (client_ctx, mut client_rx) = route_ctx(client);
20039 let route_handler = swap.handler.clone();
20040 let open_ctx = RouteCtx {
20041 connection_id: client,
20042 egress: client_ctx.egress.clone(),
20043 };
20044 let mut route_task = tokio::spawn(async move {
20045 route_handler
20046 .handle_control_frame(
20047 &open_ctx,
20048 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
20049 )
20050 .await
20051 .unwrap()
20052 });
20053 let bind = tokio::select! {
20054 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
20055 response = &mut route_task => {
20056 let response = response.unwrap();
20057 panic!(
20058 "post-cutover route.open was refused instead of relayed to the candidate: {}",
20059 parse_error(&response[0])["code"]
20060 );
20061 }
20062 };
20063 swap.handler
20064 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
20065 .await
20066 .unwrap();
20067 assert!(route_task.await.unwrap().is_empty());
20068 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
20069 match swap
20070 .forwarding
20071 .lookup_data_route(client, channel, epoch)
20072 .unwrap()
20073 {
20074 DataRoute::Client(DataRouteState::Bound(route)) => {
20075 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
20076 }
20077 other => panic!("expected a bound route on the candidate, got {other:?}"),
20078 }
20079 assert!(swap.incumbent_rx.try_recv().is_err());
20080 }
20081
20082 #[tokio::test]
20087 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
20088 let swap = swap_with_incumbent().await;
20089 register_candidate(&swap, Some(false));
20090 let update = Frame::build(
20091 FrameType::Request,
20092 control_flags(),
20093 0,
20094 0,
20095 55,
20096 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
20097 provides: manifest("aft", PROTOCOL_VERSION).provides,
20098 capabilities: None,
20099 ready: Some(true),
20100 })
20101 .unwrap(),
20102 )
20103 .unwrap();
20104
20105 let replies = swap
20106 .handler
20107 .handle_control_frame(&swap.candidate_ctx, update)
20108 .await
20109 .unwrap();
20110
20111 assert_eq!(replies.len(), 1);
20112 assert_eq!(
20113 replies[0].header.ty,
20114 FrameType::Response,
20115 "candidate catalog.update was refused: {:?}",
20116 serde_json::from_slice::<Value>(&replies[0].body).ok()
20117 );
20118 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
20119 assert_eq!(
20120 swap.registry
20121 .get_module("aft")
20122 .unwrap()
20123 .unwrap()
20124 .connection_id,
20125 INCUMBENT
20126 );
20127 }
20128 }
20129
20130 mod swap_admission {
20134 use super::*;
20135
20136 const INCUMBENT_NONCE: &str = "incumbent-nonce";
20137 const CANDIDATE_NONCE: &str = "candidate-nonce";
20138
20139 fn handler_with_incumbent(
20140 module_id: &str,
20141 reserved: bool,
20142 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
20143 let registry = Arc::new(Registry::default());
20144 let supervisor = SupervisorHandle::new();
20145 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
20146 if reserved {
20147 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
20148 }
20149 let handler =
20150 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
20151 let incumbent = handler
20152 .handle_control(
20153 ConnectionId::new(1),
20154 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
20155 )
20156 .unwrap();
20157 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
20158 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
20159 (registry, supervisor, handler)
20160 }
20161
20162 #[test]
20168 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
20169 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
20170
20171 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
20172 let replies = handler
20173 .handle_control(
20174 ConnectionId::new(connection),
20175 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
20176 )
20177 .unwrap();
20178 assert_eq!(replies[0].header.ty, FrameType::Error);
20179 assert_eq!(
20180 parse_error(&replies[0])["code"],
20181 "swap_token_invalid",
20182 "nonce {nonce:?}"
20183 );
20184 }
20185 assert!(registry.get_candidate("aft").unwrap().is_none());
20186 assert_eq!(
20187 registry.get_module("aft").unwrap().unwrap().connection_id,
20188 ConnectionId::new(1)
20189 );
20190
20191 let admitted = handler
20194 .handle_control(
20195 ConnectionId::new(4),
20196 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
20197 )
20198 .unwrap();
20199 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
20200 assert_eq!(
20201 registry
20202 .get_candidate("aft")
20203 .unwrap()
20204 .unwrap()
20205 .connection_id,
20206 ConnectionId::new(4)
20207 );
20208 assert_eq!(
20209 registry.get_module("aft").unwrap().unwrap().connection_id,
20210 ConnectionId::new(1),
20211 "the candidate must not take the active slot"
20212 );
20213 let replayed = handler
20214 .handle_control(
20215 ConnectionId::new(5),
20216 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
20217 )
20218 .unwrap();
20219 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
20220
20221 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
20225 let squatter = handler
20226 .handle_control(
20227 ConnectionId::new(6),
20228 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
20229 )
20230 .unwrap();
20231 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
20232 assert!(
20233 registry.get_module("aft").unwrap().is_none(),
20234 "a squatter took the active slot of an id being swapped"
20235 );
20236 }
20237
20238 #[test]
20243 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
20244 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
20245
20246 let replies = handler
20247 .handle_control(
20248 ConnectionId::new(2),
20249 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20250 )
20251 .unwrap();
20252
20253 assert_eq!(
20254 replies[0].header.ty,
20255 FrameType::HelloAck,
20256 "reserved candidate refused: {:?}",
20257 serde_json::from_slice::<Value>(&replies[0].body).ok()
20258 );
20259 assert_eq!(
20260 registry
20261 .get_candidate("vault")
20262 .unwrap()
20263 .unwrap()
20264 .connection_id,
20265 ConnectionId::new(2)
20266 );
20267 }
20268
20269 #[test]
20272 fn without_an_open_swap_the_ordinary_gates_decide() {
20273 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
20274 supervisor.close_swap("vault");
20275
20276 let candidate = handler
20277 .handle_control(
20278 ConnectionId::new(2),
20279 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20280 )
20281 .unwrap();
20282 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
20283 let duplicate = handler
20284 .handle_control(
20285 ConnectionId::new(3),
20286 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
20287 )
20288 .unwrap();
20289 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
20290 assert!(registry.get_candidate("vault").unwrap().is_none());
20291 }
20292 }
20293
20294 mod scopes {
20298 mod close_reasons {
20299 include!("route_close_reason_tests.rs");
20300 }
20301 mod idle_tests {
20302 use super::*;
20303 include!("scope_idle_tests.rs");
20304 }
20305 mod route_open_timings {
20306 use super::*;
20307 include!("route_open_timing_tests.rs");
20308 }
20309 mod route_open_refusal_log {
20310 use super::*;
20311 include!("route_open_refusal_log_tests.rs");
20312 }
20313 mod activity {
20314 include!("activity_scope_tests.rs");
20315 }
20316 mod delta {
20317 include!("scope_handler_delta_tests.rs");
20318 }
20319 mod watch {
20320 include!("scope_watch_tests.rs");
20321 }
20322 mod call_modes {
20323 include!("scope_call_mode_tests.rs");
20324 }
20325 mod federated {
20326 include!("scope_federated_tests.rs");
20327 }
20328 mod authorize {
20329 include!("route_authorize_tests.rs");
20330 }
20331 #[tokio::test]
20332 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
20333 let rig = rig().await;
20334 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
20335 "s".into(),
20336 1,
20337 "tool".into(),
20338 ))
20339 .unwrap();
20340 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
20341 let (direct, _rx) = wide_ctx(999);
20342 for (ctx, expected) in [
20343 (&rig.owner, FrameType::Response),
20344 (&direct, FrameType::Error),
20345 ] {
20346 let frame =
20347 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
20348 .unwrap();
20349 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
20350 assert_eq!(replies[0].header.ty, expected);
20351 if expected == FrameType::Response {
20352 let reply: subc_control::ScopeServedByReply =
20353 serde_json::from_slice(&replies[0].body).unwrap();
20354 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
20355 } else {
20356 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
20357 }
20358 }
20359 let reply = call(
20360 &rig.handler,
20361 &rig.owner,
20362 &ModuleControlRequestFromModule::ScopeDescribe {
20363 owner: Principal::Reserved {
20364 module_id: OWNER.into(),
20365 },
20366 scope_ref: "s".into(),
20367 },
20368 )
20369 .await;
20370 assert!(matches!(
20371 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
20372 ModuleControlResponseToModule::ScopeDescribe { .. }
20373 ));
20374 }
20375
20376 use subc_protocol::scope::{
20377 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
20378 ScopeStatus,
20379 };
20380
20381 use super::*;
20382
20383 const OWNER: &str = "prefrontal-core";
20384
20385 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
20386 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
20387 }
20388
20389 async fn call(
20390 handler: &ControlHandler,
20391 ctx: &RouteCtx,
20392 request: &ModuleControlRequestFromModule,
20393 ) -> Frame {
20394 let body = serde_json::to_vec(request).unwrap();
20395 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
20396 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
20397 assert_eq!(replies.len(), 1, "{replies:?}");
20398 replies.pop().unwrap()
20399 }
20400
20401 async fn sync(
20402 handler: &ControlHandler,
20403 ctx: &RouteCtx,
20404 generation: u64,
20405 scopes: Vec<ScopeRecord>,
20406 ) -> Result<ModuleControlResponseToModule, String> {
20407 let reply = call(
20408 handler,
20409 ctx,
20410 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
20411 )
20412 .await;
20413 match reply.header.ty {
20414 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
20415 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
20416 }
20417 }
20418
20419 async fn describe(
20420 handler: &ControlHandler,
20421 ctx: &RouteCtx,
20422 owner: &str,
20423 scope_ref: &str,
20424 ) -> ModuleControlResponseToModule {
20425 let reply = call(
20426 handler,
20427 ctx,
20428 &ModuleControlRequestFromModule::ScopeDescribe {
20429 owner: Principal::Reserved {
20430 module_id: owner.to_string(),
20431 },
20432 scope_ref: scope_ref.to_string(),
20433 },
20434 )
20435 .await;
20436 assert_eq!(
20437 reply.header.ty,
20438 FrameType::Response,
20439 "{:?}",
20440 parse_error(&reply)
20441 );
20442 serde_json::from_slice(&reply.body).unwrap()
20443 }
20444
20445 #[tokio::test]
20446 async fn scope_describe_never_echoes_bind_proof_or_hash() {
20447 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
20448 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
20449 let mut rig = rig().await;
20450 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
20451 .await;
20452 let _bound = rig
20453 .bound(
20454 Some(AFT),
20455 PLEXUS,
20456 Some(
20457 rig.selector("s", Some(1))
20458 .with_bind_proof(Some(PROOF.into())),
20459 ),
20460 )
20461 .await;
20462 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
20463 let output = serde_json::to_string(&reply).unwrap();
20464 assert!(output.contains("\"status\":\"live\""));
20465 assert!(!output.contains(PROOF));
20466 assert!(!output.contains(HASH));
20467 }
20468
20469 async fn module(
20471 handler: &ControlHandler,
20472 connection: u64,
20473 module_id: &str,
20474 nonce: Option<&str>,
20475 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20476 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
20477 hello_via_sink(
20478 handler,
20479 &ctx,
20480 &mut rx,
20481 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
20482 )
20483 .await;
20484 (ctx, rx)
20485 }
20486
20487 #[tokio::test]
20490 async fn a_client_connection_cannot_sync_or_describe() {
20491 let handler = ControlHandler::new(Arc::new(Registry::default()));
20492 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
20493 for request in [
20494 ModuleControlRequestFromModule::ScopeSync {
20495 generation: 1,
20496 scopes: vec![head("s", 1)],
20497 },
20498 ModuleControlRequestFromModule::ScopeDescribe {
20499 owner: Principal::Direct,
20500 scope_ref: "s".to_string(),
20501 },
20502 ] {
20503 let reply = call(&handler, &ctx, &request).await;
20504 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
20505 }
20506 assert!(
20507 !handler
20508 .scopes
20509 .read()
20510 .unwrap()
20511 .describe(
20512 &Principal::Reserved {
20513 module_id: OWNER.to_string()
20514 },
20515 "s"
20516 )
20517 .owner_synced
20518 );
20519 }
20520
20521 #[tokio::test]
20524 async fn a_module_without_a_supervised_launch_cannot_sync() {
20525 let handler = ControlHandler::new(Arc::new(Registry::default()));
20526 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
20527 assert_eq!(
20528 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
20529 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20530 );
20531 }
20532
20533 #[tokio::test]
20534 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
20535 let supervisor = SupervisorHandle::new();
20536 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20537 let handler = ControlHandler::new(Arc::new(Registry::default()))
20538 .with_supervisor(supervisor.clone());
20539 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20540 sync(&handler, &incumbent, 1, vec![head("s", 1)])
20541 .await
20542 .expect("the current launch syncs");
20543
20544 supervisor.open_swap(OWNER, "n2".to_string());
20547 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
20548 assert_eq!(
20549 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20550 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20551 );
20552 sync(&handler, &incumbent, 2, vec![head("s", 1)])
20553 .await
20554 .expect("the serving owner syncs during the swap");
20555
20556 supervisor.close_swap(OWNER);
20559 assert_eq!(
20560 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20561 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20562 );
20563 sync(&handler, &incumbent, 3, vec![head("s", 1)])
20564 .await
20565 .expect("the serving owner syncs after the rollback");
20566 handler.cleanup_connection(candidate.connection_id).unwrap();
20567
20568 supervisor.open_swap(OWNER, "n3".to_string());
20573 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
20574 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
20575 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
20576 .await
20577 .expect("the promoted launch takes authority");
20578 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20579 panic!("unexpected reply {reply:?}");
20580 };
20581 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
20582 assert_eq!(
20583 sync(&handler, &incumbent, 4, Vec::new()).await,
20584 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20585 );
20586 }
20587
20588 #[tokio::test]
20591 async fn closing_the_authority_connection_frees_sync_authority() {
20592 let supervisor = SupervisorHandle::new();
20593 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20594 let handler = ControlHandler::new(Arc::new(Registry::default()))
20595 .with_supervisor(supervisor.clone());
20596 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20597 sync(&handler, &first, 10, vec![head("s", 1)])
20598 .await
20599 .unwrap();
20600 handler.cleanup_connection(first.connection_id).unwrap();
20601
20602 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
20603 sync(&handler, &second, 1, vec![head("s", 1)])
20604 .await
20605 .expect("the next connection takes the released authority");
20606 }
20607
20608 #[tokio::test]
20609 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
20610 let supervisor = SupervisorHandle::new();
20611 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20612 let handler =
20613 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
20614 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
20615 sync(&handler, &first, 10, vec![head("s", 1)])
20616 .await
20617 .unwrap();
20618 handler
20619 .handle_control_frame(
20620 &first,
20621 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
20622 )
20623 .await
20624 .unwrap();
20625 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
20626 sync(&handler, &second, 1, vec![head("s", 1)])
20627 .await
20628 .expect("GOODBYE releases authority even if the old socket remains open");
20629 }
20630
20631 #[tokio::test]
20632 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
20633 let registry = Arc::new(Registry::default());
20634 let supervisor_handle = SupervisorHandle::new();
20635 let supervisor =
20636 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20637 .with_handle(supervisor_handle.clone())
20638 .with_daemon_incarnation("incarnation-7".to_string());
20639 supervisor
20642 .supervise_configured(
20643 ModuleSpec {
20644 script: None,
20645 carry_containment: false,
20646 module_id: OWNER.to_string(),
20647 program: PathBuf::from("/nonexistent/prefrontal-core"),
20648 args: Vec::new(),
20649 env: Vec::new(),
20650 reserved: false,
20651 reserved_prefixes: Vec::new(),
20652 protocol: ModuleProtocol::Subc,
20653 overlap: Default::default(),
20654 },
20655 false,
20656 )
20657 .unwrap();
20658 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
20659 let handler =
20660 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
20661 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
20662
20663 let ModuleControlResponseToModule::ScopeDescribe {
20665 status,
20666 daemon_incarnation,
20667 owner_synced,
20668 owner_configured,
20669 scope,
20670 ..
20671 } = describe(&handler, &reader, OWNER, "s").await
20672 else {
20673 panic!("not a describe reply");
20674 };
20675 assert_eq!(status, ScopeStatus::NotLive);
20676 assert_eq!(daemon_incarnation, "incarnation-7");
20677 assert!(!owner_synced);
20678 assert!(owner_configured);
20679 assert!(scope.is_none());
20680
20681 let ModuleControlResponseToModule::ScopeDescribe {
20684 status,
20685 owner_configured,
20686 ..
20687 } = describe(&handler, &reader, "ghost", "s").await
20688 else {
20689 panic!("not a describe reply");
20690 };
20691 assert_eq!(status, ScopeStatus::NotLive);
20692 assert!(!owner_configured);
20693
20694 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
20696 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
20697 let ModuleControlResponseToModule::ScopeDescribe {
20698 status,
20699 scope_epoch,
20700 owner_synced,
20701 scope,
20702 ..
20703 } = describe(&handler, &reader, OWNER, "s").await
20704 else {
20705 panic!("not a describe reply");
20706 };
20707 assert_eq!(status, ScopeStatus::Live);
20708 assert_eq!(scope_epoch, Some(4));
20709 assert!(owner_synced);
20710 let stamp = scope.expect("a live scope carries its stamp");
20711 assert!(
20712 stamp.owner_authorized,
20713 "prefrontal-core is the default authority"
20714 );
20715 assert_eq!(stamp.kind, ScopeKind::Head);
20716 }
20717
20718 #[tokio::test]
20719 async fn scope_authority_owners_decides_owner_authorized() {
20720 let supervisor = SupervisorHandle::new();
20721 supervisor.set_spawn_nonce("broca", "b1".to_string());
20722 let handler = ControlHandler::new(Arc::new(Registry::default()))
20723 .with_supervisor(supervisor)
20724 .with_scope_authority_owners(vec!["broca".to_string()]);
20725 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
20726 let mut gated = head("s", 1);
20727 gated.attributes.agent_id = Some("agent".to_string());
20728 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
20729 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
20730 describe(&handler, &broca, "broca", "s").await
20731 else {
20732 panic!("not a describe reply");
20733 };
20734 assert!(scope.unwrap().owner_authorized);
20735 }
20736
20737 #[tokio::test]
20741 async fn scope_ops_and_the_scopes_capability_are_advertised() {
20742 let handler = ControlHandler::new(Arc::new(Registry::default()));
20743 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
20744 let ack = hello_via_sink(
20745 &handler,
20746 &ctx,
20747 &mut rx,
20748 hello_frame("m", PROTOCOL_VERSION, 1),
20749 )
20750 .await;
20751 let ack = parse_ack(&ack);
20752 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
20753 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
20754 }
20755 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
20756
20757 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
20758 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
20759 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
20760 let reply = handler
20761 .handle_control_frame(&client, frame)
20762 .await
20763 .unwrap()
20764 .pop()
20765 .unwrap();
20766 let ClientControlResponse::ServerDescribe { capabilities, .. } =
20767 serde_json::from_slice(&reply.body).unwrap()
20768 else {
20769 panic!("not a server.describe reply");
20770 };
20771 assert!(
20772 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
20773 "{capabilities:?}"
20774 );
20775 }
20776
20777 const PLEXUS: &str = "plexus";
20780 const OTHER: &str = "other";
20781 const AFT: &str = "aft";
20782 const BROCA: &str = "broca";
20783 const MAGIC: &str = "magic-context";
20784
20785 fn nonce(module_id: &str) -> String {
20786 format!("nonce-{module_id}")
20787 }
20788
20789 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20790 let (tx, rx) = mpsc::channel(64);
20791 (
20792 RouteCtx {
20793 connection_id: ConnectionId::new(connection),
20794 egress: FrameSink::new(tx),
20795 },
20796 rx,
20797 )
20798 }
20799
20800 struct Rig {
20804 handler: ControlHandler,
20805 forwarding: Arc<ForwardingTable>,
20806 owner: RouteCtx,
20807 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20808 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
20809 generation: u64,
20810 next_connection: u64,
20811 _supervisor: Supervisor,
20812 }
20813
20814 async fn rig() -> Rig {
20815 rig_with_flow_support(true).await
20816 }
20817
20818 async fn rig_with_flow_support(flow_support: bool) -> Rig {
20819 rig_with_scope_capabilities(flow_support, false).await
20820 }
20821
20822 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
20823 let mut provides = Vec::new();
20824 if flow_support {
20825 provides.push("flow-scopes/v1");
20826 }
20827 if run_support {
20828 provides.push("agent-run-scopes/v1");
20829 }
20830 rig_with_target_capabilities(&provides).await
20831 }
20832
20833 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
20834 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
20835 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20836 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20839 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
20840 Frame::build(
20841 FrameType::Hello,
20842 control_flags(),
20843 0,
20844 0,
20845 connection,
20846 serde_json::to_vec(&body).unwrap(),
20847 )
20848 .unwrap()
20849 }
20850
20851 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
20852 let registry = Arc::new(Registry::default());
20853 let forwarding = Arc::new(ForwardingTable::default());
20854 let supervisor_handle = SupervisorHandle::new();
20855 let supervisor =
20856 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20857 .with_handle(supervisor_handle.clone());
20858 supervisor
20859 .supervise_configured(
20860 ModuleSpec {
20861 script: None,
20862 carry_containment: false,
20863 module_id: OWNER.to_string(),
20864 program: PathBuf::from("/nonexistent/prefrontal-core"),
20865 args: Vec::new(),
20866 env: Vec::new(),
20867 reserved: false,
20868 reserved_prefixes: Vec::new(),
20869 protocol: ModuleProtocol::Subc,
20870 overlap: Default::default(),
20871 },
20872 false,
20873 )
20874 .unwrap();
20875 for module_id in [OWNER, AFT, BROCA, MAGIC] {
20876 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
20877 }
20878 let handler =
20879 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
20880 .with_supervisor(supervisor_handle);
20881 let (owner, mut owner_rx) = wide_ctx(1);
20882 hello_via_sink(
20883 &handler,
20884 &owner,
20885 &mut owner_rx,
20886 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
20887 )
20888 .await;
20889 let mut modules = BTreeMap::new();
20890 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
20891 let (ctx, mut rx) = wide_ctx(connection);
20892 let hello = hello_with_capabilities(module_id, connection, provides);
20893 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
20894 modules.insert(module_id.to_string(), (ctx, rx));
20895 }
20896 Rig {
20897 handler,
20898 forwarding,
20899 owner,
20900 _owner_rx: owner_rx,
20901 modules,
20902 generation: 0,
20903 next_connection: 100,
20904 _supervisor: supervisor,
20905 }
20906 }
20907
20908 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
20909 ScopeCarrier::new(Principal::Reserved {
20910 module_id: module_id.to_string(),
20911 })
20912 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
20913 }
20914
20915 fn session(scope_epoch: u64) -> ScopeRecord {
20918 let mut record = head("s", scope_epoch);
20919 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
20920 record.attributes.agent_id = Some("agent-1".to_string());
20921 record.attributes.delegates = true;
20922 record
20923 }
20924
20925 impl Rig {
20926 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
20927 self.generation += 1;
20928 sync(&self.handler, &self.owner, self.generation, scopes)
20929 .await
20930 .expect("the owner's sync is accepted");
20931 }
20932
20933 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
20934 ScopeSelector::new(
20935 Principal::Reserved {
20936 module_id: OWNER.to_string(),
20937 },
20938 scope_ref.to_string(),
20939 scope_epoch,
20940 )
20941 }
20942
20943 fn open_frame(
20944 &mut self,
20945 opener: Option<&str>,
20946 target: &str,
20947 scope: Option<ScopeSelector>,
20948 ) -> (
20949 RouteCtx,
20950 mpsc::Receiver<crate::router::OutboundFrame>,
20951 Frame,
20952 ) {
20953 self.next_connection += 1;
20954 let (ctx, rx) = wide_ctx(self.next_connection);
20955 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
20956 busy_refusals: None,
20957 target: RouteTarget::ToolProvider {
20958 module_id: target.to_string(),
20959 },
20960 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
20961 consumer_identity: opener.map(|module_id| ConsumerIdentity {
20962 module_id: module_id.to_string(),
20963 launch_nonce: nonce(module_id),
20964 }),
20965 consumer_capabilities: None,
20966 role_versions: None,
20967 admission_facts: None,
20968 scope,
20969 })
20970 .unwrap();
20971 let frame = Frame::build(
20972 FrameType::Request,
20973 control_flags(),
20974 0,
20975 0,
20976 self.next_connection,
20977 body,
20978 )
20979 .unwrap();
20980 (ctx, rx, frame)
20981 }
20982
20983 async fn refused(
20985 &mut self,
20986 opener: Option<&str>,
20987 target: &str,
20988 scope: Option<ScopeSelector>,
20989 ) -> String {
20990 self.refusal_body(opener, target, scope).await["code"]
20991 .as_str()
20992 .unwrap()
20993 .to_string()
20994 }
20995
20996 async fn refusal_body(
20997 &mut self,
20998 opener: Option<&str>,
20999 target: &str,
21000 scope: Option<ScopeSelector>,
21001 ) -> Value {
21002 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
21003 let replies = tokio::time::timeout(
21004 Duration::from_secs(2),
21005 self.handler.handle_control_frame(&ctx, frame),
21006 )
21007 .await
21008 .expect("the open must be refused before waiting for a bind ack")
21009 .unwrap();
21010 assert_eq!(replies.len(), 1, "{replies:?}");
21011 assert_eq!(replies[0].header.ty, FrameType::Error);
21012 let (_, module_rx) = self.modules.get_mut(target).unwrap();
21013 assert!(
21014 module_rx.try_recv().is_err(),
21015 "a refused open relays nothing"
21016 );
21017 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
21018 parse_error(&replies[0])
21019 }
21020
21021 async fn relayed(
21023 &mut self,
21024 opener: Option<&str>,
21025 target: &str,
21026 scope: Option<ScopeSelector>,
21027 ) -> Relayed {
21028 self.relayed_with_legacy_root(opener, target, scope, None)
21029 .await
21030 }
21031
21032 async fn relayed_with_legacy_root(
21033 &mut self,
21034 opener: Option<&str>,
21035 target: &str,
21036 scope: Option<ScopeSelector>,
21037 legacy_root: Option<&std::path::Path>,
21038 ) -> Relayed {
21039 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
21040 if let Some(root) = legacy_root {
21041 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
21042 body["identity"]["project_root"] = serde_json::json!(root);
21043 frame.body = serde_json::to_vec(&body).unwrap();
21044 }
21045 let handler = self.handler.clone();
21046 let task_ctx = ctx.clone();
21047 let task = tokio::spawn(async move {
21048 handler
21049 .handle_control_frame(&task_ctx, frame)
21050 .await
21051 .unwrap()
21052 });
21053 let (_, module_rx) = self.modules.get_mut(target).unwrap();
21054 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
21055 .await
21056 .expect("the target receives the relayed route.bind")
21057 .unwrap()
21058 .frame;
21059 Relayed {
21060 target: target.to_string(),
21061 client: ctx,
21062 client_rx: rx,
21063 task,
21064 bind,
21065 }
21066 }
21067
21068 async fn ack(&self, relayed: &Relayed) {
21069 let (module, _) = &self.modules[&relayed.target];
21070 self.handler
21071 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
21072 .await
21073 .unwrap();
21074 }
21075
21076 async fn bound(
21078 &mut self,
21079 opener: Option<&str>,
21080 target: &str,
21081 scope: Option<ScopeSelector>,
21082 ) -> Bound {
21083 let relayed = self.relayed(opener, target, scope).await;
21084 self.commit_relayed(relayed).await
21085 }
21086
21087 async fn bound_with_legacy_root(
21088 &mut self,
21089 opener: Option<&str>,
21090 target: &str,
21091 scope: Option<ScopeSelector>,
21092 root: &std::path::Path,
21093 ) -> Bound {
21094 let relayed = self
21095 .relayed_with_legacy_root(opener, target, scope, Some(root))
21096 .await;
21097 self.commit_relayed(relayed).await
21098 }
21099
21100 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
21101 self.ack(&relayed).await;
21102 let Relayed {
21103 target,
21104 client,
21105 mut client_rx,
21106 task,
21107 bind,
21108 } = relayed;
21109 assert!(
21110 task.await.unwrap().is_empty(),
21111 "the open is answered by commit"
21112 );
21113 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
21114 Bound {
21115 target,
21116 client,
21117 client_rx,
21118 channel,
21119 epoch,
21120 bind,
21121 }
21122 }
21123
21124 fn live(&self, route: &Bound) -> bool {
21125 matches!(
21126 self.forwarding
21127 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21128 .unwrap(),
21129 DataRoute::Client(DataRouteState::Bound(_))
21130 )
21131 }
21132 }
21133
21134 struct Relayed {
21135 target: String,
21136 client: RouteCtx,
21137 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21138 task: tokio::task::JoinHandle<Vec<Frame>>,
21139 bind: Frame,
21140 }
21141
21142 struct Bound {
21143 target: String,
21144 client: RouteCtx,
21145 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21146 channel: u16,
21147 epoch: u32,
21148 bind: Frame,
21149 }
21150
21151 impl Bound {
21152 fn closed_reason(&mut self) -> RouteCloseReason {
21155 let mut reason = None;
21156 let mut goodbye = false;
21157 while let Ok(outbound) = self.client_rx.try_recv() {
21158 let frame = outbound.frame;
21159 match frame.header.ty {
21160 FrameType::Goodbye => {
21161 assert_eq!(
21162 (frame.header.channel, frame.header.epoch),
21163 (self.channel, self.epoch)
21164 );
21165 goodbye = true;
21166 }
21167 FrameType::Push => {
21168 let ClientControlPush::RouteClosed {
21169 reason: r,
21170 module_id,
21171 ..
21172 } = serde_json::from_slice(&frame.body).unwrap()
21173 else {
21174 panic!("unexpected push");
21175 };
21176 assert_eq!(module_id, self.target);
21177 reason = Some(r);
21178 }
21179 other => panic!("unexpected frame {other:?}"),
21180 }
21181 }
21182 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
21183 reason.expect("the client is told why the route closed")
21184 }
21185
21186 fn untouched(&mut self) -> bool {
21187 self.client_rx.try_recv().is_err()
21188 }
21189
21190 fn stamp(&self) -> Option<ScopeStamp> {
21191 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
21192 ModuleControlRequest::RouteBind { scope, .. } => scope,
21193 other => panic!("expected a route.bind, got {other:?}"),
21194 }
21195 }
21196 }
21197
21198 #[tokio::test]
21199 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
21200 ) {
21201 let mut rig = rig().await;
21202 let mut record = session(1);
21203 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
21204 record.child_owners = vec![Principal::Reserved {
21205 module_id: MAGIC.to_string(),
21206 }];
21207 rig.sync(vec![record]).await;
21208 let scope = || Some(rig_selector("s", Some(1)));
21209
21210 rig.bound(Some(OWNER), PLEXUS, scope()).await;
21213 rig.bound(Some(AFT), OTHER, scope()).await;
21214 rig.bound(Some(BROCA), PLEXUS, scope()).await;
21215
21216 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
21220 assert_eq!(
21221 rig.refused(opener, target, scope()).await,
21222 error_codes::SCOPE_NOT_CARRIER,
21223 "{opener:?} -> {target}"
21224 );
21225 }
21226 }
21227
21228 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
21229 ScopeSelector::new(
21230 Principal::Reserved {
21231 module_id: OWNER.to_string(),
21232 },
21233 scope_ref.to_string(),
21234 scope_epoch,
21235 )
21236 }
21237
21238 #[tokio::test]
21239 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
21240 let mut rig = rig().await;
21241 rig.sync(vec![session(1)]).await;
21242 for opener in [OWNER, AFT] {
21243 assert_eq!(
21244 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
21245 .await,
21246 error_codes::SCOPE_EPOCH_REQUIRED,
21247 "{opener}"
21248 );
21249 }
21250 }
21251
21252 #[tokio::test]
21253 async fn admission_separates_not_synced_not_live_and_ended() {
21254 let mut rig = rig().await;
21255 let code = rig
21257 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21258 .await;
21259 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
21260 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
21261
21262 let ghost = ScopeSelector::new(
21264 Principal::Reserved {
21265 module_id: "ghost".to_string(),
21266 },
21267 "s".to_string(),
21268 Some(1),
21269 );
21270 assert_eq!(
21271 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
21272 error_codes::SCOPE_NOT_LIVE
21273 );
21274
21275 rig.sync(vec![session(2)]).await;
21276 assert_eq!(
21277 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
21278 .await,
21279 error_codes::SCOPE_NOT_LIVE
21280 );
21281 for epoch in [1, 3] {
21282 assert_eq!(
21283 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
21284 .await,
21285 error_codes::SCOPE_ENDED,
21286 "epoch {epoch}"
21287 );
21288 }
21289 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
21291 .await;
21292 }
21293
21294 #[tokio::test]
21295 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
21296 let mut rig = rig().await;
21297 rig.sync(vec![session(1)]).await;
21298 let route = rig
21299 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21300 .await;
21301 let stamp = route.stamp().expect("a scoped bind carries the stamp");
21302 assert_eq!(stamp.scope_ref, "s");
21303 assert_eq!(stamp.scope_epoch, 1);
21304 assert_eq!(stamp.kind, ScopeKind::Head);
21305 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
21306 assert!(stamp.attributes.delegates);
21307 assert!(stamp.owner_authorized);
21308 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21309 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
21310
21311 let (broca, mut broca_rx) = wide_ctx(50);
21314 hello_via_sink(
21315 &rig.handler,
21316 &broca,
21317 &mut broca_rx,
21318 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
21319 )
21320 .await;
21321 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
21322 .await
21323 .unwrap();
21324 let own = ScopeSelector::new(
21325 Principal::Reserved {
21326 module_id: BROCA.to_string(),
21327 },
21328 "b".to_string(),
21329 Some(1),
21330 );
21331 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
21332 assert!(!route.stamp().unwrap().owner_authorized);
21333 }
21334
21335 #[tokio::test]
21336 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
21337 let mut rig = rig().await;
21338 let mut record = head("s", 1);
21339 record.carriers = vec![carrier(AFT, None)];
21340 let flow_id = "Flow:run-7/step_2!~";
21341 record.attributes.flow_id = Some(flow_id.to_string());
21342 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
21343 .await
21344 .unwrap();
21345 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
21346 panic!("not a sync reply");
21347 };
21348 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
21349 let route = rig
21350 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21351 .await;
21352 assert!(rig.live(&route), "the stamped bind committed");
21353 let stamp = route.stamp().expect("a flow scope carries a stamp");
21354 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
21355 assert_eq!(stamp.attributes.agent_id, None);
21356 assert!(!stamp.attributes.delegates);
21357 assert!(stamp.owner_authorized);
21358 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21359 assert_eq!(unscoped.stamp(), None);
21360 }
21361
21362 #[tokio::test]
21363 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
21364 let mut rig = rig_with_flow_support(false).await;
21365 let mut record = session(1);
21366 record.attributes.flow_id = Some("flow:7".to_string());
21367 rig.sync(vec![record]).await;
21368 for opener in [OWNER, AFT] {
21369 let body = rig
21370 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21371 .await;
21372 assert_eq!(body["code"], "target_flow_unsupported");
21373 let message = body["message"].as_str().unwrap();
21374 for required in [PLEXUS, "flow-scopes/v1"] {
21375 assert!(message.contains(required), "{message}");
21376 }
21377 }
21378 }
21379
21380 #[tokio::test]
21381 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
21382 let mut rig = rig_with_flow_support(true).await;
21383 let mut record = session(1);
21384 record.attributes.flow_id = Some("flow:7".to_string());
21385 rig.sync(vec![record]).await;
21386 for opener in [OWNER, AFT] {
21387 let route = rig
21388 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21389 .await;
21390 assert!(rig.live(&route));
21391 assert_eq!(
21392 route.stamp().unwrap().attributes.flow_id.as_deref(),
21393 Some("flow:7")
21394 );
21395 }
21396 }
21397
21398 #[tokio::test]
21399 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
21400 use std::future::Future;
21401
21402 let mut rig = rig_with_flow_support(true).await;
21403 let mut record = session(1);
21404 record.attributes.flow_id = Some("flow:7".to_string());
21405 rig.sync(vec![record]).await;
21406 let (client, mut client_rx, frame) =
21407 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
21408 for _ in 0..64 {
21411 client.egress.try_send(route_bind_ack(1)).unwrap();
21412 }
21413 let handler = rig.handler.clone();
21414 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21415 std::future::poll_fn(|cx| {
21416 assert!(open.as_mut().poll(cx).is_pending());
21417 std::task::Poll::Ready(())
21418 })
21419 .await;
21420 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21421
21422 let old_connection = rig.modules[PLEXUS].0.connection_id;
21423 rig.handler.cleanup_connection(old_connection).unwrap();
21424 let (replacement, mut replacement_rx) = wide_ctx(200);
21425 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
21426 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
21427 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
21428 let hello = Frame::build(
21429 FrameType::Hello,
21430 control_flags(),
21431 0,
21432 0,
21433 200,
21434 serde_json::to_vec(&body).unwrap(),
21435 )
21436 .unwrap();
21437 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
21438
21439 client_rx.try_recv().unwrap();
21440 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21441 .await
21442 .expect("the replacement is refused without waiting for a bind ack")
21443 .unwrap();
21444 assert_eq!(replies.len(), 1);
21445 let body = parse_error(&replies[0]);
21446 assert_eq!(body["code"], "target_flow_unsupported");
21447 for required in [PLEXUS, "flow-scopes/v1"] {
21448 assert!(body["message"].as_str().unwrap().contains(required));
21449 }
21450 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
21451 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21452 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21453 assert!(rig
21454 .handler
21455 .registry
21456 .get_module_by_connection(replacement.connection_id)
21457 .unwrap()
21458 .is_some());
21459 }
21460
21461 #[tokio::test]
21462 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
21463 use std::future::Future;
21464 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
21465 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21466 const NEXT_HASH: &str =
21467 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21468 let mut rig = rig().await;
21469 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
21470 record.carriers.push(ScopeCarrier::new(Principal::Direct));
21471 rig.sync(vec![record.clone()]).await;
21472 let (client, mut rx, frame) = rig.open_frame(
21473 None,
21474 PLEXUS,
21475 Some(
21476 rig.selector("s", Some(1))
21477 .with_bind_proof(Some(PROOF.into())),
21478 ),
21479 );
21480 for _ in 0..64 {
21483 client.egress.try_send(route_bind_ack(1)).unwrap();
21484 }
21485 let handler = rig.handler.clone();
21486 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21487 std::future::poll_fn(|cx| {
21488 assert!(open.as_mut().poll(cx).is_pending());
21489 std::task::Poll::Ready(())
21490 })
21491 .await;
21492 record.bind_proof_sha256 = Some(NEXT_HASH.into());
21493 rig.sync(vec![record]).await;
21494 rx.try_recv().unwrap();
21495 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21496 .await
21497 .unwrap()
21498 .unwrap();
21499 assert_eq!(
21500 parse_error(&replies[0])["code"],
21501 error_codes::SCOPE_BIND_PROOF_MISMATCH
21502 );
21503 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
21504 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21505 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21506 }
21507
21508 #[tokio::test]
21509 async fn scope_bind_proof_reserved_openers_ignore_proof() {
21510 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21511 let mut rig = rig().await;
21512 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21513 .await;
21514 for opener in [OWNER, AFT] {
21515 for proof in [None, Some("malformed+proof=")] {
21516 let route = rig
21517 .bound(
21518 Some(opener),
21519 PLEXUS,
21520 Some(
21521 rig.selector("s", Some(1))
21522 .with_bind_proof(proof.map(str::to_string)),
21523 ),
21524 )
21525 .await;
21526 assert!(rig.live(&route));
21527 }
21528 }
21529 }
21530
21531 #[tokio::test]
21532 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
21533 use std::future::Future;
21534 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21535 const NEXT_HASH: &str =
21536 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21537 let mut rig = rig().await;
21538 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21539 .await;
21540 let (client, mut rx, frame) =
21541 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
21542 for _ in 0..64 {
21543 client.egress.try_send(route_bind_ack(1)).unwrap();
21544 }
21545 let handler = rig.handler.clone();
21546 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21547 std::future::poll_fn(|cx| {
21548 assert!(open.as_mut().poll(cx).is_pending());
21549 std::task::Poll::Ready(())
21550 })
21551 .await;
21552 rig.sync(vec![
21553 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
21554 ])
21555 .await;
21556 rx.try_recv().unwrap();
21557 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21558 let bind = std::future::poll_fn(|cx| {
21559 assert!(open.as_mut().poll(cx).is_pending());
21560 module_rx.poll_recv(cx)
21561 })
21562 .await
21563 .unwrap()
21564 .frame;
21565 handler
21566 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
21567 .await
21568 .unwrap();
21569 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21570 .await
21571 .unwrap()
21572 .unwrap();
21573 assert_eq!(replies.len(), 1);
21574 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
21577 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21578 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
21579 let route = rig
21580 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
21581 .await;
21582 assert!(rig.live(&route));
21583 }
21584
21585 #[tokio::test]
21586 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
21587 {
21588 let mut rig = rig_with_flow_support(false).await;
21589 rig.sync(vec![session(1)]).await;
21590 let route = rig
21591 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21592 .await;
21593 assert!(rig.live(&route));
21594 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
21595 let mut record = session(1);
21596 record.attributes.flow_id = Some("flow:7".to_string());
21597 rig.sync(vec![record]).await;
21598 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
21599 assert!(rig.live(&unscoped));
21600 assert_eq!(unscoped.stamp(), None);
21601 }
21602
21603 #[tokio::test]
21604 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
21605 let mut rig = rig().await;
21606 let mut record = session(1);
21607 record.attributes.flow_id = Some("flow:7".to_string());
21608 rig.sync(vec![record.clone()]).await;
21609 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21610 let mut owner_route = rig
21611 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21612 .await;
21613 let mut carrier_route = rig
21614 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
21615 .await;
21616 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21617 rig.sync(vec![record.clone()]).await;
21618 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21619 assert!(rig.live(&owner_route) && owner_route.untouched());
21620 assert!(rig.live(&carrier_route) && carrier_route.untouched());
21621
21622 record.attributes.flow_id = Some("flow:8".to_string());
21623 rig.sync(vec![record]).await;
21624 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
21625 let before = before.unwrap();
21626 assert_eq!(after.scope_epoch, before.scope_epoch);
21627 assert!(after.version > before.version);
21628 for route in [&mut owner_route, &mut carrier_route] {
21629 assert!(!rig.live(route));
21630 assert_eq!(
21631 route.closed_reason(),
21632 RouteCloseReason::ScopeDelegationChanged
21633 );
21634 }
21635 assert!(rig.live(&unscoped) && unscoped.untouched());
21636 for target in [PLEXUS, OTHER] {
21639 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
21640 let goodbye = module_rx
21641 .try_recv()
21642 .expect("the provider sees the drain")
21643 .frame;
21644 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21645 assert!(module_rx.try_recv().is_err());
21646 }
21647 let rebound = rig
21648 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21649 .await;
21650 assert_eq!(
21651 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
21652 Some("flow:8")
21653 );
21654 }
21655
21656 #[tokio::test]
21661 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
21662 let mut rig = rig().await;
21663 rig.sync(vec![session(1)]).await;
21664 let mut cotenant = rig
21665 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21666 .await;
21667
21668 let mut changed = session(1);
21669 changed.child_owners.push(Principal::Reserved {
21670 module_id: MAGIC.to_string(),
21671 });
21672 let mut ended = None;
21673 for (code, next) in [
21674 (error_codes::SCOPE_CHANGED, vec![changed]),
21675 (error_codes::SCOPE_ENDED, Vec::new()),
21676 ] {
21677 let relayed = rig
21678 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21679 .await;
21680 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
21681 ended = Some(next.is_empty());
21682 rig.sync(next).await;
21683 rig.ack(&relayed).await;
21684 let replies = relayed.task.await.unwrap();
21685 assert_eq!(replies.len(), 1, "{replies:?}");
21686 assert_eq!(parse_error(&replies[0])["code"], code);
21687 assert_eq!(
21688 subc_protocol::error_codes::is_retryable_route_open(code),
21689 code == error_codes::SCOPE_CHANGED
21690 );
21691 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21692 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21694 let mut goodbyes = Vec::new();
21697 while let Ok(outbound) = plexus_rx.try_recv() {
21698 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
21699 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
21700 }
21701 assert!(
21702 goodbyes.contains(&(bind_channel, bind_epoch)),
21703 "{goodbyes:?}"
21704 );
21705 assert!(rig
21706 .handler
21707 .registry
21708 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
21709 .unwrap()
21710 .is_some());
21711 }
21712 assert_eq!(ended, Some(true));
21713 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
21716 }
21717
21718 #[tokio::test]
21721 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
21722 struct Case {
21723 name: &'static str,
21724 change: fn(&mut ScopeRecord),
21725 closed: [Option<RouteCloseReason>; 4],
21728 }
21729 use RouteCloseReason::*;
21730 let cases = [
21731 Case {
21732 name: "a carrier entry removed",
21733 change: |r| {
21734 r.carriers.retain(|c| {
21735 c.principal
21736 != Principal::Reserved {
21737 module_id: AFT.to_string(),
21738 }
21739 })
21740 },
21741 closed: [None, Some(ScopeCarrierRemoved), None, None],
21742 },
21743 Case {
21744 name: "a target removed from a carrier",
21745 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
21746 closed: [None, None, None, Some(ScopeCarrierRemoved)],
21747 },
21748 Case {
21749 name: "a bare carrier narrowed to targets",
21750 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
21751 closed: [None, Some(ScopeCarrierRemoved), None, None],
21752 },
21753 Case {
21754 name: "delegates turned off",
21755 change: |r| r.attributes.delegates = false,
21756 closed: [Some(ScopeDelegationChanged); 4],
21757 },
21758 Case {
21759 name: "agent_id changed",
21760 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
21761 closed: [Some(ScopeDelegationChanged); 4],
21762 },
21763 Case {
21764 name: "a carrier added, child owners changed, the record re-sent",
21765 change: |r| {
21766 r.carriers.push(carrier(MAGIC, None));
21767 r.child_owners.push(Principal::Reserved {
21768 module_id: MAGIC.to_string(),
21769 });
21770 },
21771 closed: [None; 4],
21772 },
21773 Case {
21774 name: "a target added",
21775 change: |r| {
21776 r.carriers[1]
21777 .targets
21778 .as_mut()
21779 .unwrap()
21780 .push("third".to_string())
21781 },
21782 closed: [None; 4],
21783 },
21784 Case {
21785 name: "delegates turned on",
21786 change: |r| r.attributes.delegates = true,
21787 closed: [None; 4],
21788 },
21789 ];
21790 for case in cases {
21791 let mut rig = rig().await;
21792 rig.sync(vec![session(1)]).await;
21793 let scope = || Some(rig_selector("s", Some(1)));
21794 let mut routes = [
21795 rig.bound(Some(OWNER), PLEXUS, scope()).await,
21796 rig.bound(Some(AFT), PLEXUS, scope()).await,
21797 rig.bound(Some(BROCA), PLEXUS, scope()).await,
21798 rig.bound(Some(BROCA), OTHER, scope()).await,
21799 ];
21800 let mut record = session(1);
21801 (case.change)(&mut record);
21802 rig.sync(vec![record]).await;
21803 for (index, expected) in case.closed.iter().enumerate() {
21804 let route = &mut routes[index];
21805 match expected {
21806 Some(reason) => {
21807 assert!(!rig.live(route), "{}: route {index} still live", case.name);
21808 assert_eq!(
21809 route.closed_reason(),
21810 *reason,
21811 "{}: route {index}",
21812 case.name
21813 );
21814 }
21815 None => {
21816 assert!(rig.live(route), "{}: route {index} closed", case.name);
21817 assert!(
21818 route.untouched(),
21819 "{}: route {index} was told something",
21820 case.name
21821 );
21822 }
21823 }
21824 }
21825 }
21826 }
21827
21828 #[tokio::test]
21829 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
21830 for next in [Vec::new(), vec![session(2)]] {
21832 let mut rig = rig().await;
21833 rig.sync(vec![session(1)]).await;
21834 let mut route = rig
21835 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21836 .await;
21837 rig.sync(next).await;
21838 assert!(!rig.live(&route));
21839 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
21840 }
21841
21842 let mut rig = rig().await;
21845 let mut child = session(1);
21846 child.scope_ref = "child".to_string();
21847 child.kind = ScopeKind::Worker;
21848 child.parent = Some(ScopeParent::new(
21849 Principal::Reserved {
21850 module_id: OWNER.to_string(),
21851 },
21852 "s".to_string(),
21853 1,
21854 ));
21855 rig.sync(vec![session(1), child.clone()]).await;
21856 let mut child_route = rig
21857 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
21858 .await;
21859 assert_eq!(
21860 child_route.stamp().unwrap().parent_state,
21861 Some(ParentState::Linked)
21862 );
21863 rig.sync(vec![child]).await;
21864 assert!(!rig.live(&child_route));
21865 assert_eq!(
21866 child_route.closed_reason(),
21867 RouteCloseReason::ScopeParentEnded
21868 );
21869 }
21870
21871 #[tokio::test]
21872 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
21873 ) {
21874 let mut rig = rig().await;
21875 rig.sync(vec![session(1)]).await;
21876 let mut route = rig
21877 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21878 .await;
21879 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21880 rig.sync(vec![session(1)]).await;
21881 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21882 assert!(rig.live(&route) && route.untouched());
21883
21884 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
21888 .forwarding
21889 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21890 .unwrap()
21891 else {
21892 panic!("the route is bound");
21893 };
21894 binding.flow.acquire_tagged(9, false).await.unwrap();
21895 let mut widened = session(1);
21896 widened.carriers.push(carrier(MAGIC, None));
21897 rig.sync(vec![widened]).await;
21898 assert!(rig.live(&route) && route.untouched());
21899 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21900 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
21901 assert_eq!(binding.flow.in_flight(), 1);
21904 binding
21905 .flow
21906 .acquire_tagged(10, false)
21907 .await
21908 .expect("the flow is still open");
21909 }
21910
21911 #[tokio::test]
21914 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
21915 let mut rig = rig().await;
21916 rig.sync(vec![session(1)]).await;
21917 let mut on_incumbent = rig
21918 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21919 .await;
21920
21921 let (candidate, _candidate_rx) = wide_ctx(9);
21924 let registration = rig
21925 .handler
21926 .registry
21927 .register_candidate_with_control_ops(
21928 manifest(PLEXUS, PROTOCOL_VERSION),
21929 PROTOCOL_VERSION,
21930 candidate.connection_id,
21931 module_baseline_control_ops(),
21932 )
21933 .unwrap();
21934 rig.forwarding
21935 .register_candidate_module_connection(
21936 candidate.connection_id,
21937 PLEXUS.to_string(),
21938 PROTOCOL_VERSION,
21939 manifest_concurrency(®istration.manifest),
21940 candidate.egress.clone(),
21941 )
21942 .unwrap();
21943 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
21944 rig.handler
21945 .registry
21946 .promote_candidate(PLEXUS)
21947 .unwrap()
21948 .unwrap();
21949 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
21950
21951 rig.sync(Vec::new()).await;
21952 assert!(!rig.live(&on_incumbent));
21953 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
21954 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21955 let goodbye = incumbent_rx
21956 .try_recv()
21957 .expect("the superseded endpoint is told")
21958 .frame;
21959 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21960 }
21961 }
21962}
21963
21964#[cfg(test)]
21965mod concurrency_default_exposure_tests {
21966 use super::*;
21967
21968 fn hello_body(role_json: &str) -> Vec<u8> {
21969 format!(
21970 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
21971 )
21972 .into_bytes()
21973 }
21974
21975 fn manifest_from(body: &[u8]) -> ModuleManifest {
21976 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
21977 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
21978 .expect("manifest parses")
21979 }
21980
21981 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
21982
21983 #[test]
21984 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
21985 let body = hello_body(&format!(
21986 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
21987 ));
21988 let manifest = manifest_from(&body);
21989 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21992 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
21993 }
21994
21995 #[test]
21996 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
21997 let body = hello_body(&format!(
21998 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
21999 ));
22000 let manifest = manifest_from(&body);
22001 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
22002 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
22003 }
22004
22005 #[test]
22006 fn non_management_roles_are_never_reported() {
22007 let body = hello_body(
22008 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
22009 );
22010 let manifest = manifest_from(&body);
22011 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
22012 }
22013}