1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26 error_codes,
27 manifest::{
28 validate_hello_capability_grammar, validate_hello_confirm_operations,
29 validate_hello_display_name, validate_hello_event_declarations,
30 validate_hello_flow_operations, validate_hello_operation_deferral,
31 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32 validate_hello_self_signal_declarations, validate_role_declarations_value,
33 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34 ProviderRole,
35 },
36 peer::{
37 ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY,
38 PEER_BUILD_CAPABILITY,
39 },
40 provider_declarations::ProviderDeclarations,
41 scope::{
42 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
43 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
44 SCOPE_SYNC_OP,
45 },
46 session::{
47 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
48 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
49 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
50 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
51 },
52 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
53 Priority, RouteTarget, PROTOCOL_VERSION,
54};
55use tokio::time::{timeout_at, Instant};
56use tracing::{debug, info, instrument::WithSubscriber, warn};
57
58use crate::{
59 caller_identity::CallerIdentity,
60 capability_requirements::{
61 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
62 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
63 RequirementStatus, RuntimeModule,
64 },
65 child_roster::DaemonShutdownFlag,
66 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
67 forwarding::{
68 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
69 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
70 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
71 },
72 observability::{
73 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
74 },
75 privileged_prompt::{prompt_text, PrivilegedOperation},
76 provenance::{
77 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
78 },
79 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
80 router::{daemon_error_flags, RouteCtx, RouterError},
81 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
82 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
83 stderr_tail::{CaptureState, TailEntry},
84 supervise::{
85 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
86 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
87 },
88 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
89};
90
91pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
97
98const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
99const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
100const CAP_PING_PONG: &str = "ping_pong_v1";
101const CAP_SESSION_ATTACH: &str = "session_attach_v1";
102const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
103
104const SUBC_CONTROL_OPS: &[&str] = &[
105 "claim.mint",
106 "claim.present",
107 ops::DAEMON_SHUTDOWN_INTENT,
108 ops::ACTIVITY_RECORD,
109 ops::ACTIVITY_QUERY,
110 ops::ACTIVITY_SUBSCRIBE,
111 ops::SERVER_DESCRIBE,
112 ops::CATALOG_LIST,
113 ops::CATALOG_WATCH,
114 ops::ROUTE_OPEN,
115 ops::ROUTE_POLL,
116 ops::ROUTE_CLOSING,
117 ops::ROUTE_CLOSED,
118 ops::SUPERVISOR_LIST,
119 ops::SUPERVISOR_RESTART,
120 ops::SUPERVISOR_SWAP,
121 ops::SUPERVISOR_RELOAD,
122 ops::SUPERVISOR_RESCAN,
123 ops::SUPERVISOR_RELEASE_RESERVED,
124 ops::SUPERVISOR_SET_ENABLED,
125 ops::SUPERVISOR_HEALTH_PROBE,
126 ops::SUPERVISOR_HEALTH,
127 ops::SUPERVISOR_STDERR_TAIL,
128 ops::SUPERVISOR_TERMINALS,
129 ops::SUPERVISOR_ROUTES,
130 ops::SUPERVISOR_PROVENANCE,
131 ops::SUPERVISOR_SPAWN_SNAPSHOT,
132 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
133];
134
135const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[
142 ops::SUPERVISOR_RESCAN_CHECKED,
143 "config.status",
144 "config.plan",
145 "config.apply",
146 "config.stamp",
147 "config.unstamp",
148 "config.history",
149 "config.rollback",
150];
151
152mod caller_attestation;
153mod channel_operations;
154#[cfg(all(test, unix, feature = "test-support"))]
155#[path = "control/channel_operations_tests.rs"]
156mod channel_operations_tests;
157pub(crate) mod claims;
158#[cfg(test)]
159#[path = "control/config_advertisement_tests.rs"]
160mod config_advertisement_tests;
161mod config_gate;
162use caller_attestation::{caller_for_target, scope_acts_for_another};
163#[cfg(all(test, unix, feature = "test-support"))]
164#[path = "control/caller_attestation_tests.rs"]
165mod caller_attestation_tests;
166mod scope_expiry;
167mod scope_watch;
168mod usage;
169
170mod agent_commands;
171pub(crate) mod catalog;
172mod rescan_checked;
173use rescan_checked::{RescanMode, RescanPlan};
174
175const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
176 "claim.mint",
177 "claim.present",
178 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
179 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
180 "supervisor.live_roots",
181 "supervisor.agent_commands",
182 SCOPE_SYNC_OP,
183 SCOPE_APPLY_OP,
184 SCOPE_DESCRIBE_OP,
185 ops::SCOPE_WATCH,
186 subc_protocol::scope::ROUTE_AUTHORIZE_OP,
187 "operator.confirm",
188];
189
190const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
194
195const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
196
197pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
205
206pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
221
222pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
240
241const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
242const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
243
244fn reload_verdict(
245 configured: &Path,
246 spawned_from: Option<&Path>,
247 image: subc_control::RunningImageAgreement,
248) -> PendingReloadVerdict {
249 let path = match spawned_from {
250 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
251 ReloadPathAgreement::Match
252 }
253 Some(spawned_from) => ReloadPathAgreement::Mismatch {
254 configured: configured.to_path_buf(),
255 spawned_from: spawned_from.to_path_buf(),
256 },
257 None => ReloadPathAgreement::Unavailable {
258 reason: if matches!(
259 image,
260 subc_control::RunningImageAgreement::Unavailable {
261 reason: subc_control::RunningImageUnavailableReason::NotRunning
262 }
263 ) {
264 ReloadPathUnavailableReason::NotRunning
265 } else {
266 ReloadPathUnavailableReason::SpawnedPathUnavailable
267 },
268 },
269 };
270 PendingReloadVerdict::new(path, image)
271}
272
273fn launch_spec_agreement(
274 configured: &crate::supervise::ModuleSpec,
275 running: bool,
276 spawned: Option<&crate::supervise::LaunchSpecDigest>,
277) -> subc_control::LaunchSpecAgreement {
278 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
279 if !running {
280 return Agreement::Unavailable {
281 reason: Reason::NotRunning,
282 };
283 }
284 match spawned {
285 None => Agreement::Unavailable {
286 reason: Reason::SpawnedSpecUnavailable,
287 },
288 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
289 Agreement::Match
290 }
291 Some(_) => Agreement::Mismatch,
292 }
293}
294
295fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
296 if configured == spawned_from {
297 return true;
298 }
299 #[cfg(windows)]
300 {
301 match (
307 std::fs::canonicalize(configured),
308 std::fs::canonicalize(spawned_from),
309 ) {
310 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
311 _ => false,
312 }
313 }
314 #[cfg(not(windows))]
315 {
316 false
317 }
318}
319
320#[derive(Clone)]
321struct DaemonProvenanceFacts {
322 build: DaemonBuildProvenance,
323 pid: Option<u32>,
324 started_at_ms: Option<u64>,
325 start_clock: Option<crate::clock::StartClock>,
326 executable_path: Option<PathBuf>,
327 executable_identity: Option<SpawnedFileIdentity>,
328 process_start_time: Option<u64>,
329 probe: ExecutableIdentityProbe,
330}
331
332impl Default for DaemonProvenanceFacts {
333 fn default() -> Self {
334 Self {
335 build: DaemonBuildProvenance {
336 build_git_sha: None,
337 build_lock_digest: None,
338 },
339 pid: None,
340 started_at_ms: None,
341 start_clock: None,
342 executable_path: None,
343 executable_identity: None,
344 process_start_time: None,
345 probe: ExecutableIdentityProbe::default(),
346 }
347 }
348}
349
350#[derive(Debug, Clone)]
351struct SupervisorRescanContext {
352 supervisor: Supervisor,
353 config_path: PathBuf,
354 configured_port: Option<u16>,
355 storage_config: Option<crate::daemon_config::StorageConfig>,
356 admission_facts_carrier_module_id: Option<String>,
357 admission_facts_targets: Option<Vec<String>>,
358 scope_authority_owners: Vec<String>,
359 federated_scope_owner: String,
360 privileged_operator_prompt: PrivilegedOperatorPrompt,
361 generation: Arc<std::sync::atomic::AtomicU64>,
367 module_entries: Arc<Mutex<rescan_checked::ModuleEntries>>,
368}
369
370const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
391 "reloading",
392 "supervisor_not_live",
393 "registration_not_active",
394 "no_forwarding_connection",
395 "relay_send_failed",
396];
397
398#[cfg(test)]
400struct ActivityRequestPause {
401 connection: ConnectionId,
402 started: tokio::sync::Notify,
403 cancelled: tokio::sync::Notify,
404 cleanup_started: std::sync::atomic::AtomicBool,
405 cancelled_after_cleanup: std::sync::atomic::AtomicBool,
406}
407
408#[cfg(test)]
409struct ActivityRequestCancellation(Arc<ActivityRequestPause>);
410
411#[cfg(test)]
412impl Drop for ActivityRequestCancellation {
413 fn drop(&mut self) {
414 self.0.cancelled_after_cleanup.store(
415 self.0
416 .cleanup_started
417 .load(std::sync::atomic::Ordering::Acquire),
418 std::sync::atomic::Ordering::Release,
419 );
420 self.0.cancelled.notify_one();
421 }
422}
423
424#[cfg(test)]
426type TestPause = (
427 tokio::sync::oneshot::Sender<()>,
428 std::sync::mpsc::Receiver<()>,
429);
430
431#[cfg(test)]
433type TestHook = Box<dyn FnOnce() + Send>;
434
435#[cfg(test)]
437type AsyncTestPause = (
438 tokio::sync::oneshot::Sender<()>,
439 tokio::sync::oneshot::Receiver<()>,
440);
441
442#[derive(Clone)]
444pub struct ControlHandler {
445 #[cfg(test)]
446 activity_request_pause: Arc<Mutex<Option<Arc<ActivityRequestPause>>>>,
447 #[cfg(test)]
451 cleanup_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
452 #[cfg(test)]
455 after_nonce_attestation: Arc<Mutex<Option<TestHook>>>,
456 #[cfg(test)]
459 hello_ingress_pause: Arc<Mutex<Option<AsyncTestPause>>>,
460 #[cfg(test)]
462 agent_commands_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
463 claims: Arc<claims::Claims>,
464 registry: Arc<Registry>,
465 control_usage: Arc<Mutex<usage::ControlUsage>>,
466 catalog_cache: Arc<Mutex<catalog::CatalogCache>>,
467 catalog_readiness_revision: Arc<std::sync::atomic::AtomicU64>,
468 config_runtime: Option<Arc<crate::config_gate::runtime::RuntimeOwner>>,
469 config_review_admission: Arc<dyn crate::config_gate::ReviewAdmission>,
470 config_review_admission_is_default: bool,
471 activity: Option<crate::activity::ActivityLedger>,
472 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
473 catalog_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
474 scope_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
475 forwarding: Arc<ForwardingTable>,
476 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
477 supervisor: SupervisorHandle,
478 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
479 agent_declarations: agent_commands::Queues,
480 list_admission: Arc<tokio::sync::Semaphore>,
481 subc_capabilities: Arc<[String]>,
482 route_bind_relay_timeout: Duration,
486 route_bind_relay_timeouts: BTreeMap<String, Duration>,
490 route_bind_breakers: RouteBindBreakers,
493 route_bind_concurrency: RouteBindConcurrency,
496 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
501 route_bind_breaker_threshold: u32,
503 route_bind_breaker_cooldown: Duration,
505 health_probe_timeout: Duration,
506 storage_config: Option<crate::daemon_config::StorageConfig>,
509 machine_id: Option<crate::machine_id::MachineId>,
513 admission_facts_carrier_module_id: Option<String>,
514 admission_facts_targets: Option<Vec<String>>,
515 scopes: Arc<RwLock<ScopeTable>>,
519 scope_deadline_added: Arc<tokio::sync::Notify>,
523 scope_authority_owners: Vec<String>,
526 federated_scope_owner: String,
527 privileged_operator_prompt: PrivilegedOperatorPrompt,
529 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
532 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
533 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
534 captured_peers: Arc<Mutex<HashMap<ConnectionId, subc_os::local_peer::CapturedPeer>>>,
535 config_desktop_stamps:
536 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
537 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
538 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
539 pub(crate) operator_apps:
540 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
541 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
546 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
552 #[cfg(test)]
554 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
555 #[cfg(test)]
556 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
557 #[cfg(test)]
558 recovery_exit_test_after_step: Option<fn(usize) -> Result<(), crate::config_gate::GateError>>,
559 #[cfg(test)]
560 recovery_exit_test_completed: Option<fn(u64)>,
561 #[cfg(test)]
562 apply_reconcile_pause: Arc<Mutex<Option<AsyncTestPause>>>,
563 #[cfg(test)]
564 recovery_exit_test_quiet_retry: bool,
565 #[cfg(test)]
566 recovery_exit_test_retry_pause: Option<fn(u64) -> Option<PathBuf>>,
567 #[cfg(test)]
568 config_delivery_test_wakes:
569 Option<tokio::sync::mpsc::UnboundedSender<(&'static str, tokio::time::Instant)>>,
570 #[cfg(all(test, unix))]
571 config_delivery_test_idle:
572 Option<tokio::sync::mpsc::UnboundedSender<tokio::sync::oneshot::Sender<()>>>,
573 #[allow(dead_code)]
577 #[cfg(windows)]
578 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
579 #[allow(dead_code)]
580 #[cfg(windows)]
581 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
582 #[allow(dead_code)]
586 #[cfg(windows)]
587 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
588 #[cfg(all(test, windows))]
591 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
592 #[cfg(test)]
593 caller_lookup: Option<TestCallerLookup>,
594 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
597 rescan: Option<SupervisorRescanContext>,
598 connected_clients: ConnectedClients,
599 counters: DaemonCounters,
600 local_endpoint_unavailable: Option<String>,
603 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
604 daemon_shutdown: DaemonShutdownFlag,
609 daemon_provenance: DaemonProvenanceFacts,
610 #[cfg(test)]
611 control_dispatch_delay: Option<Duration>,
612 #[cfg(test)]
613 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
614 #[cfg(test)]
618 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
619 #[cfg(test)]
625 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
626 #[cfg(test)]
627 hello_ack_to_publication_hold: Option<Arc<std::sync::Barrier>>,
628 #[cfg(test)]
629 hello_before_publication_hold: Option<Arc<std::sync::Barrier>>,
630 #[cfg(test)]
631 hello_after_launch_check_hold: Option<Arc<std::sync::Barrier>>,
632 #[cfg(test)]
633 hello_inside_publication_hold: Option<Arc<std::sync::Barrier>>,
634}
635
636impl fmt::Debug for ControlHandler {
637 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
638 f.debug_struct("ControlHandler")
639 .field("registry", &self.registry)
640 .field("forwarding", &self.forwarding)
641 .field("process_liveness", &self.process_liveness.is_some())
642 .field("supervisor", &self.supervisor)
643 .field("subc_capabilities", &self.subc_capabilities)
644 .finish()
645 }
646}
647
648struct RouteOpenRequest {
649 target: RouteTarget,
650 identity: BindIdentity,
651 consumer_identity: Option<ConsumerIdentity>,
652 consumer_capabilities: Option<Vec<String>>,
653 role_versions: Option<BTreeMap<String, String>>,
654 busy_refusals: Option<bool>,
655 admission_facts: Option<serde_json::Value>,
656 scope: Option<ScopeSelector>,
657 accepts_close_reasons: Option<Vec<String>>,
658}
659
660#[derive(Clone, Copy)]
661enum RouteOpenPhase {
662 Admission,
663 ProjectRoot,
664 EgressReservation,
665 BindSend,
666 BindAckWait,
667}
668
669struct RouteOpenTimings {
673 module_id: String,
674 principal: String,
675 admission: Duration,
676 project_root: Duration,
677 egress_reservation: Duration,
678 bind_send: Duration,
679 bind_ack_wait: Duration,
680 phase: Option<RouteOpenPhase>,
681 phase_started_at: Instant,
682}
683
684impl RouteOpenTimings {
685 fn new() -> Self {
686 Self {
687 module_id: String::new(),
688 principal: "unresolved".to_string(),
690 admission: Duration::ZERO,
691 project_root: Duration::ZERO,
692 egress_reservation: Duration::ZERO,
693 bind_send: Duration::ZERO,
694 bind_ack_wait: Duration::ZERO,
695 phase: Some(RouteOpenPhase::Admission),
696 phase_started_at: Instant::now(),
697 }
698 }
699
700 fn finish(&mut self) {
701 let elapsed = self.phase_started_at.elapsed();
702 match self.phase.take() {
703 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
704 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
705 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
706 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
707 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
708 None => {}
709 }
710 }
711
712 fn enter(&mut self, phase: RouteOpenPhase) {
713 self.finish();
714 self.phase = Some(phase);
715 self.phase_started_at = Instant::now();
716 }
717}
718
719struct RouteBindReservationGuard {
720 forwarding: Arc<ForwardingTable>,
721 endpoint: ModuleEndpointId,
722 relay_corr: u64,
723 armed: bool,
724}
725
726struct ModuleControlRpcGuard {
727 forwarding: Arc<ForwardingTable>,
728 endpoint: ModuleEndpointId,
729 corr: u64,
730 armed: bool,
731}
732
733impl ModuleControlRpcGuard {
734 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
735 Self {
736 forwarding,
737 endpoint,
738 corr,
739 armed: true,
740 }
741 }
742
743 fn disarm(&mut self) {
744 self.armed = false;
745 }
746}
747
748impl Drop for ModuleControlRpcGuard {
749 fn drop(&mut self) {
750 if self.armed {
751 let _ = self
752 .forwarding
753 .cancel_module_control_rpc(self.endpoint, self.corr);
754 }
755 }
756}
757
758impl RouteBindReservationGuard {
759 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
760 Self {
761 forwarding,
762 endpoint,
763 relay_corr,
764 armed: true,
765 }
766 }
767
768 fn release_and_disarm(&mut self) {
769 if !self.armed {
770 return;
771 }
772 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
773 self.endpoint,
774 self.relay_corr,
775 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
776 ) {
777 send_goodbye_target_best_effort(
778 &self.forwarding.counters(),
779 &target,
780 "canceled route.bind",
781 );
782 }
783 self.armed = false;
784 }
785
786 fn disarm(&mut self) {
787 self.armed = false;
788 }
789}
790
791impl Drop for RouteBindReservationGuard {
792 fn drop(&mut self) {
793 self.release_and_disarm();
794 }
795}
796
797#[derive(Debug, Clone, Default)]
827pub(crate) struct RouteBindBreakers {
828 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
829}
830
831#[derive(Debug, Clone, Default)]
832pub(crate) struct RouteBindConcurrency {
833 modules: Arc<Mutex<HashMap<String, usize>>>,
834}
835
836struct RouteBindConcurrencyGuard {
837 concurrency: RouteBindConcurrency,
838 module_id: String,
839}
840
841impl RouteBindConcurrency {
842 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
845 let mut modules = self
846 .modules
847 .lock()
848 .expect("route.bind concurrency mutex poisoned");
849 let in_flight = modules.entry(module_id.to_string()).or_default();
850 if *in_flight >= limit {
851 return Err(*in_flight);
852 }
853 *in_flight += 1;
854 Ok(RouteBindConcurrencyGuard {
855 concurrency: self.clone(),
856 module_id: module_id.to_string(),
857 })
858 }
859}
860
861impl Drop for RouteBindConcurrencyGuard {
862 fn drop(&mut self) {
863 let mut modules = self
864 .concurrency
865 .modules
866 .lock()
867 .expect("route.bind concurrency mutex poisoned");
868 let remove = {
869 let in_flight = modules
870 .get_mut(&self.module_id)
871 .expect("admitted route.bind has a concurrency entry");
872 *in_flight -= 1;
873 *in_flight == 0
874 };
875 if remove {
876 modules.remove(&self.module_id);
877 }
878 }
879}
880
881#[derive(Debug, Default)]
882struct ModuleBreakerState {
883 consecutive_timeouts: u32,
885 cooldown_until: Option<Instant>,
888 probe_in_flight: Option<Arc<()>>,
893}
894
895enum RouteBindAdmission<'a> {
897 Admitted {
898 guard: RouteBindBreakerGuard<'a>,
899 probe: bool,
902 },
903 Refused {
904 consecutive_timeouts: u32,
905 retry_in: Duration,
909 probe_in_flight: bool,
910 },
911}
912
913struct RouteBindBreakerGuard<'a> {
920 breakers: RouteBindBreakers,
921 module_id: &'a str,
922 probe_token: Option<Arc<()>>,
923 settled: bool,
924}
925
926impl RouteBindBreakerGuard<'_> {
927 fn record_accepted(&mut self) -> bool {
931 self.settled = true;
932 self.breakers.record_accepted(self.module_id)
933 }
934
935 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
938 self.settled = true;
939 self.breakers.record_timeout(
940 self.module_id,
941 self.probe_token.as_ref(),
942 threshold,
943 cooldown,
944 )
945 }
946
947 fn record_inconclusive(&mut self) {
956 self.settled = true;
957 self.breakers
958 .record_inconclusive(self.module_id, self.probe_token.as_ref());
959 }
960}
961
962impl Drop for RouteBindBreakerGuard<'_> {
963 fn drop(&mut self) {
964 if !self.settled {
965 self.breakers
966 .record_inconclusive(self.module_id, self.probe_token.as_ref());
967 }
968 }
969}
970
971struct BreakerOpened {
975 consecutive_timeouts: u32,
976 reopened_after_probe: bool,
979}
980
981impl RouteBindBreakers {
982 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
983 self.modules
984 .lock()
985 .expect("route.bind breaker mutex poisoned")
986 }
987
988 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
991 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
992 probe: probe_token.is_some(),
993 guard: RouteBindBreakerGuard {
994 breakers: self.clone(),
995 module_id,
996 probe_token,
997 settled: false,
998 },
999 };
1000
1001 let mut modules = self.lock();
1002 let Some(state) = modules.get_mut(module_id) else {
1003 return admitted(None);
1004 };
1005 let Some(cooldown_until) = state.cooldown_until else {
1006 return admitted(None);
1007 };
1008 if state.probe_in_flight.is_some() {
1009 return RouteBindAdmission::Refused {
1010 consecutive_timeouts: state.consecutive_timeouts,
1011 retry_in: Duration::ZERO,
1012 probe_in_flight: true,
1013 };
1014 }
1015 let now = Instant::now();
1016 if now < cooldown_until {
1017 return RouteBindAdmission::Refused {
1018 consecutive_timeouts: state.consecutive_timeouts,
1019 retry_in: cooldown_until - now,
1020 probe_in_flight: false,
1021 };
1022 }
1023 let token = Arc::new(());
1024 state.probe_in_flight = Some(Arc::clone(&token));
1025 admitted(Some(token))
1026 }
1027
1028 fn record_accepted(&self, module_id: &str) -> bool {
1029 self.lock()
1030 .remove(module_id)
1031 .is_some_and(|state| state.cooldown_until.is_some())
1032 }
1033
1034 fn record_timeout(
1035 &self,
1036 module_id: &str,
1037 probe_token: Option<&Arc<()>>,
1038 threshold: u32,
1039 cooldown: Duration,
1040 ) -> Option<BreakerOpened> {
1041 let mut modules = self.lock();
1042 let state = modules.entry(module_id.to_string()).or_default();
1043 let was_open = state.cooldown_until.is_some();
1044 let was_probe = Self::owns_probe(state, probe_token);
1045 if was_probe {
1046 state.probe_in_flight = None;
1047 }
1048 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
1049 if state.consecutive_timeouts < threshold {
1050 return None;
1051 }
1052 state.cooldown_until = Some(Instant::now() + cooldown);
1053 Some(BreakerOpened {
1054 consecutive_timeouts: state.consecutive_timeouts,
1055 reopened_after_probe: was_open && was_probe,
1056 })
1057 }
1058
1059 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
1060 state
1064 .probe_in_flight
1065 .as_ref()
1066 .zip(token)
1067 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
1068 }
1069
1070 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
1071 if let Some(state) = self.lock().get_mut(module_id) {
1072 if Self::owns_probe(state, probe_token) {
1073 state.probe_in_flight = None;
1074 }
1075 }
1076 }
1077
1078 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
1093 self.lock()
1094 .remove(module_id)
1095 .map(|state| state.consecutive_timeouts)
1096 .filter(|discarded| *discarded > 0)
1097 }
1098
1099 fn open_snapshot(&self) -> Option<serde_json::Value> {
1106 let now = Instant::now();
1107 let modules = self.lock();
1108 let open = modules
1109 .iter()
1110 .filter_map(|(module_id, state)| {
1111 let cooldown_until = state.cooldown_until?;
1112 Some((
1113 module_id.clone(),
1114 serde_json::json!({
1115 "consecutive_timeouts": state.consecutive_timeouts,
1116 "cooldown_remaining_ms":
1117 cooldown_until.saturating_duration_since(now).as_millis() as u64,
1118 "probe_in_flight": state.probe_in_flight.is_some(),
1119 }),
1120 ))
1121 })
1122 .collect::<serde_json::Map<String, serde_json::Value>>();
1123 (!open.is_empty()).then_some(serde_json::Value::Object(open))
1124 }
1125}
1126
1127impl ControlHandler {
1128 pub(crate) fn install_registration_service(
1129 &self,
1130 router: std::sync::Weak<crate::Router>,
1131 auth: crate::server::ServerAuth,
1132 ) {
1133 self.supervisor.install_registration_service(router, auth);
1134 }
1135 pub fn new(registry: Arc<Registry>) -> Self {
1136 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
1137 }
1138
1139 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
1140 let counters = forwarding.counters();
1141 let route_bind_breakers = forwarding.route_bind_breakers();
1145 let route_bind_concurrency = forwarding.route_bind_concurrency();
1146 let route_outages = forwarding.route_outages();
1147 Self {
1148 registry,
1149 control_usage: Arc::new(Mutex::new(usage::ControlUsage::default())),
1150 catalog_cache: Arc::new(Mutex::new(catalog::CatalogCache::default())),
1151 catalog_readiness_revision: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1152 #[cfg(test)]
1153 activity_request_pause: Arc::new(Mutex::new(None)),
1154 #[cfg(test)]
1155 cleanup_pause: Arc::new(Mutex::new(None)),
1156 #[cfg(test)]
1157 after_nonce_attestation: Arc::new(Mutex::new(None)),
1158 #[cfg(test)]
1159 hello_ingress_pause: Arc::new(Mutex::new(None)),
1160 #[cfg(test)]
1161 agent_commands_pause: Arc::new(Mutex::new(None)),
1162 claims: Arc::new(claims::Claims::default()),
1163 config_runtime: None,
1164 config_review_admission: Arc::new(crate::config_gate::MissingReviewAdmission),
1165 config_review_admission_is_default: true,
1166 activity: None,
1167 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1168 catalog_streams: Arc::new(Mutex::new(HashMap::new())),
1169 scope_streams: Arc::new(Mutex::new(HashMap::new())),
1170 forwarding,
1171 process_liveness: None,
1172 supervisor: SupervisorHandle::new(),
1173 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1174 env!("CARGO_PKG_VERSION").to_owned(),
1175 )),
1176 agent_declarations: agent_commands::Queues::default(),
1177 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1178 subc_capabilities: Arc::from([
1179 CAP_MANIFEST_REGISTRATION.to_string(),
1180 CAP_CHANNEL_LIFECYCLE.to_string(),
1181 CAP_PING_PONG.to_string(),
1182 CAP_SESSION_ATTACH.to_string(),
1183 CAP_ADMISSION_FACTS_RELAY.to_string(),
1184 CAP_SCOPES_V1.to_string(),
1185 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1186 subc_protocol::peer::CALLER_ATTESTATION_CAPABILITY.to_string(),
1187 PEER_BUILD_CAPABILITY.to_string(),
1188 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1189 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1190 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1191 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1192 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1193 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1194 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1195 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1196 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1197 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1198 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1199 subc_protocol::ROUTE_CLOSE_REASONS_CAPABILITY.to_string(),
1200 ]),
1201 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1202 route_bind_relay_timeouts: BTreeMap::new(),
1203 route_bind_breakers,
1204 route_bind_concurrency,
1205 route_outages,
1206 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1207 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1208 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1209 storage_config: None,
1210 machine_id: None,
1211 admission_facts_carrier_module_id: None,
1212 admission_facts_targets: None,
1213 scopes: Arc::new(RwLock::new(ScopeTable::new(
1214 crate::daemon_config::default_scope_authority_owners(),
1215 ))),
1216 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1217 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1218 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1219 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1220 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1221 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1222 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1223 captured_peers: Arc::new(Mutex::new(HashMap::new())),
1224 config_desktop_stamps: Arc::new(Mutex::new(HashMap::new())),
1225 local_connections: Arc::new(Mutex::new(HashSet::new())),
1226 operator_app_policy: Default::default(),
1227 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1228 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1229 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1230 #[cfg(test)]
1231 app_open_test_platform: None,
1232 #[cfg(test)]
1233 app_open_test_launch: None,
1234 #[cfg(test)]
1235 recovery_exit_test_after_step: None,
1236 #[cfg(test)]
1237 recovery_exit_test_completed: None,
1238 #[cfg(test)]
1239 apply_reconcile_pause: Arc::new(Mutex::new(None)),
1240 #[cfg(test)]
1241 recovery_exit_test_quiet_retry: false,
1242 #[cfg(test)]
1243 recovery_exit_test_retry_pause: None,
1244 #[cfg(test)]
1245 config_delivery_test_wakes: None,
1246 #[cfg(all(test, unix))]
1247 config_delivery_test_idle: None,
1248 #[cfg(windows)]
1249 windows_verify: None,
1250 #[cfg(windows)]
1251 windows_flash: None,
1252 #[cfg(windows)]
1253 windows_track: None,
1254 #[cfg(all(test, windows))]
1255 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1256 #[cfg(test)]
1257 caller_lookup: None,
1258 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1259 rescan: None,
1260 connected_clients: ConnectedClients::new(),
1261 local_endpoint_unavailable: None,
1262 counters,
1263 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1264 daemon_shutdown: DaemonShutdownFlag::default(),
1265 daemon_provenance: DaemonProvenanceFacts::default(),
1266 #[cfg(test)]
1267 control_dispatch_delay: None,
1268 #[cfg(test)]
1269 provenance_probe_override: None,
1270 #[cfg(test)]
1271 route_bind_relay_sent_hold: None,
1272 #[cfg(test)]
1273 hello_registry_to_forwarding_hold: None,
1274 #[cfg(test)]
1275 hello_ack_to_publication_hold: None,
1276 #[cfg(test)]
1277 hello_before_publication_hold: None,
1278 #[cfg(test)]
1279 hello_after_launch_check_hold: None,
1280 #[cfg(test)]
1281 hello_inside_publication_hold: None,
1282 }
1283 }
1284
1285 pub fn with_storage_config(
1288 mut self,
1289 storage_config: Option<crate::daemon_config::StorageConfig>,
1290 ) -> Self {
1291 self.storage_config = storage_config;
1292 self
1293 }
1294
1295 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1298 self.activity = Some(crate::activity::ActivityLedger::open(
1299 run_dir.join("activity.jsonl"),
1300 incarnation,
1301 ));
1302 self.supervisor.set_activity(self.activity.clone());
1303 self
1304 }
1305
1306 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1309 self.machine_id = machine_id;
1310 self
1311 }
1312
1313 pub fn with_admission_facts_config(
1317 mut self,
1318 carrier_module_id: Option<String>,
1319 targets: Option<Vec<String>>,
1320 ) -> Self {
1321 self.admission_facts_carrier_module_id = carrier_module_id;
1322 self.admission_facts_targets = targets;
1323 self
1324 }
1325
1326 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1330 self.scopes = Arc::new(RwLock::new(
1331 ScopeTable::new(owners.iter().cloned())
1332 .with_federated_owner(self.federated_scope_owner.clone()),
1333 ));
1334 self.scope_authority_owners = owners;
1335 self
1336 }
1337
1338 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1340 self.scopes = Arc::new(RwLock::new(
1341 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1342 .with_federated_owner(owner.clone()),
1343 ));
1344 self.federated_scope_owner = owner;
1345 self
1346 }
1347
1348 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1350 self.privileged_operator_prompt = mode;
1351 self
1352 }
1353
1354 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1357 self.route_bind_relay_timeout = timeout;
1358 self
1359 }
1360
1361 pub fn with_route_bind_relay_timeouts(
1367 mut self,
1368 timeouts: impl IntoIterator<Item = (String, Duration)>,
1369 ) -> Self {
1370 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1371 self
1372 }
1373
1374 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1380 self.route_bind_relay_timeouts
1381 .get(module_id)
1382 .copied()
1383 .unwrap_or(self.route_bind_relay_timeout)
1384 }
1385
1386 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1394 self.route_bind_breaker_threshold = threshold.max(1);
1395 self.route_bind_breaker_cooldown = cooldown;
1396 self
1397 }
1398
1399 #[cfg(test)]
1400 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1401 self.health_probe_timeout = timeout;
1402 self
1403 }
1404
1405 #[cfg(test)]
1406 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1407 self.control_dispatch_delay = Some(delay);
1408 self
1409 }
1410
1411 #[cfg(test)]
1415 pub(crate) fn with_route_bind_relay_sent_hold(
1416 mut self,
1417 hold: Arc<tokio::sync::Barrier>,
1418 ) -> Self {
1419 self.route_bind_relay_sent_hold = Some(hold);
1420 self
1421 }
1422
1423 #[cfg(test)]
1428 pub(crate) fn with_hello_registry_to_forwarding_hold(
1429 mut self,
1430 hold: Arc<std::sync::Barrier>,
1431 ) -> Self {
1432 self.hello_registry_to_forwarding_hold = Some(hold);
1433 self
1434 }
1435
1436 #[cfg(test)]
1437 fn hold_between_registry_and_forwarding(&self) {
1438 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1439 hold.wait();
1440 hold.wait();
1441 }
1442 }
1443
1444 #[cfg(test)]
1447 fn hold_between_ack_and_publication(&self) {
1448 if let Some(hold) = &self.hello_ack_to_publication_hold {
1449 hold.wait();
1450 hold.wait();
1451 }
1452 }
1453
1454 #[cfg(all(test, unix))]
1455 pub(crate) fn with_hello_before_publication_hold(
1456 mut self,
1457 hold: Arc<std::sync::Barrier>,
1458 ) -> Self {
1459 self.hello_before_publication_hold = Some(hold);
1460 self
1461 }
1462
1463 #[cfg(all(test, unix))]
1464 pub(crate) fn with_hello_after_launch_check_hold(
1465 mut self,
1466 hold: Arc<std::sync::Barrier>,
1467 ) -> Self {
1468 self.hello_after_launch_check_hold = Some(hold);
1469 self
1470 }
1471
1472 #[cfg(all(test, unix))]
1473 pub(crate) fn with_hello_inside_publication_hold(
1474 mut self,
1475 hold: Arc<std::sync::Barrier>,
1476 ) -> Self {
1477 self.hello_inside_publication_hold = Some(hold);
1478 self
1479 }
1480
1481 pub fn with_process_liveness(
1482 mut self,
1483 process_liveness: Arc<dyn ModuleProcessLiveness>,
1484 ) -> Self {
1485 self.process_liveness = Some(process_liveness);
1486 self
1487 }
1488
1489 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1490 self.supervisor = supervisor;
1491 if let Some(owner) = &self.config_runtime {
1492 self.supervisor.install_binding_retirement(owner);
1493 }
1494 self.supervisor.set_activity(self.activity.clone());
1495 self
1496 }
1497
1498 pub fn with_daemon_provenance(
1499 mut self,
1500 pid: u32,
1501 started_at_ms: u64,
1502 executable_path: Option<PathBuf>,
1503 build_git_sha: Option<String>,
1504 build_lock_digest: Option<String>,
1505 ) -> Self {
1506 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1507 let process_start_time = process_start_time(pid);
1508 self.daemon_provenance = DaemonProvenanceFacts {
1509 build: DaemonBuildProvenance {
1510 build_git_sha,
1511 build_lock_digest,
1512 },
1513 pid: Some(pid),
1514 started_at_ms: Some(started_at_ms),
1515 start_clock: None,
1516 executable_path,
1517 executable_identity,
1518 process_start_time,
1519 probe: ExecutableIdentityProbe::default(),
1520 };
1521 self
1522 }
1523
1524 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1525 self.daemon_provenance.start_clock = Some(clock);
1526 self
1527 }
1528
1529 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1531 self.scopes
1532 .write()
1533 .unwrap_or_else(|poisoned| poisoned.into_inner())
1534 .set_wall_clock(Arc::new(clock));
1535 self
1536 }
1537
1538 #[cfg(test)]
1539 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1540 self.provenance_probe_override = Some(result);
1541 self
1542 }
1543
1544 pub fn with_capability_config(
1548 self,
1549 modules: impl IntoIterator<Item = (String, bool)>,
1550 reserved_capabilities: BTreeMap<String, String>,
1551 ) -> Self {
1552 self.capability_evaluator
1553 .configure(modules, reserved_capabilities);
1554 self
1555 }
1556
1557 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1561 self.daemon_shutdown = flag;
1562 self
1563 }
1564
1565 pub fn with_supervisor_rescan(
1566 mut self,
1567 supervisor: Supervisor,
1568 config_path: impl Into<PathBuf>,
1569 configured_port: Option<u16>,
1570 ) -> Self {
1571 self.rescan = Some(SupervisorRescanContext {
1572 supervisor,
1573 config_path: config_path.into(),
1574 configured_port,
1575 storage_config: self.storage_config.clone(),
1576 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1577 admission_facts_targets: self.admission_facts_targets.clone(),
1578 scope_authority_owners: self.scope_authority_owners.clone(),
1579 federated_scope_owner: self.federated_scope_owner.clone(),
1580 privileged_operator_prompt: self.privileged_operator_prompt,
1581 generation: Arc::default(),
1582 module_entries: Arc::default(),
1583 });
1584 self
1585 }
1586
1587 pub(crate) fn with_supervisor_rescan_entries(
1588 self,
1589 modules: &[crate::daemon_config::ConfiguredModule],
1590 ) -> Self {
1591 if let Some(context) = &self.rescan {
1592 let mut entries = context
1593 .module_entries
1594 .lock()
1595 .unwrap_or_else(|e| e.into_inner());
1596 entries.seed(modules);
1597 }
1598 self
1599 }
1600
1601 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1602 self.connected_clients = connected_clients;
1603 self
1604 }
1605
1606 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1609 self.local_endpoint_unavailable = reason;
1610 self
1611 }
1612
1613 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1614 Arc::clone(&self.forwarding)
1615 }
1616
1617 pub(crate) fn counters(&self) -> DaemonCounters {
1618 self.counters.clone()
1619 }
1620
1621 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1624 let handler = Arc::downgrade(&self);
1625 let evaluator = self.capability_evaluator.clone();
1626 tokio::spawn(async move {
1627 loop {
1628 evaluator.wait_for_change_or_deadline().await;
1629 let Some(handler) = handler.upgrade() else {
1630 break;
1631 };
1632 handler.refresh_capability_requirements();
1633 }
1634 });
1635 }
1636
1637 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1643 let context = scope_expiry::ScopeExpiryContext::new(&self);
1644 let deadline_added = self.scope_deadline_added.clone();
1645 tokio::spawn(async move {
1646 loop {
1647 if !context.has_deadlines() {
1648 deadline_added.notified().await;
1649 continue;
1650 }
1651 tokio::time::sleep(Duration::from_secs(1)).await;
1652 if let Err(error) = context.sweep() {
1653 warn!(%error, "scope expiry sweep failed");
1654 }
1655 }
1656 });
1657 }
1658
1659 #[cfg(test)]
1660 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1661 scope_expiry::ScopeExpiryContext::new(self).sweep()
1662 }
1663
1664 fn publish_scope_changes(
1665 &self,
1666 changes: &[crate::scopes::ScopeTagChange],
1667 expired: &[crate::scopes::ScopeExpired],
1668 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1669 scope_expiry::ScopeExpiryContext::new(self).publish_scope_changes(changes, expired)
1670 }
1671
1672 fn runtime_capability_snapshot(
1673 &self,
1674 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1675 let runtime = self
1676 .supervisor
1677 .list()
1678 .into_iter()
1679 .map(|module| {
1680 let status = module.status().map_err(|err| {
1681 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1682 })?;
1683 Ok(RuntimeModule {
1684 module_id: status.module_id,
1685 state: status.state,
1686 enabled: status.enabled,
1687 })
1688 })
1689 .collect::<Result<Vec<_>, RouterError>>()?;
1690 let registrations = self.registry.list_registered_modules().map_err(|err| {
1696 RouterError::backend(
1697 0,
1698 0,
1699 format!("failed to list capability registrations: {err}"),
1700 )
1701 })?;
1702 let registrations = registrations
1703 .into_iter()
1704 .map(|registration| RegisteredModule {
1705 module_id: registration.manifest.module_id,
1706 module_version: registration.manifest.module_version,
1707 capabilities: registration.manifest.capabilities,
1708 })
1709 .collect();
1710 Ok((runtime, registrations))
1711 }
1712
1713 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1720 let cached_registration = RegisteredModule {
1721 module_id: registration.manifest.module_id.clone(),
1722 module_version: registration.manifest.module_version.clone(),
1723 capabilities: registration.manifest.capabilities.clone(),
1724 };
1725 let change = self.capability_evaluator.record_hello(&cached_registration);
1726 log_manifest_change(&cached_registration.module_id, &change);
1727 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1728 self.enforce_capability_denies();
1729 }
1730 self.refresh_capability_requirements();
1731 }
1732
1733 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1737 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1738 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1739 self.supervisor.set_swap_promotion_observer(observer);
1740 }
1741
1742 pub fn refresh_capability_requirements(&self) {
1743 match self.runtime_capability_snapshot() {
1744 Ok((runtime, registrations)) => {
1745 let events = self
1746 .capability_evaluator
1747 .evaluate_now(&runtime, ®istrations);
1748 if !events.is_empty() {
1749 self.catalog_readiness_revision
1750 .fetch_add(1, std::sync::atomic::Ordering::AcqRel);
1751 }
1752 log_requirement_events(events, &self.daemon_shutdown);
1753 }
1754 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1755 }
1756 }
1757
1758 fn enforce_capability_denies(&self) {
1762 let registrations = match self.registry.list_registered_modules() {
1763 Ok(snapshot) => snapshot,
1764 Err(err) => {
1765 warn!(error = %err, "failed to read registrations for capability deny census");
1766 return;
1767 }
1768 };
1769 let manifests = registrations
1770 .into_iter()
1771 .map(|registration| {
1772 (
1773 registration.manifest.module_id.clone(),
1774 registration.manifest,
1775 )
1776 })
1777 .collect::<BTreeMap<_, _>>();
1778 let census = match self.forwarding.route_census(None) {
1779 Ok(census) => census,
1780 Err(err) => {
1781 warn!(error = %err, "failed to read route census for capability deny enforcement");
1782 return;
1783 }
1784 };
1785
1786 for (target_module_id, routes) in census {
1787 let Some(target_manifest) = manifests.get(&target_module_id) else {
1788 continue;
1789 };
1790 let mut closed_routes = Vec::new();
1791 let mut goodbyes = Vec::new();
1792 for route in routes {
1793 let Principal::Reserved {
1794 module_id: opening_module_id,
1795 } = &route.principal
1796 else {
1797 continue;
1798 };
1799 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1800 continue;
1801 };
1802 let Some(capability) = denied_capability(
1803 opening_manifest.capabilities.as_ref(),
1804 target_manifest.capabilities.as_ref(),
1805 ) else {
1806 continue;
1807 };
1808
1809 match self.forwarding.release_client_route(
1810 route.goodbye_target.connection_id,
1811 route.goodbye_target.channel,
1812 route.goodbye_target.epoch,
1813 ) {
1814 Ok(RouteRelease::Removed(module_goodbye)) => {
1815 warn!(
1816 opening_module_id,
1817 target_module_id,
1818 capability,
1819 "force-closing route because an attested capability deny edge now matches"
1820 );
1821 goodbyes.push(module_goodbye);
1822 goodbyes.push(route.goodbye_target.clone());
1823 closed_routes.push(route);
1824 }
1825 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1826 Err(err) => warn!(
1827 opening_module_id,
1828 target_module_id,
1829 capability,
1830 error = %err,
1831 "failed to force-close capability-denied route"
1832 ),
1833 }
1834 }
1835
1836 if closed_routes.is_empty() {
1837 continue;
1838 }
1839 send_route_control_pushes(
1840 &self.forwarding,
1841 closed_routes,
1842 ClientControlPush::RouteClosed {
1843 module_id: target_module_id,
1844 channels: Vec::new(),
1845 reason: RouteCloseReason::CapabilityDenied,
1846 drained: false,
1847 abandoned: 0,
1848 excluded_subscriptions: 0,
1849 terminal: Some(false),
1850 },
1851 );
1852 self.emit_route_goodbyes(goodbyes);
1855 }
1856 }
1857
1858 fn not_ready_reason(
1864 &self,
1865 registration: &crate::registry::ModuleRegistration,
1866 ) -> Option<NotReadyReason> {
1867 if !registration.ready {
1868 return Some(NotReadyReason {
1869 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1870 capability: None,
1871 });
1872 }
1873 self.first_unprovided_required_capability(registration)
1874 .map(|capability| NotReadyReason {
1875 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1876 capability: Some(capability),
1877 })
1878 }
1879
1880 fn first_unprovided_required_capability(
1898 &self,
1899 registration: &crate::registry::ModuleRegistration,
1900 ) -> Option<String> {
1901 self.first_unprovided_required_capability_for(
1902 ®istration.manifest.module_id,
1903 registration.manifest.capabilities.as_ref(),
1904 )
1905 }
1906
1907 fn first_unprovided_required_capability_for(
1908 &self,
1909 module_id: &str,
1910 capabilities: Option<&subc_protocol::manifest::CapabilityDeclarations>,
1911 ) -> Option<String> {
1912 let required = capabilities
1913 .into_iter()
1914 .flat_map(|declarations| declarations.requires.iter())
1915 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1916 .map(|requirement| requirement.capability.as_str())
1917 .collect::<BTreeSet<_>>();
1918 if required.is_empty() {
1919 return None;
1920 }
1921 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1922 if required
1923 .iter()
1924 .any(|capability| verdict(capability).is_none())
1925 {
1926 self.refresh_capability_requirements();
1927 }
1928 required
1929 .into_iter()
1930 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1931 .map(str::to_string)
1932 }
1933
1934 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1935 self.capability_evaluator
1936 .statuses()
1937 .into_iter()
1938 .map(capability_requirement_status)
1939 .collect()
1940 }
1941
1942 fn deregister_connection(
1949 &self,
1950 connection_id: ConnectionId,
1951 reason: RegistrationEndReason,
1952 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1953 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
1954 spawn.end_primary_registration();
1955 }
1956 self.registry
1957 .deregister_connection_with_reason(connection_id, reason)
1958 }
1959
1960 fn rollback_unpublished_hello(
1965 &self,
1966 connection: ConnectionId,
1967 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1968 self.registry.deregister_connection_with_reason(
1969 connection,
1970 RegistrationEndReason::RegistrationFailed,
1971 )
1972 }
1973
1974 fn request_refusal_close(
1975 &self,
1976 connection: ConnectionId,
1977 sink: Option<&crate::FrameSink>,
1978 reason: CloseReason,
1979 ) {
1980 if let Some(sink) = sink {
1981 sink.terminate_ingress();
1982 }
1983 if self.claims.close_notifications_enabled() {
1984 self.forwarding.request_connection_close(connection, reason);
1985 }
1986 }
1987
1988 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1989 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1990 return None;
1991 }
1992 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1993 parse_client_control_request(&frame.body)
1994 else {
1995 return None;
1996 };
1997 Some(target_module_id(&target).to_string())
1998 }
1999
2000 pub(crate) fn route_open_capacity_refusal(
2001 &self,
2002 ctx: &RouteCtx,
2003 frame: &Frame,
2004 target_module_id: &str,
2005 in_flight: usize,
2006 limit: usize,
2007 ) -> Result<Frame, RouterError> {
2008 let caller = self.route_open_caller_from_frame(ctx, frame);
2011 self.route_open_admission_refusal_frame(
2012 ctx,
2013 &caller,
2014 frame,
2015 target_module_id,
2016 "open_admission_full",
2017 (in_flight, limit),
2018 format!(
2019 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
2020 ),
2021 )
2022 }
2023
2024 fn route_open_target_capacity_refusal(
2025 &self,
2026 ctx: &RouteCtx,
2027 caller: &RouteOpenCaller,
2028 frame: &Frame,
2029 target_module_id: &str,
2030 in_flight: usize,
2031 ) -> Result<Frame, RouterError> {
2032 self.route_open_admission_refusal_frame(
2033 ctx,
2034 caller,
2035 frame,
2036 target_module_id,
2037 "target_binds_full",
2038 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
2039 format!(
2040 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2041 ),
2042 )
2043 }
2044
2045 #[allow(clippy::too_many_arguments)]
2053 fn route_open_admission_refusal_frame(
2054 &self,
2055 ctx: &RouteCtx,
2056 caller: &RouteOpenCaller,
2057 frame: &Frame,
2058 target_module_id: &str,
2059 reason: &'static str,
2060 (in_flight, limit): (usize, usize),
2061 message: impl Into<String>,
2062 ) -> Result<Frame, RouterError> {
2063 let code = error_codes::TARGET_UNAVAILABLE;
2064 self.counters.increment_route_open_refused(code);
2065 info!(
2066 target: "control",
2067 code,
2068 reason,
2069 module_id = ?target_module_id,
2070 connection_id = ctx.connection_id.get(),
2071 principal = %caller.principal,
2072 scope = %caller.scope,
2073 harness = %caller.harness,
2074 in_flight,
2075 limit,
2076 "route.open refused"
2077 );
2078 control_error_frame(frame, code, message.into())
2079 }
2080
2081 #[cfg(test)]
2087 pub fn handle_control(
2088 &self,
2089 connection_id: ConnectionId,
2090 frame: Frame,
2091 ) -> Result<Vec<Frame>, RouterError> {
2092 match frame.header.ty {
2093 FrameType::Ping => Ok(vec![pong(&frame)?]),
2094 FrameType::Hello => self.handle_hello(connection_id, None, frame),
2095 FrameType::Goodbye => self.handle_goodbye(connection_id),
2096 ty => Ok(vec![control_error_frame(
2097 &frame,
2098 "unsupported_control_frame",
2099 format!("unsupported channel-0 frame {ty:?}"),
2100 )?]),
2101 }
2102 }
2103
2104 pub async fn handle_control_frame(
2105 &self,
2106 ctx: &RouteCtx,
2107 frame: Frame,
2108 ) -> Result<Vec<Frame>, RouterError> {
2109 self.handle_control_frame_timed(ctx, frame, None).await
2110 }
2111
2112 #[cfg(any(test, feature = "test-support"))]
2113 pub fn with_test_operator_app_policy(
2114 mut self,
2115 policy: crate::operator_apps::TestOperatorAppPolicy,
2116 ) -> Self {
2117 self.operator_app_policy = policy.into();
2118 self
2119 }
2120
2121 #[cfg(all(windows, any(test, feature = "test-support")))]
2125 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
2126 self.windows_verify = Some(verify);
2127 self
2128 }
2129
2130 #[cfg(all(windows, any(test, feature = "test-support")))]
2132 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
2133 self.windows_flash = Some(flash);
2134 self
2135 }
2136
2137 #[cfg(all(windows, any(test, feature = "test-support")))]
2141 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
2142 self.windows_track = Some(track);
2143 self
2144 }
2145
2146 pub(crate) fn record_captured_peer(
2147 &self,
2148 connection: ConnectionId,
2149 peer: subc_os::local_peer::CapturedPeer,
2150 ) {
2151 self.record_observed_peer(connection, peer.observation().clone());
2152 let identity = self.operator_app_policy.config_identity(peer.clone());
2153 #[cfg(windows)]
2154 let identity = identity.or_else(|| self.config_windows_launch_identity(&peer));
2155 if let Some(identity) = identity {
2156 self.config_desktop_stamps
2159 .lock()
2160 .unwrap_or_else(|p| p.into_inner())
2161 .insert(connection, identity);
2162 }
2163 self.captured_peers
2164 .lock()
2165 .unwrap_or_else(|p| p.into_inner())
2166 .insert(connection, peer.clone());
2167 #[cfg(windows)]
2168 if let Some(stamp) = self.stamp_windows_launch(&peer) {
2169 self.operator_apps
2170 .lock()
2171 .unwrap_or_else(|p| p.into_inner())
2172 .insert(connection, stamp);
2173 }
2174 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
2175 self.operator_apps
2176 .lock()
2177 .unwrap_or_else(|p| p.into_inner())
2178 .insert(connection, stamp);
2179 }
2180 }
2181
2182 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
2183 self.local_connections
2184 .lock()
2185 .unwrap_or_else(|p| p.into_inner())
2186 .insert(connection);
2187 }
2188
2189 fn connection_operator_app(
2192 &self,
2193 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
2194 ) -> Option<subc_protocol::peer::OperatorApp> {
2195 stamp
2196 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
2197 .map(|stamp| stamp.app.clone())
2198 }
2199
2200 fn handle_connection_describe(
2201 &self,
2202 ctx: &RouteCtx,
2203 frame: Frame,
2204 ) -> Result<Vec<Frame>, RouterError> {
2205 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
2206 return Ok(vec![control_error_frame(
2207 &frame,
2208 "invalid_control_body",
2209 "malformed connection.describe body",
2210 )?]);
2211 }
2212 let local = self
2213 .local_connections
2214 .lock()
2215 .unwrap_or_else(|p| p.into_inner())
2216 .contains(&ctx.connection_id);
2217 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
2218 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
2219 Ok(vec![control_response_body_frame(
2220 &frame,
2221 &subc_control::ConnectionDescribeReply::new(local, operator_app),
2222 ops::CONNECTION_DESCRIBE,
2223 )?])
2224 }
2225
2226 pub(crate) fn record_observed_peer(
2227 &self,
2228 connection: ConnectionId,
2229 peer: subc_os::local_peer::ObservedPeer,
2230 ) {
2231 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
2232 let peer = match peer {
2233 OsPeer::Unobserved => ObservedPeer::unobserved(),
2234 OsPeer::Observed(peer) => {
2235 let observed = match peer.user {
2236 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
2237 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
2238 };
2239 observed
2240 .with_exe(
2241 peer.exe
2242 .and_then(|path| path.into_os_string().into_string().ok()),
2243 )
2244 .with_signing_id(peer.signing_id)
2245 }
2246 };
2247 self.observed_peers
2248 .lock()
2249 .unwrap_or_else(|p| p.into_inner())
2250 .insert(connection, peer);
2251 }
2252
2253 pub(crate) fn record_caller_endpoints(
2254 &self,
2255 connection: ConnectionId,
2256 endpoints: Option<(SocketAddr, SocketAddr)>,
2257 ) {
2258 let mut callers = self
2259 .caller_endpoints
2260 .lock()
2261 .unwrap_or_else(|p| p.into_inner());
2262 callers.remove(&connection);
2263 if let Some(endpoints) = endpoints {
2264 callers.insert(connection, endpoints);
2265 }
2266 }
2267
2268 #[cfg(test)]
2269 pub(crate) fn with_caller_lookup_for_test(
2270 mut self,
2271 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2272 ) -> Self {
2273 self.caller_lookup = Some(Arc::new(lookup));
2274 self
2275 }
2276
2277 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2278 let endpoints = self
2279 .caller_endpoints
2280 .lock()
2281 .unwrap_or_else(|p| p.into_inner())
2282 .get(&connection)
2283 .copied();
2284 #[cfg(test)]
2285 let lookup = self.caller_lookup.clone();
2286 tokio::task::spawn_blocking(move || {
2287 #[cfg(test)]
2288 if let Some(lookup) = lookup {
2289 return lookup(endpoints);
2290 }
2291 endpoints
2292 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2293 .unwrap_or_default()
2294 })
2295 .await
2296 .unwrap_or_default()
2297 }
2298
2299 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2300 frame.header.channel == 0
2301 && frame.header.ty == FrameType::Request
2302 && self
2303 .forwarding
2304 .module_endpoint_for_connection(ctx.connection_id)
2305 .is_ok_and(|endpoint| endpoint.is_none())
2306 && parse_client_control_request(&frame.body)
2307 .ok()
2308 .is_some_and(|request| privileged_operation(&request).is_some())
2309 }
2310
2311 pub(crate) async fn handle_control_frame_timed(
2312 &self,
2313 ctx: &RouteCtx,
2314 frame: Frame,
2315 dispatch_started_at: Option<StdInstant>,
2316 ) -> Result<Vec<Frame>, RouterError> {
2317 let first = ctx.egress.first_ingress_frame();
2318 self.handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
2319 .await
2320 }
2321
2322 pub(crate) async fn handle_control_frame_ingress(
2323 &self,
2324 ctx: &RouteCtx,
2325 frame: Frame,
2326 dispatch_started_at: Option<StdInstant>,
2327 first: bool,
2328 ) -> Result<Vec<Frame>, RouterError> {
2329 if ctx.egress.ingress_terminal() {
2330 return Ok(Vec::new());
2331 }
2332 #[cfg(test)]
2333 self.claims.record_dispatch(frame.header.corr);
2334 let claim_binding = self.claims.binding(ctx.connection_id);
2335 let mut prefix_attempt = (frame.header.ty == FrameType::Hello
2336 && claim_binding
2337 .as_ref()
2338 .is_some_and(|binding| binding.prefix_module_id().is_some()))
2339 .then(|| self.prefix_hello_attempt(ctx));
2340 if claim_binding
2341 .as_ref()
2342 .is_some_and(|binding| binding.prefix_module_id().is_some())
2343 {
2344 let registered = self
2345 .forwarding
2346 .module_endpoint_for_connection(ctx.connection_id)
2347 .map_err(RouterError::Forwarding)?
2348 .is_some();
2349 if !registered && frame.header.ty != FrameType::Hello {
2350 return self
2351 .refuse_claim_connection(ctx, &frame, "claim_hello_required")
2352 .await;
2353 }
2354 if registered && frame.header.ty == FrameType::Goodbye {
2355 ctx.egress.terminate_ingress();
2356 let result = self.handle_goodbye(ctx.connection_id);
2357 self.claims.connection_closed(ctx.connection_id);
2358 self.request_refusal_close(
2359 ctx.connection_id,
2360 Some(&ctx.egress),
2361 CloseReason::new("claim_registration_ended", "prefix registration ended"),
2362 );
2363 return result;
2364 }
2365 if registered && frame.header.ty == FrameType::Request {
2366 let allowed =
2367 serde_json::from_slice::<ControlOpProbe>(&frame.body).is_ok_and(|probe| {
2368 matches!(
2369 probe.op.as_str(),
2370 "catalog.update"
2371 | "catalog.update_role_declarations"
2372 | "scope.describe"
2373 | "scope.watch"
2374 | "supervisor.live_roots"
2375 | "activity.record"
2376 )
2377 });
2378 if !allowed {
2379 return Ok(vec![control_error_frame(
2380 &frame,
2381 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2382 "prefix registrations have only brokered module operations",
2383 )?]);
2384 }
2385 }
2386 if registered
2387 && frame.header.ty == FrameType::Push
2388 && !serde_json::from_slice::<ControlOpProbe>(&frame.body)
2389 .is_ok_and(|probe| probe.op == "route.status")
2390 {
2391 return Ok(vec![control_error_frame(
2392 &frame,
2393 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2394 "prefix registrations may push only route.status for routes they serve",
2395 )?]);
2396 }
2397 }
2398 match frame.header.ty {
2399 FrameType::Ping => Ok(vec![pong(&frame)?]),
2400 FrameType::Hello => {
2401 if claim_binding
2402 .as_ref()
2403 .is_some_and(|binding| binding.prefix_module_id().is_none())
2404 {
2405 return self
2406 .refuse_claim_connection(ctx, &frame, "claim_hello_not_allowed")
2407 .await;
2408 }
2409 let channel_spawn = self.supervisor.registration_spawn(ctx.connection_id);
2413 if let Some(spawn) = &channel_spawn {
2414 spawn.wait_launch().await;
2415 if !spawn.launch_ready() {
2418 ctx.egress.terminate_ingress();
2419 return Ok(Vec::new());
2420 }
2421 #[cfg(test)]
2422 self.pause_hello_ingress_for_test().await;
2423 } else if claim_binding.is_none() {
2424 if let Ok(hello) = serde_json::from_slice::<serde_json::Value>(&frame.body) {
2425 if let (Some(module), Some(nonce)) = (
2426 hello
2427 .get("manifest")
2428 .and_then(|value| value.get("module_id"))
2429 .and_then(|value| value.as_str()),
2430 hello.get("launch_nonce").and_then(|value| value.as_str()),
2431 ) {
2432 if !self
2433 .config_runtime
2434 .as_ref()
2435 .is_some_and(|owner| owner.module_is_gated(module))
2436 && !self.supervisor.wait_program_binding(module, nonce).await
2437 {
2438 self.request_refusal_close(
2439 ctx.connection_id,
2440 Some(&ctx.egress),
2441 CloseReason::new(
2442 "program_refused",
2443 "the spawned program binding failed",
2444 ),
2445 );
2446 return Ok(vec![control_error_frame(
2447 &frame,
2448 if self
2449 .config_runtime
2450 .as_ref()
2451 .is_some_and(|owner| owner.is_recovering())
2452 {
2453 "config_gated"
2454 } else {
2455 "program_refused"
2456 },
2457 "the spawned program binding failed",
2458 )?]);
2459 }
2460 }
2461 }
2462 }
2463 let handler = self.clone();
2464 let connection = ctx.connection_id;
2465 let egress = ctx.egress.clone();
2466 let correlation = frame.header.corr;
2467 let dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
2468 let span = tracing::Span::current();
2469 let hello_binding = claim_binding.clone();
2470 let replies = tokio::task::spawn_blocking(move || {
2471 tracing::dispatcher::with_default(&dispatcher, || {
2472 span.in_scope(|| {
2473 handler.handle_hello_checked(
2474 connection,
2475 Some(egress),
2476 frame,
2477 true,
2478 hello_binding,
2479 channel_spawn,
2480 )
2481 })
2482 })
2483 })
2484 .await
2485 .map_err(|error| RouterError::backend(0, correlation, error.to_string()))??;
2486 #[cfg(test)]
2487 if prefix_attempt.is_some() {
2488 self.claims.pause_prefix_commit_for_test(false).await;
2489 }
2490 if let Some(attempt) = &mut prefix_attempt {
2491 let accepted = !replies
2492 .iter()
2493 .any(|reply| reply.header.ty == FrameType::Error)
2494 && self
2495 .registry
2496 .get_module_by_connection(ctx.connection_id)
2497 .map_err(|err| RouterError::backend(0, correlation, err.to_string()))?
2498 .is_some();
2499 if accepted {
2500 attempt.accepted = true;
2501 } else {
2502 ctx.egress.terminate_ingress();
2503 for reply in replies {
2504 ctx.egress.send_flushed(reply).await?;
2505 }
2506 return Ok(Vec::new());
2507 }
2508 }
2509 Ok(replies)
2510 }
2511 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2512 FrameType::Cancel => {
2513 if self
2519 .agent_declarations
2520 .cancel(ctx.connection_id, frame.header.corr)
2521 || self
2522 .forwarding
2523 .operator_confirms()
2524 .cancel(ctx.connection_id, frame.header.corr)
2525 || self.cancel_config_approval(ctx.connection_id, frame.header.corr)
2526 || self
2527 .supervisor
2528 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2529 || self.cancel_activity_subscription(ctx, &frame)?
2530 || self.cancel_catalog_subscription(ctx.connection_id, frame.header.corr)
2531 || self.cancel_scope_subscription(ctx.connection_id, frame.header.corr)
2532 {
2533 Ok(Vec::new())
2534 } else {
2535 Ok(vec![control_error_frame(
2536 &frame,
2537 "unknown_subscription",
2538 "no subscription has this correlation id",
2539 )?])
2540 }
2541 }
2542 FrameType::Request => {
2543 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2546 if let Some(probe) = &op {
2547 self.record_control_usage(ctx.connection_id, &probe.op);
2548 }
2549 let claim_present = op.as_ref().is_some_and(|probe| probe.op == "claim.present");
2550 if let Some(refusal) = self
2551 .recovery_request_refusal(
2552 ctx,
2553 &frame,
2554 op.as_ref().map(|probe| probe.op.as_str()),
2555 )
2556 .await?
2557 {
2558 if claim_present {
2559 self.reclaim_presented_expiry(&frame);
2560 return self
2561 .refuse_claim_connection(ctx, &frame, "config_gated")
2562 .await;
2563 }
2564 return Ok(vec![refusal]);
2565 }
2566 if claim_present {
2567 return self.handle_claim_present(ctx, &frame, first).await;
2568 }
2569 if op.as_ref().is_some_and(|probe| probe.op == "claim.mint") {
2570 return self.handle_claim_mint(ctx, &frame).await;
2571 }
2572 if let Some(refusal) = self.registration_operation_refusal(
2573 ctx,
2574 &frame,
2575 op.as_ref().map(|probe| probe.op.as_str()),
2576 )? {
2577 return Ok(vec![refusal]);
2578 }
2579 if op
2580 .as_ref()
2581 .is_some_and(|probe| probe.op == ops::CATALOG_WATCH)
2582 {
2583 return self.handle_catalog_watch(ctx, frame);
2584 }
2585 if op
2586 .as_ref()
2587 .is_some_and(|probe| probe.op == ops::SCOPE_WATCH)
2588 {
2589 return self.handle_scope_watch(ctx, frame);
2590 }
2591 if op
2592 .as_ref()
2593 .is_some_and(|probe| probe.op == MODULE_TO_SUBC_OP_CATALOG_UPDATE)
2594 && subc_protocol::provider_declarations::catalog_update_has_provider_declarations(&frame.body)
2595 .unwrap_or(false)
2596 {
2597 return Ok(vec![control_error_frame(
2598 &frame,
2599 error_codes::CATALOG_UPDATE_FROZEN_DECLARATIONS,
2600 "provider_declarations are fixed at HELLO; restart the module to change them",
2601 )?]);
2602 }
2603 if let Some(probe) = op
2604 .as_ref()
2605 .filter(|probe| config_gate::is_module_config_op(&probe.op))
2606 {
2607 return self
2608 .handle_module_config_request(ctx, frame, &probe.op)
2609 .await;
2610 }
2611 if op
2612 .as_ref()
2613 .is_some_and(|probe| config_gate::is_config_op(&probe.op))
2614 {
2615 return self.handle_config_gate_request(ctx, frame).await;
2616 }
2617 if op
2618 .as_ref()
2619 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2620 {
2621 return self.handle_activity_record(ctx, frame).await;
2622 }
2623 if op.as_ref().is_some_and(|probe| {
2624 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2625 }) {
2626 return self.handle_activity_read(ctx, frame).await;
2627 }
2628 if op
2629 .as_ref()
2630 .is_some_and(|probe| probe.op == "operator.confirm")
2631 {
2632 return self.handle_operator_confirm(ctx, frame);
2633 }
2634 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2635 return self
2636 .observe_control_wait(
2637 ctx,
2638 None,
2639 ops::APP_OPEN,
2640 self.handle_app_open(ctx, frame),
2641 )
2642 .await;
2643 }
2644 if op
2645 .as_ref()
2646 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2647 {
2648 return self.handle_connection_describe(ctx, frame);
2649 }
2650 if op
2651 .as_ref()
2652 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2653 {
2654 return self.handle_scope_served_by(ctx.connection_id, frame);
2655 }
2656 if self
2657 .forwarding
2658 .module_endpoint_for_connection(ctx.connection_id)
2659 .map_err(RouterError::Forwarding)?
2660 .is_some()
2661 {
2662 if !is_known_module_request_op(&frame.body) {
2663 return Ok(vec![control_error_frame(
2664 &frame,
2665 "unsupported_control_frame",
2666 "module-originated channel-0 REQUEST is not supported",
2667 )?]);
2668 }
2669 let request = match parse_module_control_request_from_module(&frame.body) {
2670 Ok(request) => request,
2671 Err((err, ControlRequestBodyError::UnknownOp)) => {
2672 return Ok(vec![control_error_frame(
2673 &frame,
2674 "unsupported_control_frame",
2675 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2676 )?])
2677 }
2678 Err((err, ControlRequestBodyError::InvalidBody)) => {
2679 return Ok(vec![control_error_frame(
2680 &frame,
2681 "invalid_control_body",
2682 format!("malformed module control body: {err}"),
2683 )?])
2684 }
2685 };
2686 let op = module_control_request_op(&request);
2687 let corr = frame.header.corr;
2688 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2689 if let ModuleControlRequestFromModule::AgentCommands {
2690 started,
2691 finished,
2692 known,
2693 } = request
2694 {
2695 return self
2696 .agent_declarations
2697 .enqueue(
2698 self,
2699 ctx,
2700 frame,
2701 crate::resource_totals::Declaration {
2702 started,
2703 finished,
2704 known,
2705 },
2706 )
2707 .await;
2708 }
2709 let result =
2710 self.handle_module_control_request(ctx.connection_id, frame, request);
2711 log_slow_control_dispatch(
2712 dispatch_started_at,
2713 op,
2714 ctx.connection_id,
2715 corr,
2716 None,
2717 );
2718 return result;
2719 }
2720
2721 if is_known_module_request_op(&frame.body) {
2722 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2723 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2724 let code = if is_agents {
2725 "not_supervised"
2726 } else if op
2727 .as_ref()
2728 .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2729 {
2730 error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2731 } else {
2732 "not_registered"
2733 };
2734 return Ok(vec![control_error_frame(
2735 &frame,
2736 code,
2737 "module-originated control requests require the module's own registered connection",
2738 )?]);
2739 }
2740
2741 let request = match parse_client_control_request(&frame.body) {
2742 Ok(request) => request,
2743 Err((err, ControlRequestBodyError::UnknownOp)) => {
2744 return Ok(vec![control_error_frame(
2745 &frame,
2746 "unknown_control_op",
2747 format!("unknown client control op: {err}"),
2748 )?])
2749 }
2750 Err((err, ControlRequestBodyError::InvalidBody)) => {
2751 return Ok(vec![control_error_frame(
2752 &frame,
2753 "invalid_control_body",
2754 format!("malformed client control body: {err}"),
2755 )?])
2756 }
2757 };
2758 let op = client_control_request_op(&request);
2759 let corr = frame.header.corr;
2760 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2761 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2762 #[cfg(test)]
2763 if let Some(delay) = self.control_dispatch_delay {
2764 tokio::time::sleep(delay).await;
2765 }
2766 let result = self
2767 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2768 .await;
2769 if let Some(timings) = route_open_timings.as_mut() {
2770 timings.finish();
2771 }
2772 log_slow_control_dispatch(
2773 dispatch_started_at,
2774 op,
2775 ctx.connection_id,
2776 corr,
2777 route_open_timings.as_ref(),
2778 );
2779 result
2780 }
2781 FrameType::Push => {
2782 let Some(endpoint) = self
2783 .forwarding
2784 .module_endpoint_for_connection(ctx.connection_id)
2785 .map_err(RouterError::Forwarding)?
2786 else {
2787 return Ok(vec![control_error_frame(
2788 &frame,
2789 "unsupported_control_frame",
2790 "client-originated channel-0 PUSH is not supported",
2791 )?]);
2792 };
2793 self.handle_status_update(endpoint, frame)
2794 }
2795 FrameType::Response | FrameType::Error
2796 if self
2797 .forwarding
2798 .module_endpoint_for_connection(ctx.connection_id)
2799 .map_err(RouterError::Forwarding)?
2800 .is_some() =>
2801 {
2802 self.handle_module_relay_response(ctx.connection_id, frame)
2803 }
2804 ty => Ok(vec![control_error_frame(
2805 &frame,
2806 "unsupported_control_frame",
2807 format!("unsupported channel-0 frame {ty:?}"),
2808 )?]),
2809 }
2810 }
2811
2812 #[cfg(test)]
2813 fn pause_cleanup_for_test(&self, stage: u8) {
2814 let mut slot = self.cleanup_pause.lock().unwrap_or_else(|p| p.into_inner());
2815 if slot.as_ref().is_some_and(|(at, _)| *at == stage) {
2816 let (_, (reached, release)) = slot.take().unwrap();
2817 drop(slot);
2818 let _ = reached.send(());
2819 let _ = release.recv_timeout(Duration::from_secs(5));
2820 }
2821 }
2822
2823 #[cfg(all(test, unix, feature = "test-support"))]
2826 pub(crate) fn after_nonce_attestation_for_test(&self, hook: impl FnOnce() + Send + 'static) {
2827 *self
2828 .after_nonce_attestation
2829 .lock()
2830 .unwrap_or_else(|p| p.into_inner()) = Some(Box::new(hook));
2831 }
2832
2833 #[cfg(test)]
2834 pub(super) fn pause_after_nonce_attestation_for_test(&self) {
2835 let hook = self
2836 .after_nonce_attestation
2837 .lock()
2838 .unwrap_or_else(|p| p.into_inner())
2839 .take();
2840 if let Some(hook) = hook {
2841 hook();
2842 }
2843 }
2844
2845 #[cfg(test)]
2846 async fn pause_hello_ingress_for_test(&self) {
2847 let pause = self
2848 .hello_ingress_pause
2849 .lock()
2850 .unwrap_or_else(|p| p.into_inner())
2851 .take();
2852 if let Some((reached, release)) = pause {
2853 let _ = reached.send(());
2854 let _ = release.await;
2855 }
2856 }
2857
2858 pub fn cleanup_connection(
2859 &self,
2860 connection_id: ConnectionId,
2861 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2862 self.agent_declarations.close(connection_id);
2876 self.close_catalog_subscriptions(connection_id);
2877 self.close_scope_subscriptions(connection_id);
2878 #[cfg(test)]
2879 self.pause_cleanup_for_test(0);
2880 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
2881 spawn.retire_registration_channel();
2882 }
2883 self.supervisor
2884 .forget_registration_connection(connection_id);
2885 #[cfg(test)]
2886 self.pause_cleanup_for_test(1);
2887 self.local_connections
2888 .lock()
2889 .unwrap_or_else(|p| p.into_inner())
2890 .remove(&connection_id);
2891 self.caller_endpoints
2892 .lock()
2893 .unwrap_or_else(|p| p.into_inner())
2894 .remove(&connection_id);
2895 self.build_class_warned
2896 .lock()
2897 .unwrap_or_else(|p| p.into_inner())
2898 .remove(&connection_id);
2899 self.observed_peers
2900 .lock()
2901 .unwrap_or_else(|p| p.into_inner())
2902 .remove(&connection_id);
2903 self.captured_peers
2904 .lock()
2905 .unwrap_or_else(|p| p.into_inner())
2906 .remove(&connection_id);
2907 self.config_desktop_stamps
2908 .lock()
2909 .unwrap_or_else(|p| p.into_inner())
2910 .remove(&connection_id);
2911 self.operator_apps
2912 .lock()
2913 .unwrap_or_else(|p| p.into_inner())
2914 .remove(&connection_id);
2915 self.forwarding
2916 .operator_confirms()
2917 .connection_closed(connection_id);
2918 let cleanup = self.cleanup_connection_with_end_reason(
2919 connection_id,
2920 RegistrationEndReason::ConnectionClosed,
2921 );
2922 self.claims.connection_closed(connection_id);
2925 cleanup
2926 }
2927
2928 fn cleanup_connection_with_end_reason(
2929 &self,
2930 connection_id: ConnectionId,
2931 requested_reason: RegistrationEndReason,
2932 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2933 #[cfg(test)]
2934 if let Some(pause) = self
2935 .activity_request_pause
2936 .lock()
2937 .unwrap()
2938 .as_ref()
2939 .filter(|pause| pause.connection == connection_id)
2940 {
2941 pause
2942 .cleanup_started
2943 .store(true, std::sync::atomic::Ordering::Release);
2944 }
2945 if let Some(runtime) = &self.config_runtime {
2946 runtime.disconnected(connection_id);
2947 }
2948 self.agent_declarations.close(connection_id);
2949 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2950 self.registry
2951 .get_module_by_connection(connection_id)?
2952 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2953 .and_then(|module| module.registration_end_reason().ok().flatten())
2954 .unwrap_or(requested_reason)
2955 } else {
2956 requested_reason
2957 };
2958 let crash_closed = self
2959 .registry
2960 .get_module_by_connection(connection_id)?
2961 .and_then(|registration| {
2962 self.forwarding
2963 .module_endpoint_for_connection(connection_id)
2964 .ok()
2965 .flatten()
2966 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2967 .map(|routes| (registration.manifest.module_id, routes))
2968 });
2969 let crash_closed = crash_closed.map(|(module_id, routes)| {
2970 let terminal = match self.supervisor.get(&module_id) {
2971 None => false,
2972 Some(module) => match module.will_recover_after_connection_loss() {
2973 Ok(will_recover) => !will_recover,
2974 Err(err) => {
2975 warn!(
2976 %module_id,
2977 error = %err,
2978 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2979 );
2980 false
2981 }
2982 },
2983 };
2984 let (reason, drained) = match self
2988 .forwarding
2989 .daemon_drain_result_for_connection(connection_id)
2990 {
2991 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2992 Ok(None) => (RouteCloseReason::Crash, false),
2993 Err(err) => {
2994 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2995 (RouteCloseReason::Crash, false)
2996 }
2997 };
2998 (module_id, routes, reason, drained, terminal)
2999 });
3000 let registrations = self.deregister_connection(connection_id, end_reason);
3001 let ended_authority = self
3004 .forwarding
3005 .target_authority_for_connection(connection_id)
3006 .ok()
3007 .flatten();
3008 let cleanup = if crash_closed.is_some() {
3009 self.forwarding.cleanup_connection_counted(connection_id)
3010 } else {
3011 self.forwarding
3017 .cleanup_connection(connection_id)
3018 .map(|released| crate::forwarding::ConnectionCleanup {
3019 released,
3020 abandoned_relays: 0,
3021 })
3022 };
3023 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
3028 let abandoned = cleanup
3029 .as_ref()
3030 .map_or(0, |cleanup| cleanup.abandoned_relays);
3031 send_route_control_pushes(
3032 &self.forwarding,
3033 routes,
3034 ClientControlPush::RouteClosed {
3035 module_id,
3036 channels: Vec::new(),
3037 reason,
3038 drained,
3039 abandoned,
3040 excluded_subscriptions: 0,
3041 terminal: Some(terminal),
3042 },
3043 );
3044 }
3045 if let Ok(cleanup) = cleanup {
3046 self.emit_route_goodbyes(cleanup.released);
3047 }
3048 if let Some(record) = ended_authority {
3049 self.change_spawn_authority(&record);
3050 }
3051 if matches!(®istrations, Ok(r) if !r.is_empty()) {
3055 crate::supervise::notify_registration_release();
3056 self.capability_evaluator.wake_deadline_loop();
3057 self.refresh_capability_requirements();
3058 }
3059 self.supervisor.remove_spawn_subscribers(connection_id);
3060 self.activity_streams
3061 .lock()
3062 .unwrap_or_else(|p| p.into_inner())
3063 .retain(|(id, _), task| {
3064 if *id == connection_id {
3065 task.abort();
3066 false
3067 } else {
3068 true
3069 }
3070 });
3071 self.hello_launch_nonces
3074 .lock()
3075 .unwrap_or_else(|poisoned| poisoned.into_inner())
3076 .forget(connection_id);
3077 self.scopes
3078 .write()
3079 .unwrap_or_else(|poisoned| poisoned.into_inner())
3080 .release_connection(connection_id);
3081 registrations
3082 }
3083
3084 pub(crate) fn handle_route_goodbye(
3085 &self,
3086 connection_id: ConnectionId,
3087 route_channel: u16,
3088 route_epoch: u32,
3089 ) -> Result<bool, RouterError> {
3090 debug!(
3091 connection_id = connection_id.get(),
3092 route_channel, route_epoch, "handling route GOODBYE"
3093 );
3094 let RouteRelease::Removed(released_route) = self
3095 .forwarding
3096 .release_client_route(connection_id, route_channel, route_epoch)
3097 .map_err(RouterError::Forwarding)?
3098 else {
3099 return Ok(false);
3100 };
3101 self.emit_route_goodbyes(vec![released_route]);
3102 Ok(true)
3103 }
3104
3105 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
3106 scope_expiry::ScopeExpiryContext::new(self).emit_route_goodbyes(released_routes);
3107 }
3108
3109 fn send_abandoned_route_bind_goodbye(
3121 &self,
3122 module_sink: &crate::FrameSink,
3123 negotiated_ver: u8,
3124 module_channel: u16,
3125 module_epoch: u32,
3126 ) {
3127 let frame = match Frame::build_with_version(
3128 negotiated_ver,
3129 FrameType::Goodbye,
3130 control_flags(),
3131 module_channel,
3132 module_epoch,
3133 0,
3134 Vec::new(),
3135 ) {
3136 Ok(frame) => frame,
3137 Err(err) => {
3138 warn!(
3139 route_channel = module_channel,
3140 error = %err,
3141 "failed to build GOODBYE for abandoned route.bind"
3142 );
3143 return;
3144 }
3145 };
3146 crate::forwarding::send_module_route_goodbye(
3147 &self.counters,
3148 module_sink,
3149 frame,
3150 None,
3151 "abandoned route.bind",
3152 );
3153 }
3154
3155 #[cfg(test)]
3156 fn handle_hello(
3157 &self,
3158 connection_id: ConnectionId,
3159 egress: Option<crate::router::FrameSink>,
3160 frame: Frame,
3161 ) -> Result<Vec<Frame>, RouterError> {
3162 let binding = self.claims.binding(connection_id);
3163 let channel_spawn = self.supervisor.registration_spawn(connection_id);
3164 self.handle_hello_checked(connection_id, egress, frame, false, binding, channel_spawn)
3165 }
3166
3167 fn handle_hello_checked(
3168 &self,
3169 connection_id: ConnectionId,
3170 sink: Option<crate::FrameSink>,
3171 frame: Frame,
3172 wait_for_gate: bool,
3173 prefix_binding: Option<claims::ClaimBinding>,
3174 channel_spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3175 ) -> Result<Vec<Frame>, RouterError> {
3176 let primary_sink = channel_spawn.as_ref().and_then(|_| sink.clone());
3177 #[cfg(test)]
3178 if prefix_binding.is_some() {
3179 self.claims.pause_prefix_hello_for_test(5);
3180 match self.claims.take_prefix_hello_failure_for_test() {
3181 Some("error") => {
3182 return Err(RouterError::backend(
3183 0,
3184 frame.header.corr,
3185 "injected HELLO failure",
3186 ))
3187 }
3188 Some("panic") => panic!("injected HELLO task panic"),
3189 _ => {}
3190 }
3191 }
3192 if let Some(binding) = &prefix_binding {
3193 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3194 let module = value
3195 .as_ref()
3196 .and_then(|value| value["manifest"]["module_id"].as_str());
3197 if !binding.admit_prefix_hello(module)
3198 || !self.prefix_binding_live(binding, connection_id)
3199 || !module.is_some_and(|module| {
3200 self.supervisor
3201 .prefix_claim_authorized(&binding.registration.module_id, module)
3202 })
3203 || binding.registration.check().is_err()
3204 {
3205 return Ok(vec![control_error_frame(
3206 &frame,
3207 "claim_prefix_refused",
3208 "HELLO must name the prefix claim's exact module id once",
3209 )?]);
3210 }
3211 }
3212 struct RefuseUnregisteredChannel {
3213 spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3214 sink: Option<crate::FrameSink>,
3215 }
3216 impl Drop for RefuseUnregisteredChannel {
3217 fn drop(&mut self) {
3218 if let Some(spawn) = &self.spawn {
3219 if !spawn.registered() {
3220 if let Some(sink) = &self.sink {
3221 sink.terminate_ingress();
3222 }
3223 spawn.retire_registration_channel();
3224 }
3225 }
3226 }
3227 }
3228 let _channel_attempt = RefuseUnregisteredChannel {
3229 spawn: channel_spawn.clone(),
3230 sink: sink.clone(),
3231 };
3232 if let Some(spawn) = &channel_spawn {
3233 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3234 if let Err(code) = spawn.admit_hello(
3235 value
3236 .as_ref()
3237 .and_then(|value| value["manifest"]["module_id"].as_str()),
3238 ) {
3239 return Ok(vec![control_error_frame(
3240 &frame,
3241 code,
3242 "registration channel HELLO refused",
3243 )?]);
3244 }
3245 }
3246 debug!(
3247 connection_id = connection_id.get(),
3248 corr = frame.header.corr,
3249 "handling HELLO"
3250 );
3251 if self
3255 .registry
3256 .get_module_by_connection(connection_id)
3257 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3258 .is_some()
3259 {
3260 return Ok(vec![control_error_frame(
3261 &frame,
3262 "invalid_hello",
3263 "connection is already registered as a module",
3264 )?]);
3265 }
3266 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
3267 Ok(value) => value,
3268 Err(err) => {
3269 return Ok(vec![control_error_frame(
3270 &frame,
3271 "invalid_hello",
3272 format!("malformed HELLO body: {err}"),
3273 )?])
3274 }
3275 };
3276 if let Err(detail) = catalog::validate_hello_declarations(&hello_value) {
3277 return Ok(vec![control_error_frame(
3278 &frame,
3279 error_codes::INVALID_PROVIDER_DECLARATIONS,
3280 detail,
3281 )?]);
3282 }
3283 if let Some(module) = hello_value
3284 .get("manifest")
3285 .and_then(|value| value.get("module_id"))
3286 .and_then(|value| value.as_str())
3287 {
3288 let recovering = self
3289 .config_runtime
3290 .as_ref()
3291 .is_some_and(|owner| owner.check_recovery().unwrap_or(true));
3292 if recovering
3296 && channel_spawn.as_ref().is_none_or(|spawn| {
3297 !self.supervisor.has_live_verified_channel_binding(spawn)
3298 || spawn.check().is_err()
3299 })
3300 && !(channel_spawn.is_none()
3301 && hello_value
3302 .get("launch_nonce")
3303 .and_then(|value| value.as_str())
3304 .is_some_and(|nonce| {
3305 self.supervisor.spawned_consumer_authorized(module, nonce)
3306 && self
3307 .supervisor
3308 .has_live_verified_program_binding(module, nonce)
3309 }))
3310 {
3311 return Ok(vec![control_error_frame(
3312 &frame,
3313 "config_gated",
3314 "recovery admits only previously bound supervised launches",
3315 )?]);
3316 }
3317 if self
3318 .config_runtime
3319 .as_ref()
3320 .is_some_and(|owner| owner.module_is_gated(module))
3321 && (channel_spawn
3322 .as_ref()
3323 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3324 || self.config_runtime.as_ref().is_some_and(|owner| {
3325 !recovering && owner.check_module_hold(module).is_err()
3326 }))
3327 {
3328 return Ok(vec![control_error_frame(
3329 &frame,
3330 "reserved_module",
3331 "a gated identity belongs only to its supervised child",
3332 )?]);
3333 }
3334 }
3335 if channel_spawn.is_none() {
3336 if let (Some(module), Some(nonce)) = (
3337 hello_value
3338 .get("manifest")
3339 .and_then(|value| value.get("module_id"))
3340 .and_then(|value| value.as_str()),
3341 hello_value
3342 .get("launch_nonce")
3343 .and_then(|value| value.as_str()),
3344 ) {
3345 let binding = self.supervisor.has_program_binding_record(module, nonce);
3346 let required = self
3347 .config_runtime
3348 .as_ref()
3349 .is_some_and(|owner| owner.module_is_gated(module));
3350 if let Some(error) = (binding || required)
3351 .then(|| {
3352 self.supervisor
3353 .check_current_program_binding(module, nonce)
3354 .err()
3355 })
3356 .flatten()
3357 {
3358 let code = if error.code == "program_refused" {
3359 "program_refused"
3360 } else {
3361 "config_delivery_deferred"
3362 };
3363 self.request_refusal_close(
3364 connection_id,
3365 sink.as_ref(),
3366 CloseReason::new(code, error.detail.clone()),
3367 );
3368 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3369 }
3370 }
3371 }
3372 if let Err(err) = validate_hello_display_name(&hello_value) {
3373 return Ok(vec![control_error_frame(
3374 &frame,
3375 "invalid_manifest",
3376 err.to_string(),
3377 )?]);
3378 }
3379 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
3380 return Ok(vec![control_error_frame(
3381 &frame,
3382 "invalid_capability_grammar",
3383 err.to_string(),
3384 )?]);
3385 }
3386 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
3387 return Ok(vec![control_error_frame(
3388 &frame,
3389 "invalid_manifest",
3390 err.to_string(),
3391 )?]);
3392 }
3393 if let Err(err) = validate_hello_event_declarations(&hello_value) {
3394 return Ok(vec![control_error_frame(
3395 &frame,
3396 "invalid_event_declaration",
3397 err.to_string(),
3398 )?]);
3399 }
3400 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
3401 return Ok(vec![control_error_frame(
3402 &frame,
3403 "invalid_manifest",
3404 err.to_string(),
3405 )?]);
3406 }
3407 if let Some(provenance) = hello_value
3408 .get("manifest")
3409 .and_then(|manifest| manifest.get("provenance"))
3410 {
3411 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
3412 return Ok(vec![control_error_frame(
3413 &frame,
3414 "invalid_manifest",
3415 format!("malformed manifest provenance: {err}"),
3416 )?]);
3417 }
3418 }
3419 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
3420 let module_id = hello_value["manifest"]["module_id"]
3421 .as_str()
3422 .unwrap_or("<unknown>");
3423 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
3424 }
3425 if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
3426 let module_id = hello_value
3427 .get("manifest")
3428 .and_then(|m| m.get("module_id"))
3429 .and_then(serde_json::Value::as_str)
3430 .unwrap_or("unknown");
3431 warn!(module_id, %reason, "dropping operation_deferral from HELLO");
3432 }
3433 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
3434 let module_id = hello_value["manifest"]["module_id"]
3435 .as_str()
3436 .unwrap_or("<unknown>");
3437 warn!(module_id, %reason, "dropping flow_operations from HELLO");
3438 }
3439 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
3440 let module_id = hello_value["manifest"]["module_id"]
3441 .as_str()
3442 .unwrap_or("<unknown>");
3443 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
3444 }
3445 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
3446 Ok(hello) => hello,
3447 Err(err) => {
3448 return Ok(vec![control_error_frame(
3449 &frame,
3450 "invalid_hello",
3451 format!("malformed HELLO body: {err}"),
3452 )?])
3453 }
3454 };
3455
3456 if hello.protocol_ver != hello.manifest.protocol_ver {
3457 return Ok(vec![control_error_frame(
3458 &frame,
3459 "invalid_manifest",
3460 format!(
3461 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
3462 hello.protocol_ver, hello.manifest.protocol_ver
3463 ),
3464 )?]);
3465 }
3466
3467 if hello.manifest.module_id.trim().is_empty() {
3468 return Ok(vec![control_error_frame(
3469 &frame,
3470 "invalid_manifest",
3471 "manifest module_id must not be empty",
3472 )?]);
3473 }
3474
3475 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
3476 Ok(negotiated_ver) => negotiated_ver,
3477 Err(message) => {
3478 return Ok(vec![control_error_frame(
3479 &frame,
3480 "version_unsupported",
3481 message,
3482 )?])
3483 }
3484 };
3485
3486 let _publication_gate = if let Some(owner) = &self.config_runtime {
3492 let gate = owner
3493 .module_gate(&hello.manifest.module_id)
3494 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3495 Some(if wait_for_gate {
3496 gate.blocking_lock_owned()
3497 } else {
3498 match gate.try_lock_owned() {
3499 Ok(guard) => guard,
3500 Err(_) => {
3501 return Ok(vec![control_error_frame(
3502 &frame,
3503 "config_delivery_deferred",
3504 "module admission is busy",
3505 )?])
3506 }
3507 }
3508 })
3509 } else {
3510 None
3511 };
3512 #[cfg(test)]
3513 if let Some(hold) = &self.hello_before_publication_hold {
3514 hold.wait();
3515 hold.wait();
3516 }
3517 let module = &hello.manifest.module_id;
3518 let required = self
3519 .config_runtime
3520 .as_ref()
3521 .is_some_and(|owner| owner.module_is_gated(module));
3522 let recovering = self
3523 .config_runtime
3524 .as_ref()
3525 .is_some_and(|owner| owner.is_recovering());
3526 if recovering
3534 && channel_spawn.as_ref().is_none_or(|spawn| {
3535 !self.supervisor.has_live_verified_channel_binding(spawn) || spawn.check().is_err()
3536 })
3537 && !(channel_spawn.is_none()
3538 && hello.launch_nonce.as_deref().is_some_and(|nonce| {
3539 self.supervisor.spawned_consumer_authorized(module, nonce)
3540 && self
3541 .supervisor
3542 .has_live_verified_program_binding(module, nonce)
3543 }))
3544 {
3545 return Ok(vec![control_error_frame(
3546 &frame,
3547 "config_gated",
3548 "recovery admits only previously bound supervised launches",
3549 )?]);
3550 }
3551 if required
3552 && (channel_spawn
3553 .as_ref()
3554 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3555 || self
3556 .config_runtime
3557 .as_ref()
3558 .is_some_and(|owner| !recovering && owner.check_module_hold(module).is_err()))
3559 {
3560 return Ok(vec![control_error_frame(
3561 &frame,
3562 "reserved_module",
3563 "gated admission requires the current supervised child's registration channel and verified image binding",
3564 )?]);
3565 }
3566 if let Some(spawn) = &channel_spawn {
3567 if let Err(code) = spawn.check() {
3568 return Ok(vec![control_error_frame(
3569 &frame,
3570 code,
3571 "registration launch is no longer live",
3572 )?]);
3573 }
3574 } else if let Some(nonce) = hello.launch_nonce.as_deref() {
3575 if required || self.supervisor.has_program_binding_record(module, nonce) {
3576 if let Err(error) = self.supervisor.check_current_program_binding(module, nonce) {
3577 let code = if error.code == "program_refused" {
3578 "program_refused"
3579 } else {
3580 "config_delivery_deferred"
3581 };
3582 self.request_refusal_close(
3583 connection_id,
3584 sink.as_ref(),
3585 CloseReason::new(code, error.detail.clone()),
3586 );
3587 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3588 }
3589 }
3590 }
3591
3592 let swap_admission = match &channel_spawn {
3600 Some(spawn) if spawn.is_candidate() => SwapHelloAdmission::Candidate,
3601 Some(_) => SwapHelloAdmission::NotSwapping,
3602 None if prefix_binding.is_some() => SwapHelloAdmission::NotSwapping,
3603 None => self
3604 .supervisor
3605 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref()),
3606 };
3607 if swap_admission == SwapHelloAdmission::ProtocolNone {
3608 warn!(
3609 module_id = %hello.manifest.module_id,
3610 connection_id = connection_id.get(),
3611 "HELLO refused: the module_id belongs to a protocol:none module, which never registers"
3612 );
3613 return Ok(vec![control_error_frame(
3616 &frame,
3617 "swap_token_invalid",
3618 format!(
3619 "module_id '{}' is a protocol:none module; a protocol:none module may not register",
3620 hello.manifest.module_id
3621 ),
3622 )?]);
3623 }
3624 if swap_admission == SwapHelloAdmission::ChannelRequired {
3625 warn!(
3626 module_id = %hello.manifest.module_id,
3627 connection_id = connection_id.get(),
3628 "HELLO refused: the module_id is gated and the HELLO did not arrive over its registration channel"
3629 );
3630 return Ok(vec![control_error_frame(
3633 &frame,
3634 "swap_token_invalid",
3635 format!(
3636 "module_id '{}' is gated; it must register over its supervised registration channel",
3637 hello.manifest.module_id
3638 ),
3639 )?]);
3640 }
3641 if swap_admission == SwapHelloAdmission::Refused {
3642 warn!(
3643 module_id = %hello.manifest.module_id,
3644 connection_id = connection_id.get(),
3645 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
3646 );
3647 return Ok(vec![control_error_frame(
3648 &frame,
3649 "swap_token_invalid",
3650 format!(
3651 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
3652 hello.manifest.module_id
3653 ),
3654 )?]);
3655 }
3656 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
3657
3658 let verification = if swap_candidate && channel_spawn.is_none() {
3663 ReservedHelloVerification {
3664 nonce_verified: true,
3665 prefix_owner: None,
3666 }
3667 } else {
3668 let verification = if let Some(binding) = &prefix_binding {
3669 if !self.supervisor.prefix_claim_authorized(
3670 &binding.registration.module_id,
3671 &hello.manifest.module_id,
3672 ) {
3673 return Ok(vec![control_error_frame(
3674 &frame,
3675 "claim_prefix_refused",
3676 "prefix ownership changed",
3677 )?]);
3678 }
3679 Ok(ReservedHelloVerification {
3680 nonce_verified: true,
3681 prefix_owner: Some(binding.registration.module_id.clone()),
3682 })
3683 } else if channel_spawn.is_some() {
3684 self.supervisor
3685 .channel_hello_verification(&hello.manifest.module_id)
3686 } else {
3687 self.supervisor.reserved_hello_verification(
3688 &hello.manifest.module_id,
3689 hello.launch_nonce.as_deref(),
3690 )
3691 };
3692 match verification {
3693 Ok(verified) => verified,
3694 Err(rejection) => {
3695 let message = match rejection {
3696 ReservedHelloRejection::Exact { module_id } => format!(
3697 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
3698 ),
3699 ReservedHelloRejection::Prefix {
3700 prefix,
3701 owner_module_id,
3702 } => format!(
3703 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
3704 hello.manifest.module_id
3705 ),
3706 };
3707 return Ok(vec![control_error_frame(
3708 &frame,
3709 "reserved_module",
3710 message,
3711 )?]);
3712 }
3713 }
3714 };
3715
3716 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
3717 &hello.manifest.module_id,
3718 hello.manifest.capabilities.as_ref(),
3719 );
3720 if let Some(refusal) = reserved_capability_refusals.first() {
3721 let capability = refusal.capability.clone();
3722 let bound_module = refusal.claimants[0].clone();
3723 log_duplicate_claim_events(reserved_capability_refusals);
3724 return Ok(vec![control_error_frame(
3725 &frame,
3726 "reserved_capability",
3727 format!(
3728 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3729 capability, bound_module, hello.manifest.module_id
3730 ),
3731 )?]);
3732 }
3733
3734 if self
3737 .forwarding
3738 .connection_has_client_routes(connection_id)
3739 .map_err(RouterError::Forwarding)?
3740 {
3741 return Ok(vec![control_error_frame(
3742 &frame,
3743 "invalid_hello",
3744 "connection has open client routes and cannot also register as a module",
3745 )?]);
3746 }
3747
3748 self.hello_launch_nonces
3756 .lock()
3757 .unwrap_or_else(|poisoned| poisoned.into_inner())
3758 .record(
3759 connection_id,
3760 if prefix_binding.is_some() || channel_spawn.is_some() {
3761 None
3762 } else {
3763 hello.launch_nonce.as_deref()
3764 },
3765 );
3766 let control_ops = effective_module_control_ops(hello.control_ops);
3767 let prefix_submodule = verification.prefix_owner.is_some();
3768 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
3771 #[cfg(test)]
3772 struct PublicationFinished(Arc<std::sync::Barrier>);
3773 #[cfg(test)]
3774 impl Drop for PublicationFinished {
3775 fn drop(&mut self) {
3776 self.0.wait();
3777 }
3778 }
3779 #[cfg(test)]
3780 let _publication_finished = self.hello_after_launch_check_hold.as_ref().map(|hold| {
3781 hold.wait();
3782 hold.wait();
3783 PublicationFinished(hold.clone())
3784 });
3785 let publication_spawn = channel_spawn
3786 .as_ref()
3787 .or_else(|| prefix_binding.as_ref().map(|binding| &binding.registration));
3788 #[cfg(test)]
3789 if prefix_binding.is_some() {
3790 self.claims.pause_prefix_hello_for_test(7);
3791 }
3792 let _registration_publication = match publication_spawn {
3793 Some(spawn) => Some(match spawn.begin_publication() {
3794 Ok(guard) => guard,
3795 Err(code) => {
3796 return Ok(vec![control_error_frame(
3797 &frame,
3798 code,
3799 "registration ended before HELLO publication",
3800 )?])
3801 }
3802 }),
3803 None => None,
3804 };
3805 let (_prefix_live, _prefix_ownership) = if let Some(binding) = &prefix_binding {
3806 let live = self.claims.lock_prefix_binding(binding, connection_id);
3807 let ownership = self.supervisor.prefix_claim_publication(
3808 &binding.registration.module_id,
3809 &hello.manifest.module_id,
3810 );
3811 if live.is_none()
3812 || ownership.is_none()
3813 || !self.claim_primary_registered(binding.spawn, &binding.registration)
3814 || binding.prefix_module_id() != Some(hello.manifest.module_id.as_str())
3815 {
3816 return Ok(vec![control_error_frame(
3817 &frame,
3818 "claim_prefix_refused",
3819 "prefix authority ended before publication",
3820 )?]);
3821 }
3822 (live, ownership)
3823 } else {
3824 (None, None)
3825 };
3826 if let Some(spawn) = publication_spawn {
3827 self.forwarding
3828 .bind_spawn(connection_id, spawn)
3829 .map_err(RouterError::Forwarding)?;
3830 }
3831 let hello_authority = match (&channel_spawn, &prefix_binding) {
3841 (Some(spawn), _) => Some(spawn.authority.clone()),
3842 (None, None) => hello.launch_nonce.as_deref().and_then(|nonce| {
3843 self.supervisor
3844 .spawned_consumer_record(&hello.manifest.module_id, nonce)
3845 }),
3846 (None, Some(_)) => None,
3847 }
3848 .map(|record| HelloAuthority {
3849 record,
3850 requested: hello
3851 .manifest
3852 .provider_declarations
3853 .clone()
3854 .unwrap_or_default(),
3855 froze: Default::default(),
3856 });
3857 #[cfg(test)]
3858 if prefix_binding.is_some() {
3859 self.claims.pause_prefix_hello_for_test(8);
3860 }
3861 if swap_candidate {
3862 let module_id = hello.manifest.module_id.clone();
3863 let mut differing_declarations = Default::default();
3864 let reply = self.register_swap_candidate(
3865 connection_id,
3866 sink,
3867 &frame,
3868 hello.manifest,
3869 negotiated_ver,
3870 control_ops,
3871 hello_ack,
3872 hello_authority.as_ref(),
3873 &mut differing_declarations,
3874 #[cfg(windows)]
3875 hello.launch_nonce.as_deref(),
3876 )?;
3877 if let Some(spawn) = &channel_spawn {
3878 if self
3879 .registry
3880 .get_module_by_connection(connection_id)
3881 .ok()
3882 .flatten()
3883 .is_some()
3884 {
3885 if let Some(sink) = primary_sink {
3886 spawn.bind_primary_sink(sink);
3887 }
3888 spawn.hello_accepted();
3889 }
3890 } else if self
3891 .registry
3892 .get_module_by_connection(connection_id)
3893 .ok()
3894 .flatten()
3895 .is_some()
3896 {
3897 self.supervisor
3898 .retire_unused_registration_channel(&module_id, true);
3899 }
3900 if self
3901 .registry
3902 .get_module_by_connection(connection_id)
3903 .ok()
3904 .flatten()
3905 .is_some()
3906 {
3907 self.file_development_widening(
3908 hello_authority.map(|authority| authority.record),
3909 differing_declarations,
3910 );
3911 }
3912 return Ok(reply);
3913 }
3914 let registration = match self
3921 .registry
3922 .register_unpublished_verified_with_control_ops(
3923 hello.manifest,
3924 negotiated_ver,
3925 connection_id,
3926 control_ops,
3927 verification.nonce_verified,
3928 ) {
3929 Ok(registration) => registration,
3930 Err(RegistryError::DuplicateModuleId { module_id }) => {
3931 return Ok(vec![control_error_frame(
3932 &frame,
3933 "duplicate_module_id",
3934 format!(
3935 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
3936 ),
3937 )?])
3938 }
3939 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3940 return Ok(vec![control_error_frame(
3941 &frame,
3942 "invalid_module_id",
3943 err.to_string(),
3944 )?])
3945 }
3946 Err(err) => {
3947 return Ok(vec![control_error_frame(
3948 &frame,
3949 "registry_error",
3950 err.to_string(),
3951 )?])
3952 }
3953 };
3954 let differing_declarations =
3955 match self.admit_hello_authority(connection_id, hello_authority.as_ref()) {
3956 Ok(differing) => differing,
3957 Err(err) => {
3958 if matches!(
3959 self.rollback_unpublished_hello(connection_id),
3960 Ok(r) if !r.is_empty()
3961 ) {
3962 crate::supervise::notify_registration_release();
3963 }
3964 return Err(RouterError::Forwarding(err));
3965 }
3966 };
3967 #[cfg(test)]
3968 self.hold_between_registry_and_forwarding();
3969
3970 let mut reply = Vec::new();
3971 let publication = self
3974 .registry
3975 .publish_registration_after(connection_id, || {
3976 if let Some(sink) = sink {
3977 let concurrency = manifest_concurrency(®istration.manifest);
3993 self.forwarding.register_module_connection_acked(
3994 connection_id,
3995 registration.manifest.module_id.clone(),
3996 negotiated_ver,
3997 concurrency,
3998 sink,
3999 crate::forwarding::ModuleHelloAck {
4000 frame: hello_ack,
4001 verified_prefix_owner: verification.prefix_owner,
4002 },
4003 )?;
4004 } else {
4005 reply.push(hello_ack);
4008 }
4009 #[cfg(test)]
4010 self.hold_between_ack_and_publication();
4011 Ok(())
4012 })
4013 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4014 let published = match publication {
4015 Ok(published) => published,
4016 Err(err) => {
4017 if matches!(
4024 self.rollback_unpublished_hello(connection_id),
4025 Ok(r) if !r.is_empty()
4026 ) {
4027 crate::supervise::notify_registration_release();
4028 }
4029 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
4030 return Ok(vec![control_error_frame(
4031 &frame,
4032 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
4033 "invalid_hello"
4034 } else {
4035 forwarding_error_code(&err)
4036 },
4037 err.to_string(),
4038 )?]);
4039 }
4040 };
4041 if !published {
4042 debug!(
4043 module_id = %registration.manifest.module_id,
4044 connection_id = connection_id.get(),
4045 "registration was removed before it could be published"
4046 );
4047 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
4048 return Ok(reply);
4049 }
4050
4051 drop(_prefix_ownership);
4052 drop(_prefix_live);
4053
4054 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
4063 info!(
4064 module_id = %registration.manifest.module_id,
4065 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
4066 );
4067 }
4068 if registration.manifest.registration_channel == Some(false)
4069 && self
4070 .supervisor
4071 .module_is_gated(®istration.manifest.module_id)
4072 && self
4073 .config_runtime
4074 .as_ref()
4075 .and_then(|owner| owner.registration_declaration(®istration.manifest.module_id))
4076 .as_deref()
4077 == Some("declared")
4078 {
4079 warn!(module_id = %registration.manifest.module_id,
4080 "HELLO registration_channel is false despite the approved channel declaration; registration remains admitted");
4081 }
4082
4083 self.apply_registration_capabilities(®istration);
4084
4085 self.supervisor
4086 .record_activity_registration(®istration.manifest.module_id);
4087
4088 info!(
4089 module_id = %registration.manifest.module_id,
4090 module_version = %registration.manifest.module_version,
4091 negotiated_ver,
4092 routable_provider = manifest_provides_routable_role(®istration.manifest),
4093 connection_id = connection_id.get(),
4094 "module registered"
4095 );
4096
4097 if let Some(spawn) = &channel_spawn {
4098 if let Some(sink) = primary_sink {
4099 spawn.bind_primary_sink(sink);
4100 }
4101 spawn.hello_accepted();
4102 } else if !prefix_submodule {
4103 self.supervisor
4104 .retire_unused_registration_channel(®istration.manifest.module_id, false);
4105 }
4106 self.file_development_widening(
4107 hello_authority.map(|authority| authority.record),
4108 differing_declarations,
4109 );
4110 #[cfg(windows)]
4111 if let Some(nonce) = hello.launch_nonce.as_deref() {
4112 self.supervisor
4113 .program_hello_accepted(®istration.manifest.module_id, nonce);
4114 }
4115
4116 #[cfg(test)]
4117 if let Some(hold) = &self.hello_inside_publication_hold {
4118 hold.wait();
4119 hold.wait();
4120 }
4121
4122 Ok(reply)
4123 }
4124
4125 #[allow(clippy::too_many_arguments)]
4142 fn register_swap_candidate(
4143 &self,
4144 connection_id: ConnectionId,
4145 sink: Option<crate::FrameSink>,
4146 frame: &Frame,
4147 manifest: ModuleManifest,
4148 negotiated_ver: u8,
4149 control_ops: Vec<String>,
4150 hello_ack: Frame,
4151 hello_authority: Option<&HelloAuthority>,
4152 differing_declarations: &mut ProviderDeclarations,
4153 #[cfg(windows)] launch_nonce: Option<&str>,
4154 ) -> Result<Vec<Frame>, RouterError> {
4155 let module_id = manifest.module_id.clone();
4156 self.supervisor
4159 .with_swap_candidate_admission(&module_id, || {
4160 let reply = self.publish_swap_candidate(
4161 connection_id,
4162 sink,
4163 frame,
4164 manifest,
4165 negotiated_ver,
4166 control_ops,
4167 hello_ack,
4168 hello_authority,
4169 differing_declarations,
4170 #[cfg(windows)]
4171 launch_nonce,
4172 );
4173 let admitted = self
4174 .registry
4175 .get_candidate(&module_id)
4176 .ok()
4177 .flatten()
4178 .is_some_and(|candidate| candidate.connection_id == connection_id);
4179 (reply, admitted)
4180 })
4181 .unwrap_or_else(|| {
4182 Ok(vec![control_error_frame(
4183 frame,
4184 "swap_token_invalid",
4185 "swap ended before candidate HELLO publication",
4186 )?])
4187 })
4188 }
4189
4190 #[allow(clippy::too_many_arguments)]
4191 fn publish_swap_candidate(
4192 &self,
4193 connection_id: ConnectionId,
4194 sink: Option<crate::FrameSink>,
4195 frame: &Frame,
4196 manifest: ModuleManifest,
4197 negotiated_ver: u8,
4198 control_ops: Vec<String>,
4199 hello_ack: Frame,
4200 hello_authority: Option<&HelloAuthority>,
4201 differing_declarations: &mut ProviderDeclarations,
4202 #[cfg(windows)] launch_nonce: Option<&str>,
4203 ) -> Result<Vec<Frame>, RouterError> {
4204 let module_id = manifest.module_id.clone();
4205 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
4206 manifest,
4207 negotiated_ver,
4208 connection_id,
4209 control_ops,
4210 ) {
4211 Ok(registration) => registration,
4212 Err(RegistryError::DuplicateModuleId { module_id }) => {
4213 return Ok(vec![control_error_frame(
4214 frame,
4215 "duplicate_module_id",
4216 format!(
4217 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
4218 ),
4219 )?])
4220 }
4221 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
4222 return Ok(vec![control_error_frame(
4223 frame,
4224 "invalid_module_id",
4225 err.to_string(),
4226 )?])
4227 }
4228 Err(err) => {
4229 return Ok(vec![control_error_frame(
4230 frame,
4231 "registry_error",
4232 err.to_string(),
4233 )?])
4234 }
4235 };
4236 *differing_declarations = match self.admit_hello_authority(connection_id, hello_authority) {
4237 Ok(differing) => differing,
4238 Err(err) => {
4239 if matches!(
4240 self.rollback_unpublished_hello(connection_id),
4241 Ok(r) if !r.is_empty()
4242 ) {
4243 crate::supervise::notify_registration_release();
4244 }
4245 return Err(RouterError::Forwarding(err));
4246 }
4247 };
4248 #[cfg(test)]
4249 self.hold_between_registry_and_forwarding();
4250 let mut reply = Vec::new();
4251 let publication = self
4252 .registry
4253 .publish_registration_after(connection_id, || {
4254 if let Some(sink) = sink {
4255 let concurrency = manifest_concurrency(®istration.manifest);
4259 self.forwarding.register_candidate_module_connection_acked(
4260 connection_id,
4261 module_id.clone(),
4262 negotiated_ver,
4263 concurrency,
4264 sink,
4265 hello_ack,
4266 )?;
4267 } else {
4268 reply.push(hello_ack);
4269 }
4270 #[cfg(test)]
4271 self.hold_between_ack_and_publication();
4272 Ok(())
4273 })
4274 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4275 let published = match publication {
4276 Ok(published) => published,
4277 Err(err) => {
4278 if matches!(
4279 self.rollback_unpublished_hello(connection_id),
4280 Ok(r) if !r.is_empty()
4281 ) {
4282 crate::supervise::notify_registration_release();
4283 }
4284 self.refuse_hello_authority(connection_id, hello_authority);
4285 return Ok(vec![control_error_frame(
4286 frame,
4287 forwarding_error_code(&err),
4288 err.to_string(),
4289 )?]);
4290 }
4291 };
4292 if !published {
4293 debug!(
4294 module_id = %module_id,
4295 connection_id = connection_id.get(),
4296 "swap candidate registration was removed before it could be published"
4297 );
4298 self.refuse_hello_authority(connection_id, hello_authority);
4299 return Ok(reply);
4300 }
4301 info!(
4302 module_id = %module_id,
4303 module_version = %registration.manifest.module_version,
4304 negotiated_ver,
4305 ready = registration.ready,
4306 connection_id = connection_id.get(),
4307 "swap candidate registered; not routable until cutover"
4308 );
4309 #[cfg(windows)]
4310 if let Some(nonce) = launch_nonce {
4311 self.supervisor.program_hello_accepted(&module_id, nonce);
4312 }
4313 Ok(reply)
4314 }
4315
4316 fn admit_hello_authority(
4322 &self,
4323 connection: ConnectionId,
4324 authority: Option<&HelloAuthority>,
4325 ) -> Result<ProviderDeclarations, ForwardingError> {
4326 let Some(authority) = authority else {
4327 return Ok(ProviderDeclarations::new());
4328 };
4329 let froze = authority
4330 .record
4331 .freeze_declarations(authority.requested.clone());
4332 authority
4333 .froze
4334 .store(froze.is_some(), std::sync::atomic::Ordering::Relaxed);
4335 self.forwarding
4336 .bind_target_authority(connection, authority.record.clone())
4337 .inspect_err(|_| self.refuse_hello_authority(connection, Some(authority)))?;
4338 Ok(froze.unwrap_or_default())
4339 }
4340
4341 fn refuse_hello_authority(&self, connection: ConnectionId, authority: Option<&HelloAuthority>) {
4346 let Some(authority) = authority else {
4347 return;
4348 };
4349 if let Err(err) = self.forwarding.unbind_target_authority(connection) {
4350 warn!(error = %err, "failed to unbind a refused HELLO from its spawn");
4351 }
4352 if authority
4353 .froze
4354 .swap(false, std::sync::atomic::Ordering::Relaxed)
4355 {
4356 authority.record.unfreeze_refused_declarations();
4357 }
4358 }
4359
4360 fn build_hello_ack(
4361 &self,
4362 frame: &Frame,
4363 negotiated_ver: u8,
4364 module_id: &str,
4365 ) -> Result<Frame, RouterError> {
4366 let ack = ModuleHelloAckBody {
4367 negotiated_ver,
4368 subc_ops: module_subc_ops(),
4369 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
4370 storage: self
4371 .storage_config
4372 .as_ref()
4373 .map(|cfg| cfg.descriptor_for(module_id)),
4374 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4375 };
4376 let body = serde_json::to_vec(&ack).map_err(|err| {
4377 RouterError::backend(
4378 0,
4379 frame.header.corr,
4380 format!("failed to encode HELLO_ACK: {err}"),
4381 )
4382 })?;
4383
4384 Frame::build_with_version(
4385 negotiated_ver,
4386 FrameType::HelloAck,
4387 control_flags(),
4388 0,
4389 0,
4390 frame.header.corr,
4391 body,
4392 )
4393 .map_err(RouterError::FrameBuild)
4394 }
4395
4396 async fn handle_client_control_request(
4397 &self,
4398 ctx: &RouteCtx,
4399 frame: Frame,
4400 request: ClientControlRequest,
4401 route_open_timings: &mut Option<RouteOpenTimings>,
4402 ) -> Result<Vec<Frame>, RouterError> {
4403 if let Some(owner) = &self.config_runtime {
4404 let module = match &request {
4405 ClientControlRequest::SupervisorRestart { module_id, .. }
4406 | ClientControlRequest::SupervisorReload { module_id }
4407 | ClientControlRequest::SupervisorSwap { module_id, .. }
4408 | ClientControlRequest::SupervisorSetEnabled {
4409 module_id,
4410 enabled: true,
4411 } => Some(module_id.as_str()),
4412 _ => None,
4413 };
4414 let rescan = matches!(
4415 &request,
4416 ClientControlRequest::SupervisorRescan { preview: false }
4417 | ClientControlRequest::SupervisorRescanChecked { preview: false, .. }
4418 );
4419 let held = if module.is_some() || rescan {
4420 let module = module.map(str::to_owned);
4421 owner
4422 .on_blocking(move |owner| owner.checked_admission_hold(module.as_deref()))
4423 .await
4424 } else {
4425 Ok(())
4426 };
4427 if let Err(error) = held {
4428 if error.code != "config_delivery_deferred" {
4429 let code = if error.code == "store_tampered" {
4430 "store_tampered"
4431 } else {
4432 "config_gated"
4433 };
4434 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
4435 }
4436 }
4437 }
4438 if let Some(operation) = privileged_operation(&request) {
4439 let caller = self.resolve_caller(ctx.connection_id).await;
4440 let op = client_control_request_op(&request);
4441 let text = prompt_text(operation, &caller);
4442 match self.privileged_operator_prompt {
4443 PrivilegedOperatorPrompt::Log => {
4444 info!(target: "subc_daemon::privileged_prompt", operation = op,
4445 target = ?operation, caller = %caller.description(), summary = %text,
4446 "privileged request would prompt the operator");
4447 }
4448 PrivilegedOperatorPrompt::Enforce => {
4449 let key =
4450 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
4451 let stamp = self
4452 .operator_apps
4453 .lock()
4454 .unwrap_or_else(|p| p.into_inner())
4455 .get(&ctx.connection_id)
4456 .cloned();
4457 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
4458 &self.operator_app_policy,
4459 &self.launch_table,
4460 stamp.as_ref(),
4461 )
4462 .then(|| self.shortcut_context(stamp, None, None));
4463 let outcome = self
4464 .forwarding
4465 .operator_confirms()
4466 .approve_privileged_with_shortcut(
4467 ctx,
4468 &frame,
4469 op,
4470 text,
4471 caller.description(),
4472 key,
4473 shortcut,
4474 )
4475 .await;
4476 if outcome.code != "confirmed" {
4477 return Ok(vec![outcome.frame(&frame)]);
4478 }
4479 }
4480 }
4481 }
4482 match request {
4483 ClientControlRequest::DaemonShutdownIntent { intent } => {
4484 if intent.as_str() != Some("restart") {
4485 return Ok(vec![control_error_frame(
4486 &frame,
4487 "invalid_params",
4488 "intent must be 'restart'",
4489 )?]);
4490 }
4491 if let Some(owner) = &self.config_runtime {
4496 match owner
4497 .on_blocking(|owner| owner.assign_daemon_restart())
4498 .await
4499 {
4500 Ok(Some(generation)) => {
4501 info!(
4502 generation,
4503 "daemon restart will warm up approved generation"
4504 )
4505 }
4506 Ok(None) => {}
4507 Err(error) => warn!(
4508 %error,
4509 "daemon restart intent not recorded; the next start boots the last active generation"
4510 ),
4511 }
4512 }
4513 let response = ClientControlResponse::DaemonShutdownIntent {
4514 intent: "restart".to_owned(),
4515 expires_at_ms: self.supervisor.announce_daemon_restart(),
4516 };
4517 Ok(vec![control_response_body_frame(
4518 &frame,
4519 &response,
4520 "ClientControlResponse::DaemonShutdownIntent",
4521 )?])
4522 }
4523 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
4524 ClientControlRequest::CatalogList { module_id } => {
4525 self.handle_catalog_list(frame, module_id)
4526 }
4527 ClientControlRequest::RouteOpen {
4528 target,
4529 identity,
4530 consumer_identity,
4531 consumer_capabilities,
4532 role_versions,
4533 busy_refusals,
4534 admission_facts,
4535 scope,
4536 ..
4537 } => {
4538 let extensions = match serde_json::from_slice::<subc_control::RouteOpenRequestExt>(
4539 &frame.body,
4540 ) {
4541 Ok(extensions) => extensions,
4542 Err(err) => {
4543 return Ok(vec![control_error_frame(
4544 &frame,
4545 "invalid_control_body",
4546 format!("malformed route.open extensions: {err}"),
4547 )?]);
4548 }
4549 };
4550 self.handle_route_open(
4551 ctx,
4552 frame,
4553 RouteOpenRequest {
4554 target,
4555 identity,
4556 consumer_identity,
4557 consumer_capabilities,
4558 role_versions,
4559 busy_refusals,
4560 admission_facts,
4561 scope,
4562 accepts_close_reasons: extensions.accepts_close_reasons,
4563 },
4564 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
4565 )
4566 .await
4567 }
4568 ClientControlRequest::RoutePoll {
4569 route_channel,
4570 route_epoch,
4571 kind,
4572 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
4573 ClientControlRequest::SupervisorList {} => {
4574 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
4575 self.agent_declarations.refuse(
4576 self,
4577 ctx,
4578 control_error_frame(
4579 &frame,
4580 "busy",
4581 "supervisor list work is at capacity; retry",
4582 )?,
4583 );
4584 return Ok(Vec::new());
4585 };
4586 let handler = self.clone();
4588 let sink = ctx.egress.clone();
4589 tokio::spawn(
4590 async move {
4591 let _permit = permit;
4592 let result = handler.handle_supervisor_list(frame.clone()).await;
4593 let replies = match result {
4594 Ok(replies) => replies,
4595 Err(error) => {
4596 control_error_frame(&frame, "backend_error", error.to_string())
4597 .into_iter()
4598 .collect()
4599 }
4600 };
4601 for reply in replies {
4602 if !matches!(
4603 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
4604 .await,
4605 Ok(Ok(()))
4606 ) {
4607 break;
4608 }
4609 }
4610 }
4611 .with_current_subscriber(),
4612 );
4613 Ok(Vec::new())
4614 }
4615 ClientControlRequest::SupervisorSpawnSnapshot {} => {
4616 self.handle_supervisor_spawn_snapshot(frame)
4617 }
4618 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
4619 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
4620 }
4621 ClientControlRequest::SupervisorRestart {
4622 module_id,
4623 drain_timeout_ms,
4624 } => {
4625 self.log_supervisor_request_received(
4626 ctx,
4627 frame.header.corr,
4628 ops::SUPERVISOR_RESTART,
4629 Some(&module_id),
4630 None,
4631 )?;
4632 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4633 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
4634 .await
4635 }
4636 ClientControlRequest::SupervisorSwap {
4637 module_id,
4638 ready_timeout_ms,
4639 } => {
4640 self.log_supervisor_request_received(
4641 ctx,
4642 frame.header.corr,
4643 ops::SUPERVISOR_SWAP,
4644 Some(&module_id),
4645 None,
4646 )?;
4647 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4648 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
4649 .await
4650 }
4651 ClientControlRequest::SupervisorReload { module_id } => {
4652 self.log_supervisor_request_received(
4653 ctx,
4654 frame.header.corr,
4655 ops::SUPERVISOR_RELOAD,
4656 Some(&module_id),
4657 None,
4658 )?;
4659 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4660 self.handle_supervisor_reload(frame, module_id, requester)
4661 .await
4662 }
4663 ClientControlRequest::SupervisorRescan { preview } => {
4664 if !preview {
4665 self.log_supervisor_request_received(
4666 ctx,
4667 frame.header.corr,
4668 ops::SUPERVISOR_RESCAN,
4669 None,
4670 None,
4671 )?;
4672 }
4673 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4674 let operation = self.handle_supervisor_rescan(
4675 frame,
4676 if preview {
4677 RescanMode::Preview
4678 } else {
4679 RescanMode::Execute
4680 },
4681 requester,
4682 );
4683 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
4684 .await
4685 }
4686 ClientControlRequest::SupervisorRescanChecked {
4687 preview,
4688 candidate,
4689 expect,
4690 } => {
4691 self.observe_control_wait(
4692 ctx,
4693 None,
4694 ops::SUPERVISOR_RESCAN,
4695 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
4696 )
4697 .await
4698 }
4699 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
4700 self.log_supervisor_request_received(
4701 ctx,
4702 frame.header.corr,
4703 ops::SUPERVISOR_RELEASE_RESERVED,
4704 Some(&module_id),
4705 None,
4706 )?;
4707 self.handle_supervisor_release_reserved(frame, module_id)
4708 .await
4709 }
4710 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
4711 self.log_supervisor_request_received(
4712 ctx,
4713 frame.header.corr,
4714 ops::SUPERVISOR_SET_ENABLED,
4715 Some(&module_id),
4716 Some(enabled),
4717 )?;
4718 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4719 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
4720 .await
4721 }
4722 ClientControlRequest::SupervisorHealthProbe { module_id } => {
4723 self.handle_supervisor_health_probe(ctx, frame, module_id)
4724 .await
4725 }
4726 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
4727 ClientControlRequest::SupervisorRoutes { module_id } => {
4728 self.handle_supervisor_routes(frame, module_id)
4729 }
4730 ClientControlRequest::SupervisorProvenance { module_id } => {
4731 self.handle_supervisor_provenance(ctx, frame, module_id)
4732 .await
4733 }
4734 ClientControlRequest::SupervisorStderrTail {
4735 module_id,
4736 max_lines,
4737 max_bytes,
4738 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
4739 ClientControlRequest::SupervisorTerminals { module_id } => {
4740 self.handle_supervisor_terminals(ctx, frame, module_id)
4741 .await
4742 }
4743 }
4744 }
4745
4746 fn log_supervisor_request_received(
4747 &self,
4748 ctx: &RouteCtx,
4749 corr: u64,
4750 op: &'static str,
4751 module_id: Option<&str>,
4752 enabled: Option<bool>,
4753 ) -> Result<(), RouterError> {
4754 let caller = self.supervisor_requester(ctx, corr)?;
4755 let caller = principal_label(&caller);
4756 info!(
4759 target: "control",
4760 op,
4761 module_id,
4762 enabled,
4763 connection_id = ctx.connection_id.get(),
4764 caller = %caller,
4765 "supervisor request received"
4766 );
4767 Ok(())
4768 }
4769
4770 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
4771 Ok(self
4772 .registry
4773 .get_module_by_connection(ctx.connection_id)
4774 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4775 .map(|registration| Principal::Reserved {
4776 module_id: registration.manifest.module_id,
4777 })
4778 .unwrap_or(Principal::Direct))
4779 }
4780
4781 fn handle_module_control_request(
4782 &self,
4783 connection_id: ConnectionId,
4784 frame: Frame,
4785 request: ModuleControlRequestFromModule,
4786 ) -> Result<Vec<Frame>, RouterError> {
4787 match request {
4788 ModuleControlRequestFromModule::RouteAuthorize {
4789 principal,
4790 scope,
4791 target,
4792 expected,
4793 } => self.handle_route_authorize(
4794 connection_id,
4795 frame,
4796 principal,
4797 scope,
4798 target,
4799 expected,
4800 ),
4801 ModuleControlRequestFromModule::CatalogUpdate {
4802 provides,
4803 capabilities,
4804 ready,
4805 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
4806 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
4807 self.handle_catalog_update_role_declarations(
4808 connection_id,
4809 frame,
4810 role_declarations,
4811 )
4812 }
4813 ModuleControlRequestFromModule::LiveRoots {} => {
4814 let registered = self
4815 .registry
4816 .get_module_by_connection(connection_id)
4817 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4818 let Some(registration) = registered else {
4819 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
4820 };
4821 let response = self
4822 .forwarding
4823 .live_roots(®istration.manifest.module_id)
4824 .map_err(RouterError::Forwarding)?;
4825 Ok(vec![control_response_body_frame(
4826 &frame,
4827 &response,
4828 "ModuleControlResponseToModule::LiveRoots",
4829 )?])
4830 }
4831 ModuleControlRequestFromModule::AgentCommands {
4832 started,
4833 finished,
4834 known,
4835 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
4836 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
4837 self.handle_scope_sync(connection_id, frame, generation, scopes)
4838 }
4839 ModuleControlRequestFromModule::ScopeApply {
4840 generation,
4841 upsert,
4842 end,
4843 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
4844 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
4845 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
4846 }
4847 _ => Ok(vec![control_error_frame(
4848 &frame,
4849 "unsupported_control_frame",
4850 "unsupported module control operation",
4851 )?]),
4852 }
4853 }
4854
4855 fn handle_scope_served_by(
4856 &self,
4857 connection_id: ConnectionId,
4858 frame: Frame,
4859 ) -> Result<Vec<Frame>, RouterError> {
4860 let registered = self
4861 .registry
4862 .get_module_by_connection(connection_id)
4863 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4864 let Some(registered) = registered else {
4865 return Ok(vec![control_error_frame(
4866 &frame,
4867 "not_registered",
4868 "scope.served_by requires an active module registration owned by this connection",
4869 )?]);
4870 };
4871 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
4872 {
4873 Ok(request) => request,
4874 Err(err) => {
4875 return Ok(vec![control_error_frame(
4876 &frame,
4877 "invalid_control_body",
4878 format!("malformed scope.served_by body: {err}"),
4879 )?])
4880 }
4881 };
4882 let answer = self
4883 .scopes
4884 .read()
4885 .unwrap_or_else(|poisoned| poisoned.into_inner())
4886 .served_by(
4887 ®istered.manifest.module_id,
4888 &request.scope_ref,
4889 request.scope_epoch,
4890 &request.module_id,
4891 );
4892 let reply = subc_control::ScopeServedByReply::new(
4893 answer,
4894 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4895 );
4896 Ok(vec![control_response_body_frame(
4897 &frame,
4898 &reply,
4899 "ScopeServedByReply",
4900 )?])
4901 }
4902
4903 fn handle_operator_confirm(
4904 &self,
4905 ctx: &RouteCtx,
4906 frame: Frame,
4907 ) -> Result<Vec<Frame>, RouterError> {
4908 use crate::operator_confirm::{audit, Outcome};
4909 let registration = self
4910 .registry
4911 .get_module_by_connection(ctx.connection_id)
4912 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4913 let Some(registration) = registration else {
4914 let outcome = Outcome::refusal("not_registered");
4915 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4916 return Ok(vec![outcome.frame(&frame)]);
4917 };
4918 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
4919 Ok(request) => request,
4920 Err(_) => {
4921 let outcome = Outcome::refusal("invalid_control_body");
4922 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4923 return Ok(vec![outcome.frame(&frame)]);
4924 }
4925 };
4926 let declared = matches!(
4927 request.kind,
4928 Some(subc_protocol::session::OperatorConfirmKind::Operation)
4929 ) && request.operation.as_ref().is_some_and(|operation| {
4930 registration
4931 .manifest
4932 .confirm_operations
4933 .as_ref()
4934 .is_some_and(|operations| operations.contains(operation))
4935 });
4936 let module_id = registration.manifest.module_id;
4937 let nonce = self
4941 .hello_launch_nonces
4942 .lock()
4943 .unwrap_or_else(|p| p.into_inner())
4944 .nonce(ctx.connection_id)
4945 .map(str::to_owned);
4946 let nonce_proven = self
4949 .confirm_channel_proven(ctx.connection_id, &module_id)
4950 .unwrap_or_else(|| {
4951 nonce.as_deref().is_some_and(|nonce| {
4952 self.supervisor
4953 .spawned_consumer_authorized(&module_id, nonce)
4954 })
4955 });
4956 let confirms = self.forwarding.operator_confirms();
4957 self.forwarding
4958 .with_operator_route(
4959 ctx.connection_id,
4960 request.route_channel,
4961 request.route_epoch,
4962 |binding| {
4963 let shortcut = binding
4964 .filter(|binding| {
4965 nonce_proven
4966 && crate::operator_confirm::valid_summary(&request.summary)
4967 && declared
4968 && crate::operator_confirm::ShortcutContext::route_eligible(
4969 &self.operator_app_policy,
4970 &self.launch_table,
4971 binding,
4972 )
4973 })
4974 .map(|binding| {
4975 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
4976 &self.operator_app_policy,
4977 binding,
4978 );
4979 let stamp = if phone.is_some() {
4980 None
4981 } else {
4982 binding.operator_app.clone()
4983 };
4984 self.shortcut_context(
4985 stamp,
4986 phone,
4987 Some((request.route_channel, request.route_epoch)),
4988 )
4989 });
4990 confirms.admit_with_shortcut(
4991 ctx,
4992 frame,
4993 module_id,
4994 nonce_proven,
4995 request,
4996 binding,
4997 shortcut,
4998 )
4999 },
5000 )
5001 .map_err(RouterError::Forwarding)
5002 }
5003
5004 fn shortcut_context(
5005 &self,
5006 stamp: Option<crate::operator_apps::OperatorAppStamp>,
5007 phone_fingerprint: Option<String>,
5008 route: Option<(u16, u32)>,
5009 ) -> crate::operator_confirm::ShortcutContext {
5010 crate::operator_confirm::ShortcutContext {
5011 ledger: self.activity.clone(),
5012 forwarding: self.forwarding.clone(),
5013 launches: self.launch_table.clone(),
5014 stamp,
5015 phone_fingerprint,
5016 route,
5017 connections: self.operator_apps.clone(),
5018 #[cfg(test)]
5019 hooks: Default::default(),
5020 }
5021 }
5022
5023 #[cfg(any(test, feature = "test-support"))]
5025 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
5026 self.operator_app_policy.phone_enabled = enabled;
5027 self
5028 }
5029
5030 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
5033 let mut scopes = ScopeRecords::from(scopes);
5034 for (index, record) in scopes.records.iter_mut().enumerate() {
5035 if let Some(refusal) = Self::canonicalize_workdir(record) {
5036 scopes.refusals.insert(index, refusal);
5037 }
5038 }
5039 scopes
5040 }
5041
5042 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
5043 let workdir = record.workdir.as_mut()?;
5044 let path = std::path::Path::new(workdir);
5045 if !path.is_absolute() {
5046 return Some((
5047 error_codes::INVALID_WORKDIR,
5048 "workdir must be absolute".to_string(),
5049 ));
5050 }
5051 match ProjectRootId::from_path_allowing_missing(path) {
5052 Ok(root) => match root.as_path().to_str() {
5053 Some(canonical) => {
5054 *workdir = canonical.to_string();
5055 None
5056 }
5057 None => Some((
5058 error_codes::INVALID_WORKDIR,
5059 "canonical workdir is not UTF-8".to_string(),
5060 )),
5061 },
5062 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
5063 }
5064 }
5065
5066 fn handle_scope_sync(
5070 &self,
5071 connection_id: ConnectionId,
5072 frame: Frame,
5073 generation: u64,
5074 scopes: Vec<ScopeRecord>,
5075 ) -> Result<Vec<Frame>, RouterError> {
5076 self.handle_scope_change(connection_id, frame, generation, scopes, None)
5077 }
5078
5079 fn handle_scope_change(
5080 &self,
5081 connection_id: ConnectionId,
5082 frame: Frame,
5083 generation: u64,
5084 scopes: Vec<ScopeRecord>,
5085 end: Option<Vec<ScopeEnd>>,
5086 ) -> Result<Vec<Frame>, RouterError> {
5087 let is_apply = end.is_some();
5088 let Some(registration) = self
5089 .registry
5090 .get_module_by_connection(connection_id)
5091 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5092 else {
5093 return Ok(vec![control_error_frame(
5094 &frame,
5095 "not_registered",
5096 "scope.sync and scope.apply require an active module registration owned by this connection",
5097 )?]);
5098 };
5099 let owner = registration.manifest.module_id;
5100 let scopes = Self::prepare_scope_records(scopes);
5101 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
5102 let is_current_launch = |connection: ConnectionId| {
5103 self.connection_is_current_launch(&owner, connection, current_nonce.as_deref())
5104 };
5105 let mut table = self
5110 .scopes
5111 .write()
5112 .unwrap_or_else(|poisoned| poisoned.into_inner());
5113 let outcome = match end {
5114 Some(end) => table.apply(
5115 &owner,
5116 connection_id,
5117 is_current_launch,
5118 generation,
5119 scopes,
5120 end,
5121 ),
5122 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
5123 };
5124 let drained = match &outcome {
5125 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
5126 Err(_) => Vec::new(),
5127 };
5128 let has_deadlines = outcome.is_ok() && table.has_deadlines();
5129 drop(table);
5130 if has_deadlines {
5131 self.scope_deadline_added.notify_one();
5132 }
5133 match outcome {
5134 Ok(applied) => {
5135 let counts = ScopeOutcomeCounts::of(&applied.results);
5136 info!(
5137 owner = %owner,
5138 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5139 generation,
5140 records = applied.results.len(),
5141 created = counts.created,
5142 replaced = counts.replaced,
5143 updated = counts.updated,
5144 unchanged = counts.unchanged,
5145 refused = counts.refused,
5146 ended = applied.ended.len(),
5147 tag_changes = applied.tag_changes.len(),
5148 routes_closed = drained.len(),
5149 "scope change accepted"
5150 );
5151 for refused in applied
5157 .results
5158 .iter()
5159 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
5160 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
5161 {
5162 warn!(
5163 owner = %owner,
5164 generation,
5165 scope_ref = %refused.scope_ref,
5166 scope_epoch = refused.scope_epoch,
5167 code = refused.code.as_deref().unwrap_or(""),
5168 "scope record refused"
5169 );
5170 }
5171 self.close_scope_drained_routes(drained);
5172 let response = if is_apply {
5173 ModuleControlResponseToModule::ScopeApply {
5174 generation,
5175 results: applied.results,
5176 end_results: applied.end_results,
5177 ended: applied.ended,
5178 }
5179 } else {
5180 ModuleControlResponseToModule::ScopeSync {
5181 generation,
5182 results: applied.results,
5183 ended: applied.ended,
5184 }
5185 };
5186 Ok(vec![control_response_body_frame(
5187 &frame,
5188 &response,
5189 "scope change response",
5190 )?])
5191 }
5192 Err(refusal) => {
5193 info!(
5194 owner = %owner,
5195 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5196 generation,
5197 code = refusal.code,
5198 "scope change refused"
5199 );
5200 Ok(vec![control_error_frame(
5201 &frame,
5202 refusal.code,
5203 refusal.message,
5204 )?])
5205 }
5206 }
5207 }
5208
5209 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
5211 scope_expiry::ScopeExpiryContext::new(self).close_scope_drained_routes(drained);
5212 }
5213
5214 fn handle_route_authorize(
5220 &self,
5221 connection_id: ConnectionId,
5222 frame: Frame,
5223 principal: Principal,
5224 scope: subc_protocol::scope::RouteAuthorizeScope,
5225 target: String,
5226 expected: subc_protocol::scope::RouteAuthorizeExpected,
5227 ) -> Result<Vec<Frame>, RouterError> {
5228 use crate::scopes::admission::{admit_route, AdmissionMode};
5229 use subc_protocol::scope::{
5230 RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
5231 };
5232 let registry = self
5233 .registry
5234 .route_admission_snapshot()
5235 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5236 let owner_configured = match &scope.owner {
5237 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5238 _ => false,
5239 };
5240 let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
5241 let answer = {
5242 let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
5243 let permitted = registry
5244 .iter()
5245 .find(|r| r.connection_id == connection_id)
5246 .is_some_and(|r| {
5247 scopes.is_federated_owner(&Principal::Reserved {
5248 module_id: r.module_id.clone(),
5249 }) && r.capabilities.as_ref().is_some_and(|caps| {
5250 caps.provides
5251 .iter()
5252 .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
5253 })
5254 });
5255 if !permitted {
5256 drop(scopes);
5257 return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
5258 "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
5259 }
5260 admit_route(
5261 &scopes,
5262 ®istry,
5263 &principal,
5264 &target,
5265 Some(&selector),
5266 owner_configured,
5267 AdmissionMode::Authorize(&expected),
5268 )
5269 };
5270 let response = match answer {
5271 Ok(admission) => {
5272 let mut response = RouteAuthorizeResponse::new(
5273 RouteAuthorizeOutcome::Authorized,
5274 "route admission authorized",
5275 );
5276 response.content_version = admission.content_version;
5277 response
5278 }
5279 Err(refusal) => refusal.response(),
5280 };
5281 Ok(vec![control_response_body_frame(
5282 &frame,
5283 &response,
5284 "RouteAuthorizeResponse",
5285 )?])
5286 }
5287
5288 fn handle_scope_describe(
5291 &self,
5292 connection_id: ConnectionId,
5293 frame: Frame,
5294 owner: Principal,
5295 scope_ref: String,
5296 ) -> Result<Vec<Frame>, RouterError> {
5297 let registered = self
5298 .registry
5299 .get_module_by_connection(connection_id)
5300 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5301 let Some(registered) = registered else {
5302 return Ok(vec![control_error_frame(
5303 &frame,
5304 "not_registered",
5305 "scope.describe requires an active module registration owned by this connection",
5306 )?]);
5307 };
5308 let scopes = self
5309 .scopes
5310 .read()
5311 .unwrap_or_else(|poisoned| poisoned.into_inner());
5312 if !scopes.reader_permitted(&owner, registered.manifest.capabilities.as_ref()) {
5313 return Ok(vec![control_error_frame(
5314 &frame,
5315 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
5316 "scope.describe of the federated owner requires scope-federated/v1",
5317 )?]);
5318 }
5319 let mut description = scopes.describe(&owner, &scope_ref);
5320 drop(scopes);
5321 if owner
5322 != (Principal::Reserved {
5323 module_id: registered.manifest.module_id.clone(),
5324 })
5325 {
5326 if let Some(facts) = description
5327 .stamp
5328 .as_mut()
5329 .and_then(|stamp| stamp.attributes.federated.as_mut())
5330 {
5331 facts.stamp_required = false;
5332 }
5333 }
5334 if !registered
5337 .manifest
5338 .capabilities
5339 .as_ref()
5340 .is_some_and(|caps| {
5341 caps.provides
5342 .iter()
5343 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5344 })
5345 {
5346 if let Some(stamp) = description.stamp.as_mut() {
5347 stamp.attributes.on_behalf_of_agent = None;
5348 }
5349 }
5350 if !registered
5351 .manifest
5352 .capabilities
5353 .as_ref()
5354 .is_some_and(|caps| {
5355 caps.provides
5356 .iter()
5357 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5358 })
5359 {
5360 if let Some(stamp) = description.stamp.as_mut() {
5361 stamp.attributes.flow_version = None;
5362 stamp.attributes.flow_code_hash = None;
5363 stamp.attributes.flow_install_id = None;
5364 }
5365 }
5366 let owner_configured = match &owner {
5367 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5372 _ => false,
5373 };
5374 let response = ModuleControlResponseToModule::ScopeDescribe {
5375 status: description.status,
5376 scope_epoch: description.scope_epoch,
5377 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
5378 owner_synced: description.owner_synced,
5379 owner_configured,
5380 scope: description.stamp,
5381 };
5382 Ok(vec![control_response_body_frame(
5383 &frame,
5384 &response,
5385 "ModuleControlResponseToModule::ScopeDescribe",
5386 )?])
5387 }
5388
5389 fn handle_catalog_update_role_declarations(
5390 &self,
5391 connection_id: ConnectionId,
5392 frame: Frame,
5393 role_declarations: serde_json::Value,
5394 ) -> Result<Vec<Frame>, RouterError> {
5395 if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
5396 return Ok(vec![control_error_frame(
5397 &frame,
5398 "invalid_role_declarations",
5399 err.to_string(),
5400 )?]);
5401 }
5402 let declarations = serde_json::from_value(role_declarations)
5403 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5404 if self
5405 .registry
5406 .replace_role_declarations_for_connection(connection_id, declarations)
5407 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5408 .is_none()
5409 {
5410 return Ok(vec![control_error_frame(
5411 &frame,
5412 "not_registered",
5413 "catalog.update_role_declarations requires an active module registration owned by this connection",
5414 )?]);
5415 }
5416 control_response_body_frame(
5417 &frame,
5418 &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
5419 "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
5420 )
5421 .map(|frame| vec![frame])
5422 }
5423
5424 fn handle_catalog_update(
5425 &self,
5426 connection_id: ConnectionId,
5427 frame: Frame,
5428 provides: Vec<ProviderRole>,
5429 capabilities: Option<CapabilityDeclarations>,
5430 ready: Option<bool>,
5431 ) -> Result<Vec<Frame>, RouterError> {
5432 self.refresh_capability_requirements();
5433 let Some(registration) = self
5434 .registry
5435 .get_module_by_connection(connection_id)
5436 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5437 else {
5438 return Ok(vec![control_error_frame(
5439 &frame,
5440 "not_registered",
5441 "catalog.update requires an active module registration owned by this connection",
5442 )?]);
5443 };
5444
5445 if let Some(message) =
5446 catalog_update_frozen_field_message(®istration.manifest, &provides)
5447 {
5448 return Ok(vec![control_error_frame(
5449 &frame,
5450 "catalog_update_frozen_field",
5451 message,
5452 )?]);
5453 }
5454
5455 let mut candidate = registration.manifest.clone();
5456 candidate.provides = provides.clone();
5457 candidate.capabilities = capabilities
5458 .clone()
5459 .or_else(|| registration.manifest.capabilities.clone());
5460 if let Some(rich_ui) = &candidate.rich_ui {
5461 if let Err(err) = rich_ui.validate() {
5462 return Ok(vec![control_error_frame(
5463 &frame,
5464 "invalid_manifest",
5465 err.to_string(),
5466 )?]);
5467 }
5468 }
5469 if let Err(err) = candidate.validate_capability_grammar() {
5470 return Ok(vec![control_error_frame(
5471 &frame,
5472 "invalid_capability_grammar",
5473 err.to_string(),
5474 )?]);
5475 }
5476
5477 let mut conflicts = self
5480 .capability_evaluator
5481 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
5482 if let Some(conflict) = conflicts.first() {
5483 let message = format!(
5484 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
5485 conflict.capability, conflict.claimants[0], candidate.module_id
5486 );
5487 for conflict in &mut conflicts {
5488 conflict.source = DuplicateClaimSource::CatalogUpdate;
5489 }
5490 log_duplicate_claim_events(conflicts);
5491 return Ok(vec![control_error_frame(
5492 &frame,
5493 "reserved_capability",
5494 message,
5495 )?]);
5496 }
5497
5498 let updated = match self.registry.replace_catalog_for_connection(
5499 connection_id,
5500 provides,
5501 capabilities,
5502 ready,
5503 ) {
5504 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => {
5505 return Ok(vec![control_error_frame(
5506 &frame,
5507 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5508 "scope-federated/v1 is fixed at HELLO; restart the module to change it",
5509 )?])
5510 }
5511 Err(crate::registry::RegistryError::CallerAttestationCapabilityChanged) => {
5512 return Ok(vec![control_error_frame(
5513 &frame,
5514 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5515 "caller-attestation/v1 is fixed at HELLO; restart the module to change it",
5516 )?])
5517 }
5518 result => {
5519 result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5520 }
5521 };
5522 if updated.is_none() {
5523 return Ok(vec![control_error_frame(
5524 &frame,
5525 "not_registered",
5526 "catalog.update requires an active module registration owned by this connection",
5527 )?]);
5528 }
5529 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
5530 log_duplicate_claim_events(
5531 self.capability_evaluator
5532 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
5533 );
5534 }
5535 if capability_census_trigger(
5536 registration.manifest.capabilities.as_ref(),
5537 updated
5538 .as_ref()
5539 .and_then(|entry| entry.manifest.capabilities.as_ref()),
5540 ) {
5541 self.enforce_capability_denies();
5542 }
5543 self.refresh_capability_requirements();
5544
5545 let response = ModuleControlResponseToModule::CatalogUpdate {};
5546 control_response_body_frame(
5547 &frame,
5548 &response,
5549 "ModuleControlResponseToModule::CatalogUpdate",
5550 )
5551 .map(|frame| vec![frame])
5552 }
5553
5554 pub(crate) fn with_activity_ledger_config(
5555 mut self,
5556 run_dir: &Path,
5557 incarnation: String,
5558 config: crate::activity::ActivityConfig,
5559 ) -> Self {
5560 self.activity = Some(crate::activity::ActivityLedger::open_config(
5561 run_dir.join("activity.jsonl"),
5562 incarnation,
5563 config,
5564 ));
5565 self.supervisor.set_activity(self.activity.clone());
5566 self
5567 }
5568
5569 pub(crate) async fn configure_activity(
5570 &self,
5571 config: crate::activity::ActivityConfig,
5572 ) -> Result<(), &'static str> {
5573 match &self.activity {
5574 Some(ledger) => ledger.configure(config).await,
5575 None => Ok(()),
5576 }
5577 }
5578
5579 fn cancel_activity_subscription(
5580 &self,
5581 ctx: &RouteCtx,
5582 frame: &Frame,
5583 ) -> Result<bool, RouterError> {
5584 let task = self
5585 .activity_streams
5586 .lock()
5587 .unwrap_or_else(|p| p.into_inner())
5588 .remove(&(ctx.connection_id, frame.header.corr));
5589 let Some(task) = task else {
5590 return Ok(false);
5591 };
5592 task.abort();
5593 let end = Frame::build_with_version(
5594 response_version(frame),
5595 FrameType::StreamEnd,
5596 control_flags(),
5597 0,
5598 0,
5599 frame.header.corr,
5600 Vec::new(),
5601 )
5602 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
5603 let sink = ctx.egress.clone();
5604 tokio::spawn(async move {
5605 let _ = sink.send(end).await;
5606 });
5607 Ok(true)
5608 }
5609
5610 async fn handle_activity_read(
5611 &self,
5612 ctx: &RouteCtx,
5613 frame: Frame,
5614 ) -> Result<Vec<Frame>, RouterError> {
5615 #[cfg(test)]
5616 {
5617 let pause = self
5618 .activity_request_pause
5619 .lock()
5620 .unwrap()
5621 .clone()
5622 .filter(|pause| pause.connection == ctx.connection_id);
5623 if let Some(pause) = pause {
5624 let _cancelled = ActivityRequestCancellation(pause.clone());
5625 pause.started.notify_one();
5626 std::future::pending::<()>().await;
5627 }
5628 }
5629 use subc_control::{
5630 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
5631 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
5632 };
5633 #[derive(Deserialize)]
5634 struct Identity {
5635 #[serde(default)]
5636 consumer_identity: Option<ConsumerIdentity>,
5637 }
5638 let mut bound_spawn = None;
5642 let legacy = match self.activity_reader(ctx.connection_id) {
5643 channel_operations::ActivityReader::Module { module_id, spawn } => {
5644 bound_spawn = Some(spawn);
5645 Some(Ok(Principal::Reserved { module_id }))
5646 }
5647 channel_operations::ActivityReader::Refused => {
5649 return Ok(vec![control_error_frame(
5650 &frame,
5651 ACTIVITY_READ_FORBIDDEN,
5652 "helper, relay and retired connections cannot read activity",
5653 )?]);
5654 }
5655 channel_operations::ActivityReader::Legacy => None,
5656 };
5657 let principal = match legacy {
5661 Some(principal) => principal,
5662 None => match self
5663 .forwarding
5664 .module_id_for_connection(ctx.connection_id)
5665 .map_err(RouterError::Forwarding)?
5666 {
5667 Some(module_id) => Ok(Principal::Reserved { module_id }),
5668 None => {
5669 let identity: Identity = match serde_json::from_slice(&frame.body) {
5670 Ok(identity) => identity,
5671 Err(_) => {
5672 return Ok(vec![control_error_frame(
5673 &frame,
5674 ACTIVITY_MALFORMED,
5675 "malformed activity read",
5676 )?])
5677 }
5678 };
5679 self.route_open_principal(
5680 ctx.connection_id,
5681 &frame,
5682 identity.consumer_identity,
5683 )?
5684 }
5685 },
5686 };
5687 let admitted = matches!(principal, Ok(Principal::Direct))
5688 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
5689 if !admitted {
5690 return Ok(vec![control_error_frame(
5691 &frame,
5692 ACTIVITY_READ_FORBIDDEN,
5693 "activity reads require an operator or trusted mirror principal",
5694 )?]);
5695 }
5696 let Some(ledger) = &self.activity else {
5697 return Ok(vec![control_error_frame(
5698 &frame,
5699 ACTIVITY_UNAVAILABLE,
5700 "activity ledger unavailable",
5701 )?]);
5702 };
5703 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
5704 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
5705 {
5706 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
5707 Ok(request) => ledger.query(request).await,
5708 Err(_) => Err(ACTIVITY_MALFORMED),
5709 };
5710 return match result {
5711 Ok(reply) => Ok(vec![control_response_body_frame(
5712 &frame,
5713 &reply,
5714 ops::ACTIVITY_QUERY,
5715 )?]),
5716 Err(code) => Ok(vec![control_error_frame(
5717 &frame,
5718 code,
5719 "activity query refused",
5720 )?]),
5721 };
5722 }
5723 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
5724 Ok(request) => request,
5725 Err(_) => {
5726 return Ok(vec![control_error_frame(
5727 &frame,
5728 ACTIVITY_MALFORMED,
5729 "malformed activity subscription",
5730 )?])
5731 }
5732 };
5733 let mut subscription = match ledger.subscribe(request.since).await {
5734 Ok(subscription) => subscription,
5735 Err(code) => {
5736 return Ok(vec![control_error_frame(
5737 &frame,
5738 code,
5739 "activity subscription refused",
5740 )?])
5741 }
5742 };
5743 let key = (ctx.connection_id, frame.header.corr);
5744 let sink = ctx.egress.clone();
5745 let mut streams = self
5746 .activity_streams
5747 .lock()
5748 .unwrap_or_else(|p| p.into_inner());
5749 streams.retain(|_, task| !task.is_finished());
5750 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
5751 return Ok(vec![control_error_frame(
5752 &frame,
5753 "duplicate_subscription",
5754 "activity subscription correlation id already in use",
5755 )?]);
5756 }
5757 let stream = async move {
5758 loop {
5759 let entry = if let Some(entry) = subscription.replay.pop_front() {
5760 Some(entry)
5761 } else {
5762 subscription.receiver.recv().await
5763 };
5764 let Some(entry) = entry else {
5765 break;
5766 };
5767 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
5768 return;
5769 };
5770 let Ok(next) = Frame::build_with_version(
5771 response_version(&frame),
5772 FrameType::StreamData,
5773 control_flags(),
5774 0,
5775 0,
5776 frame.header.corr,
5777 body,
5778 ) else {
5779 return;
5780 };
5781 if sink.send(next).await.is_err() {
5782 return;
5783 }
5784 }
5785 if let Some(code) = subscription.terminal_code() {
5788 if let Ok(end) =
5789 control_error_frame(&frame, code, "activity subscriber fell behind")
5790 {
5791 let _ = sink.send(end).await;
5792 }
5793 }
5794 };
5795 let task = tokio::spawn(async move {
5799 match bound_spawn {
5800 Some(spawn) => {
5801 tokio::select! {
5802 () = stream => {}
5803 () = spawn.wait_closed() => {}
5804 }
5805 }
5806 None => stream.await,
5807 }
5808 });
5809 streams.insert(key, task.abort_handle());
5810 Ok(Vec::new())
5811 }
5812
5813 async fn handle_activity_record(
5814 &self,
5815 ctx: &RouteCtx,
5816 frame: Frame,
5817 ) -> Result<Vec<Frame>, RouterError> {
5818 use subc_control::{
5819 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
5820 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
5821 };
5822 let result = async {
5823 let module = self
5824 .forwarding
5825 .module_id_for_connection(ctx.connection_id)
5826 .ok()
5827 .flatten()
5828 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
5829 let request: ActivityRecordRequest =
5830 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
5831 crate::activity::validate(&module, &request)?;
5832 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
5833 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
5834 let at_ms = std::time::SystemTime::now()
5835 .duration_since(std::time::UNIX_EPOCH)
5836 .unwrap_or_default()
5837 .as_millis() as u64;
5838 ledger.record(module, request, actor, at_ms).await
5839 }
5840 .await;
5841 match result {
5842 Ok(entry) => Ok(vec![control_response_body_frame(
5843 &frame,
5844 &ActivityRecordResponse { seq: entry.seq },
5845 "activity.record",
5846 )?]),
5847 Err(code) => Ok(vec![control_error_frame(
5848 &frame,
5849 code,
5850 "activity recording refused",
5851 )?]),
5852 }
5853 }
5854
5855 fn activity_actor(
5856 &self,
5857 connection: ConnectionId,
5858 module: &str,
5859 on: Option<&subc_control::ActivityOn>,
5860 ) -> Result<subc_control::ActivityActor, &'static str> {
5861 use crate::forwarding::{DataRoute, DataRouteState};
5862 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
5863 let principal = |id: &str| Principal::Reserved {
5864 module_id: id.to_owned(),
5865 };
5866 let mut actor = ActivityActor {
5867 principal: principal(module),
5868 scope_owner: None,
5869 agent_id: None,
5870 run_id: None,
5871 flow_id: None,
5872 project_root: None,
5873 workdir: None,
5874 project_id: None,
5875 };
5876 let scope = match on {
5877 None => return Ok(actor),
5878 Some(ActivityOn::Scope(scope)) => {
5879 let selector = subc_protocol::scope::ScopeSelector::new(
5880 principal(module),
5881 scope.scope_ref.clone(),
5882 Some(scope.epoch),
5883 );
5884 self.scopes
5885 .read()
5886 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5887 .attribute(&principal(module), module, &selector, true)
5888 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5889 }
5890 Some(ActivityOn::Route(route)) => {
5891 let DataRoute::Module(DataRouteState::Bound(binding)) = self
5892 .forwarding
5893 .lookup_data_route(connection, route.channel, route.epoch)
5894 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5895 else {
5896 return Err(ACTIVITY_ACTOR_UNKNOWN);
5897 };
5898 if binding.module_id != module {
5899 return Err(ACTIVITY_ACTOR_UNKNOWN);
5900 }
5901 actor.principal = binding.principal.clone();
5902 actor.project_root = binding
5903 .project_root
5904 .as_ref()
5905 .map(|root| root.as_path().to_path_buf());
5906 let Some(bound) = &binding.scope else {
5907 return Ok(actor);
5908 };
5909 let owner = principal(&bound.owner);
5910 let selector = subc_protocol::scope::ScopeSelector::new(
5911 owner.clone(),
5912 bound.scope_ref.clone(),
5913 Some(bound.tag.scope_epoch),
5914 );
5915 let admitted = self
5916 .scopes
5917 .read()
5918 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5919 .attribute(&owner, module, &selector, true)
5920 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
5921 if admitted.tag != bound.tag {
5922 return Err(ACTIVITY_ACTOR_UNKNOWN);
5923 }
5924 admitted
5925 }
5926 };
5927 actor.scope_owner = Some(scope.stamp.owner);
5928 actor.agent_id = scope.stamp.attributes.agent_id;
5929 actor.run_id = scope.stamp.attributes.run_id;
5930 actor.flow_id = scope.stamp.attributes.flow_id;
5931 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
5932 actor.project_id = scope.stamp.project_id;
5933 Ok(actor)
5934 }
5935
5936 fn record_control_usage(&self, connection: ConnectionId, op: &str) {
5937 let module_id = self
5938 .claims
5939 .binding(connection)
5940 .filter(|binding| {
5941 binding.consumer_live(connection) && binding.registration.authority.alive()
5942 })
5943 .map(|binding| binding.registration.module_id.clone())
5944 .or_else(|| {
5945 self.registry
5946 .module_id_for_connection(connection)
5947 .ok()
5948 .flatten()
5949 });
5950 let principal = module_id
5951 .map(|module_id| format!("reserved:{module_id}"))
5952 .unwrap_or_else(|| "direct".to_owned());
5953 self.control_usage
5954 .lock()
5955 .unwrap_or_else(|poisoned| poisoned.into_inner())
5956 .record(op, &principal);
5957 }
5958
5959 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5960 self.refresh_capability_requirements();
5961 let mut counters = self.counters.snapshot();
5967 if let Some(obj) = counters.as_object_mut() {
5968 obj.insert(
5969 "control_usage".into(),
5970 self.control_usage
5971 .lock()
5972 .unwrap_or_else(|poisoned| poisoned.into_inner())
5973 .snapshot(),
5974 );
5975 }
5976 let losses = self.resource_tracker.loss_diagnostics();
5977 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
5978 if let Some(obj) = counters.as_object_mut() {
5979 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
5980 }
5981 }
5982 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
5983 obj.insert(
5984 "activity_unparseable_lines".into(),
5985 ledger.read_errors().into(),
5986 );
5987 obj.insert(
5988 "activity_lifecycle_append_failures".into(),
5989 ledger.lifecycle_failures().into(),
5990 );
5991 obj.insert(
5992 "activity_started_digest_skips".into(),
5993 ledger.lifecycle_digest_skips().into(),
5994 );
5995 }
5996 if let (Ok((connections_with_routes, max)), Some(obj)) = (
5997 self.forwarding.client_route_concentration(),
5998 counters.as_object_mut(),
5999 ) {
6000 obj.insert(
6001 "client_connections_with_routes".into(),
6002 connections_with_routes.into(),
6003 );
6004 obj.insert("max_routes_on_one_connection".into(), max.into());
6005 }
6006 if let (Some(open_breakers), Some(obj)) = (
6012 self.route_bind_breakers.open_snapshot(),
6013 counters.as_object_mut(),
6014 ) {
6015 obj.insert("route_bind_breakers_open".into(), open_breakers);
6016 }
6017 if let (Some(reason), Some(obj)) = (
6022 self.local_endpoint_unavailable.clone(),
6023 counters.as_object_mut(),
6024 ) {
6025 obj.insert("local_endpoint_unavailable".into(), reason.into());
6026 }
6027 let response = ClientControlResponse::ServerDescribe {
6028 protocol_ver: PROTOCOL_VERSION,
6029 subc_ops: subc_ops(),
6030 capabilities: self.describe_capabilities(),
6031 connected_clients: self.connected_clients.count(),
6032 counters: Some(counters),
6033 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
6034 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
6035 capability_requirements: self.capability_requirement_statuses(),
6036 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
6037 config_path: self
6038 .rescan
6039 .as_ref()
6040 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
6041 };
6042 Ok(vec![control_response_body_frame(
6043 &frame,
6044 &response,
6045 "ClientControlResponse::ServerDescribe",
6046 )?])
6047 }
6048
6049 fn handle_catalog_list(
6050 &self,
6051 frame: Frame,
6052 module_id: Option<String>,
6053 ) -> Result<Vec<Frame>, RouterError> {
6054 let body = self.cached_catalog_body(frame.header.corr, module_id.as_deref())?;
6055 Ok(vec![Frame::build_with_version(
6056 response_version(&frame),
6057 FrameType::Response,
6058 control_flags(),
6059 0,
6060 0,
6061 frame.header.corr,
6062 body,
6063 )
6064 .map_err(RouterError::FrameBuild)?])
6065 }
6066
6067 #[cfg(test)]
6068 fn uncached_catalog_list(
6069 &self,
6070 corr: u64,
6071 module_id: Option<&str>,
6072 ) -> Result<ClientControlResponse, RouterError> {
6073 self.catalog_list_snapshot(corr, module_id, module_id.is_some())
6074 }
6075
6076 fn catalog_list_snapshot(
6077 &self,
6078 corr: u64,
6079 module_id: Option<&str>,
6080 include_declarations: bool,
6081 ) -> Result<ClientControlResponse, RouterError> {
6082 let (generation, modules) = self
6083 .registry
6084 .list_modules_with_nonce_verification()
6085 .map_err(|err| RouterError::backend(0, corr, format!("registry error: {err}")))?;
6086 let entries = modules
6087 .into_iter()
6088 .filter(|(registration, _)| {
6089 module_id
6090 .map(|wanted| registration.manifest.module_id == wanted)
6091 .unwrap_or(true)
6092 })
6093 .map(|(registration, nonce_verified)| {
6094 let entry = self
6095 .catalog_discovery_entry(®istration, include_declarations)
6096 .map_err(|err| RouterError::backend(0, corr, err))?;
6097 let not_ready = self.not_ready_reason(®istration);
6098 let mut entry = entry;
6099 entry.roles = registration.manifest.provides;
6100 entry.control_ops = registration.control_ops;
6101 Ok(entry
6102 .with_ready(not_ready.is_none())
6103 .with_display_name(registration.manifest.display_name)
6104 .with_not_ready(not_ready)
6105 .with_module_version(Some(registration.manifest.module_version))
6106 .with_provenance(registration.manifest.provenance)
6107 .with_capabilities(registration.manifest.capabilities)
6108 .with_self_signals(registration.manifest.self_signals)
6109 .with_events(registration.manifest.events)
6110 .with_operation_delivery(registration.manifest.operation_delivery)
6111 .with_operation_payload(registration.manifest.operation_payload)
6112 .with_flow_operations(registration.manifest.flow_operations)
6113 .with_operation_deferral(registration.manifest.operation_deferral)
6114 .with_confirm_operations(registration.manifest.confirm_operations)
6115 .with_rich_ui(registration.manifest.rich_ui)
6116 .with_role_declarations(registration.manifest.role_declarations)
6117 .with_nonce_verified(nonce_verified))
6118 })
6119 .collect::<Result<Vec<_>, RouterError>>()?;
6120 Ok(ClientControlResponse::CatalogList {
6121 generation,
6122 modules: entries,
6123 subc_ops: subc_ops(),
6124 })
6125 }
6126
6127 fn route_open_caller_from_frame(&self, ctx: &RouteCtx, frame: &Frame) -> RouteOpenCaller {
6131 let Ok(ClientControlRequest::RouteOpen {
6132 identity,
6133 consumer_identity,
6134 scope,
6135 ..
6136 }) = parse_client_control_request(&frame.body)
6137 else {
6138 return RouteOpenCaller::unknown();
6139 };
6140 let principal = self
6141 .attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref())
6142 .map(|attested| attested.principal);
6143 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
6144 }
6145
6146 fn route_open_principal(
6147 &self,
6148 connection_id: ConnectionId,
6149 frame: &Frame,
6150 consumer_identity: Option<ConsumerIdentity>,
6151 ) -> Result<Result<Principal, Frame>, RouterError> {
6152 let attested = self
6153 .attest_route_open_consumer(connection_id, consumer_identity.as_ref())
6154 .map(|attested| attested.principal);
6155 self.route_open_principal_attested(frame, consumer_identity, attested)
6156 }
6157
6158 fn route_open_principal_attested(
6162 &self,
6163 frame: &Frame,
6164 consumer_identity: Option<ConsumerIdentity>,
6165 attested: Option<Principal>,
6166 ) -> Result<Result<Principal, Frame>, RouterError> {
6167 if let Some(principal) = attested {
6168 return Ok(Ok(principal));
6169 }
6170 Ok(Err(control_error_frame(
6171 frame,
6172 "bad_consumer_identity",
6173 consumer_identity.map_or_else(
6174 || "connection claim is no longer authorized for routes".to_string(),
6175 |identity| format!(
6176 "consumer_identity for module_id '{}' conflicts with its connection claim or did not match a supervised launch nonce",
6177 identity.module_id
6178 ),
6179 ),
6180 )?))
6181 }
6182
6183 #[allow(clippy::too_many_arguments)]
6190 fn route_open_refusal_frame(
6191 &self,
6192 ctx: &RouteCtx,
6193 caller: &RouteOpenCaller,
6194 frame: &Frame,
6195 module_id: &str,
6196 reason: &'static str,
6197 code: &'static str,
6198 message: impl Into<String>,
6199 ) -> Result<Frame, RouterError> {
6200 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
6201 let detail = if matches!(
6202 reason,
6203 "registration_not_active"
6204 | "supervisor_not_live"
6205 | "no_forwarding_connection"
6206 | "reloading"
6207 ) {
6208 let expected_back = (reason == "reloading").then_some(true);
6211 self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
6212 } else {
6213 None
6214 };
6215 control_error_body_frame(
6216 frame,
6217 ErrorBody {
6218 code: code.to_string(),
6219 message: message.into(),
6220 detail,
6221 },
6222 )
6223 }
6224
6225 fn route_open_recovery_detail(
6226 &self,
6227 module_id: &str,
6228 corr: u64,
6229 detail: Option<serde_json::Value>,
6230 expected_back: Option<bool>,
6231 ) -> Result<Option<serde_json::Value>, RouterError> {
6232 let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
6233 return Ok(detail);
6234 };
6235 let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
6236 detail["state"] = serde_json::json!(status.state.to_string());
6237 if let Some(expected_back) = expected_back {
6238 detail["expected_back"] = serde_json::json!(expected_back);
6239 } else if let Some(module) = self.supervisor.get(module_id) {
6240 match module.will_recover_after_connection_loss() {
6241 Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
6242 Err(err) => warn!(
6243 %module_id,
6244 error = %err,
6245 "failed to read route.open recovery verdict; omitting expected_back"
6246 ),
6247 }
6248 }
6249 Ok(Some(detail))
6250 }
6251
6252 #[allow(clippy::too_many_arguments)]
6261 fn route_bind_rejection(
6262 &self,
6263 ctx: &RouteCtx,
6264 caller: &RouteOpenCaller,
6265 frame: &Frame,
6266 target_module_id: &str,
6267 body: ErrorBody,
6268 daemon_authored: bool,
6269 ) -> Result<Vec<Frame>, RouterError> {
6270 if daemon_authored {
6276 let scope_code = match body.code.as_str() {
6277 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
6278 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
6279 _ => None,
6280 };
6281 if let Some(code) = scope_code {
6282 self.observe_route_open_refusal(
6283 ctx,
6284 caller,
6285 target_module_id,
6286 "scope_changed_before_commit",
6287 code,
6288 );
6289 return Ok(vec![control_error_body_frame(frame, body)?]);
6290 }
6291 }
6292 self.counters
6293 .increment_route_open_refused("module_rejected");
6294 info!(
6295 target: "control",
6296 code = "module_rejected",
6297 module_code = ?body.code,
6298 module_id = ?target_module_id,
6299 connection_id = ctx.connection_id.get(),
6300 principal = %caller.principal,
6301 scope = %caller.scope,
6302 harness = %caller.harness,
6303 "route.open refused"
6304 );
6305 let refusal = if daemon_authored {
6306 control_error_body_frame(frame, body)?
6307 } else {
6308 control_relayed_error_body_frame(frame, body)?
6309 };
6310 Ok(vec![refusal])
6311 }
6312
6313 #[allow(clippy::too_many_arguments)]
6331 fn route_open_breaker_refusal_frame(
6332 &self,
6333 ctx: &RouteCtx,
6334 caller: &RouteOpenCaller,
6335 frame: &Frame,
6336 module_id: &str,
6337 consecutive_timeouts: u32,
6338 retry_in: Duration,
6339 probe_in_flight: bool,
6340 ) -> Result<Frame, RouterError> {
6341 self.counters
6342 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
6343 debug!(
6344 target: "control",
6345 code = "module_timeout",
6346 module_id = ?module_id,
6347 connection_id = ctx.connection_id.get(),
6348 principal = %caller.principal,
6349 scope = %caller.scope,
6350 harness = %caller.harness,
6351 consecutive_timeouts,
6352 retry_in_ms = retry_in.as_millis() as u64,
6353 probe_in_flight,
6354 "route.open refused by open bind-relay breaker"
6355 );
6356 let detail = if probe_in_flight {
6363 "one test route is already being tried; retry once it settles".to_string()
6364 } else {
6365 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
6366 };
6367 control_error_frame(
6368 frame,
6369 "module_timeout",
6370 format!(
6371 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
6372 timed out in a row); {detail}; its established routes are unaffected"
6373 ),
6374 )
6375 }
6376
6377 fn observe_route_open_refusal(
6390 &self,
6391 ctx: &RouteCtx,
6392 caller: &RouteOpenCaller,
6393 module_id: &str,
6394 reason: &'static str,
6395 code: &'static str,
6396 ) {
6397 self.counters.increment_route_open_refused(code);
6398 info!(
6399 target: "control",
6400 code,
6401 reason,
6402 module_id = ?module_id,
6403 connection_id = ctx.connection_id.get(),
6404 principal = %caller.principal,
6405 scope = %caller.scope,
6406 harness = %caller.harness,
6407 "route.open refused"
6408 );
6409 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
6410 self.route_outages.record_not_serving(module_id, reason);
6411 }
6412 }
6413
6414 fn observe_route_open_accept(
6459 &self,
6460 ctx: &RouteCtx,
6461 module_id: &str,
6462 principal: &str,
6463 scope: &str,
6464 ) {
6465 self.route_outages.record_accepted(module_id);
6466 self.counters.increment_route_open_accepted(principal);
6467 debug!(
6468 target: "control",
6469 principal,
6470 module_id,
6471 connection_id = ctx.connection_id.get(),
6472 scope = %scope,
6473 "route.open accepted"
6474 );
6475 }
6476
6477 fn supervised_absent_route_open_refusal_frame(
6478 &self,
6479 ctx: &RouteCtx,
6480 caller: &RouteOpenCaller,
6481 frame: &Frame,
6482 module_id: &str,
6483 code: &'static str,
6484 status: &crate::supervise::ModuleStatus,
6485 ) -> Result<Frame, RouterError> {
6486 self.counters.increment_route_open_refused(code);
6487 info!(
6488 target: "control",
6489 code,
6490 reason = "supervised_not_registered",
6491 module_id = ?module_id,
6492 connection_id = ctx.connection_id.get(),
6493 principal = %caller.principal,
6494 scope = %caller.scope,
6495 harness = %caller.harness,
6496 state = %status.state,
6497 enabled = status.enabled,
6498 live = status.live,
6499 "route.open refused"
6500 );
6501 self.route_outages
6505 .record_not_serving(module_id, "supervised_not_registered");
6506 let message = if let Some(pid) = status.waiting_for_previous_pid {
6507 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
6508 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
6509 })
6510 } else {
6511 format!(
6512 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
6513 status.state, status.enabled, status.live
6514 )
6515 };
6516 let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
6520 control_error_body_frame(
6521 frame,
6522 ErrorBody {
6523 code: code.to_string(),
6524 message,
6525 detail: self.route_open_recovery_detail(
6526 module_id,
6527 frame.header.corr,
6528 None,
6529 expected_back,
6530 )?,
6531 },
6532 )
6533 }
6534
6535 async fn handle_route_open(
6536 &self,
6537 ctx: &RouteCtx,
6538 frame: Frame,
6539 request: RouteOpenRequest,
6540 timings: &mut RouteOpenTimings,
6541 ) -> Result<Vec<Frame>, RouterError> {
6542 let RouteOpenRequest {
6543 target,
6544 mut identity,
6545 consumer_identity,
6546 consumer_capabilities,
6547 role_versions,
6548 busy_refusals,
6549 admission_facts,
6550 scope,
6551 accepts_close_reasons,
6552 } = request;
6553 let target_module_id = target_module_id(&target).to_string();
6554 timings.module_id.clone_from(&target_module_id);
6555 if self
6556 .registry
6557 .get_module_by_connection(ctx.connection_id)
6558 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6559 .is_some()
6560 {
6561 return Ok(vec![control_error_frame(
6562 &frame,
6563 "invalid_request",
6564 "module connections cannot open client routes",
6565 )?]);
6566 }
6567 debug!(
6568 connection_id = ctx.connection_id.get(),
6569 corr = frame.header.corr,
6570 module_id = %target_module_id,
6571 "handling route.open"
6572 );
6573
6574 let attested =
6580 self.attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref());
6581 let mut caller = RouteOpenCaller::new(
6582 attested.as_ref().map(|attested| &attested.principal),
6583 scope.as_ref(),
6584 &identity,
6585 );
6586
6587 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
6593 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
6594 self.observe_route_open_refusal(
6595 ctx,
6596 &caller,
6597 &target_module_id,
6598 "invalid_role_versions",
6599 error_codes::INVALID_REQUEST,
6600 );
6601 return Ok(vec![control_error_body_frame(
6602 &frame,
6603 ErrorBody {
6604 code: error_codes::INVALID_REQUEST.to_string(),
6605 message: error.to_string(),
6606 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
6607 },
6608 )?]);
6609 }
6610
6611 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
6612 if status.waiting_for_previous_pid.is_some() {
6613 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6614 ctx,
6615 &caller,
6616 &frame,
6617 &target_module_id,
6618 "module_warming",
6619 &status,
6620 )?]);
6621 }
6622 }
6623
6624 let admission_registry = self
6643 .registry
6644 .route_admission_snapshot()
6645 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6646 let Some(registration) =
6647 crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
6648 else {
6649 if let Some((status, warming)) =
6650 self.supervisor_status(&target_module_id, frame.header.corr)?
6651 {
6652 if status.protocol == ModuleProtocol::None {
6659 return Ok(vec![self.route_open_refusal_frame(
6660 ctx,
6661 &caller,
6662 &frame,
6663 &target_module_id,
6664 "protocol_none",
6665 error_codes::MODULE_NO_PROTOCOL,
6666 format!(
6667 "module_id '{target_module_id}' is declared protocol: none; \
6668 it speaks no subc wire and serves no routes"
6669 ),
6670 )?]);
6671 }
6672 let code = if warming {
6673 "module_warming"
6674 } else {
6675 "target_unavailable"
6676 };
6677 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6678 ctx,
6679 &caller,
6680 &frame,
6681 &target_module_id,
6682 code,
6683 &status,
6684 )?]);
6685 }
6686 if let Some(removed_ago_ms) =
6687 self.supervisor.removal_tombstone_age_ms(&target_module_id)
6688 {
6689 return Ok(vec![self.route_open_refusal_frame(
6690 ctx,
6691 &caller,
6692 &frame,
6693 &target_module_id,
6694 "removed",
6695 error_codes::MODULE_REMOVED,
6696 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
6697 )?]);
6698 }
6699 return Ok(vec![self.route_open_refusal_frame(
6700 ctx,
6701 &caller,
6702 &frame,
6703 &target_module_id,
6704 "not_registered",
6705 error_codes::UNKNOWN_MODULE,
6706 format!("module_id '{target_module_id}' is not registered"),
6707 )?]);
6708 };
6709
6710 if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
6714 .is_err()
6715 {
6716 self.counters
6717 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
6718 info!(
6719 target: "control",
6720 code = error_codes::MODULE_WARMING,
6721 module_id = ?target_module_id,
6722 connection_id = ctx.connection_id.get(),
6723 principal = %caller.principal,
6724 scope = %caller.scope,
6725 harness = %caller.harness,
6726 reason = "declared_not_ready",
6727 "route.open refused"
6728 );
6729 self.route_outages
6732 .record_not_serving(&target_module_id, "declared_not_ready");
6733 return Ok(vec![control_error_body_frame(
6736 &frame,
6737 ErrorBody {
6738 code: error_codes::MODULE_WARMING.to_string(),
6739 message: format!(
6740 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
6741 ),
6742 detail: self.route_open_recovery_detail(
6743 &target_module_id,
6744 frame.header.corr,
6745 Some(serde_json::json!({ "reason": "declared_not_ready" })),
6746 Some(true),
6747 )?,
6748 },
6749 )?]);
6750 }
6751
6752 if let Some(capability) = self.first_unprovided_required_capability_for(
6771 ®istration.module_id,
6772 registration.capabilities.as_ref(),
6773 ) {
6774 self.counters
6775 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
6776 info!(
6777 target: "control",
6778 code = error_codes::MODULE_WARMING,
6779 module_id = ?target_module_id,
6780 connection_id = ctx.connection_id.get(),
6781 principal = %caller.principal,
6782 scope = %caller.scope,
6783 harness = %caller.harness,
6784 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6785 capability = %capability,
6786 "route.open refused"
6787 );
6788 return Ok(vec![control_error_body_frame(
6789 &frame,
6790 ErrorBody {
6791 code: error_codes::MODULE_WARMING.to_string(),
6792 message: format!(
6793 "module_id '{target_module_id}' requires capability '{capability}', \
6794 which no registered module provides; retry"
6795 ),
6796 detail: Some(serde_json::json!({
6797 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6798 "capability": capability,
6799 })),
6800 },
6801 )?]);
6802 }
6803
6804 if !target_has_required_role(&target, ®istration.roles) {
6805 return Ok(vec![self.route_open_refusal_frame(
6806 ctx,
6807 &caller,
6808 &frame,
6809 &target_module_id,
6810 "role_not_provided",
6811 "target_unavailable",
6812 format!("module_id '{target_module_id}' does not provide the requested target"),
6813 )?]);
6814 }
6815
6816 if registration.state != ChannelState::Active {
6817 return Ok(vec![self.route_open_refusal_frame(
6818 ctx,
6819 &caller,
6820 &frame,
6821 &target_module_id,
6822 "registration_not_active",
6823 "target_unavailable",
6824 format!("module_id '{target_module_id}' is not active"),
6825 )?]);
6826 }
6827
6828 if self
6829 .forwarding
6830 .module_is_draining(&target_module_id)
6831 .map_err(RouterError::Forwarding)?
6832 {
6833 return Ok(vec![self.route_open_refusal_frame(
6834 ctx,
6835 &caller,
6836 &frame,
6837 &target_module_id,
6838 "reloading",
6839 "module_reloading",
6840 format!("module_id '{target_module_id}' is reloading"),
6841 )?]);
6842 }
6843
6844 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
6845 process_liveness.process_live(&target_module_id) == Some(false)
6846 }) {
6847 if process_liveness.process_replacing(&target_module_id) {
6854 return Ok(vec![self.route_open_refusal_frame(
6855 ctx,
6856 &caller,
6857 &frame,
6858 &target_module_id,
6859 "reloading",
6860 "module_reloading",
6861 format!("module_id '{target_module_id}' is reloading"),
6862 )?]);
6863 }
6864 return Ok(vec![self.route_open_refusal_frame(
6865 ctx,
6866 &caller,
6867 &frame,
6868 &target_module_id,
6869 "supervisor_not_live",
6870 "target_unavailable",
6871 format!("module_id '{target_module_id}' is not live"),
6872 )?]);
6873 }
6874
6875 if !self
6876 .forwarding
6877 .has_live_module_connection(&target_module_id)
6878 .map_err(RouterError::Forwarding)?
6879 {
6880 return Ok(vec![self.route_open_refusal_frame(
6881 ctx,
6882 &caller,
6883 &frame,
6884 &target_module_id,
6885 "no_forwarding_connection",
6886 "target_unavailable",
6887 format!("module_id '{target_module_id}' has no live forwarding connection"),
6888 )?]);
6889 }
6890
6891 if let Some(error) =
6892 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
6893 {
6894 self.observe_route_open_refusal(
6895 ctx,
6896 &caller,
6897 &target_module_id,
6898 "op_not_allowed",
6899 "op_not_allowed",
6900 );
6901 return Ok(vec![error]);
6902 }
6903
6904 let principal = match (&attested, consumer_identity.is_some()) {
6905 (None, false) => {
6908 self.observe_route_open_refusal(
6909 ctx,
6910 &caller,
6911 &target_module_id,
6912 "registration_channel_closed",
6913 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6914 );
6915 return Ok(vec![control_error_frame(
6916 &frame,
6917 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6918 "the claim binding's launch has ended",
6919 )?]);
6920 }
6921 _ => match self.route_open_principal_attested(
6922 &frame,
6923 consumer_identity,
6924 attested.as_ref().map(|attested| attested.principal.clone()),
6925 )? {
6926 Ok(principal) => principal,
6927 Err(error) => {
6928 self.observe_route_open_refusal(
6929 ctx,
6930 &caller,
6931 &target_module_id,
6932 "bad_consumer_identity",
6933 "bad_consumer_identity",
6934 );
6935 return Ok(vec![error]);
6936 }
6937 },
6938 };
6939 let attested = attested.expect("an attested principal was resolved above");
6940 timings.principal = principal_label(&principal);
6941
6942 let Ok(mut caller_capture) = attested.capture() else {
6945 self.observe_route_open_refusal(
6946 ctx,
6947 &caller,
6948 &target_module_id,
6949 "registration_channel_closed",
6950 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6951 );
6952 return Ok(vec![control_error_frame(
6953 &frame,
6954 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6955 "the opener's launch has ended",
6956 )?]);
6957 };
6958 let caller_module_id = match &attested.spawn {
6962 Some((record, _)) => Some(record.module_id().to_owned()),
6963 None => match &principal {
6964 Principal::Reserved { module_id } => Some(module_id.clone()),
6965 _ => None,
6966 },
6967 };
6968
6969 let build_class = identity.build_class.take().filter(|build_class| {
6972 if caller_module_id.as_deref() == Some("callosum") {
6973 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
6974 } else {
6975 if self
6976 .build_class_warned
6977 .lock()
6978 .unwrap_or_else(|p| p.into_inner())
6979 .insert(ctx.connection_id)
6980 {
6981 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
6982 "ignoring build_class from a non-Callosum opener");
6983 }
6984 false
6985 }
6986 });
6987
6988 if let Some(opening_registration) = self.caller_registration(&attested, &admission_registry)
6992 {
6993 let opening_module_id = opening_registration.module_id.as_str();
6994 if let Some(capability) = denied_capability(
6995 opening_registration.capabilities.as_ref(),
6996 registration.capabilities.as_ref(),
6997 ) {
6998 warn!(
6999 opening_module_id,
7000 target_module_id,
7001 capability,
7002 "refusing route.open because an attested capability deny edge matches"
7003 );
7004 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
7005 &frame,
7006 &target_module_id,
7007 "capability_deny_edge",
7008 "capability_forbidden",
7009 format!(
7010 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
7011 ),
7012 )?]);
7013 }
7014 }
7015
7016 if admission_facts.is_some() {
7017 let carrier_matches = caller_module_id.is_some()
7018 && self.admission_facts_carrier_module_id.as_deref() == caller_module_id.as_deref();
7019 if !carrier_matches {
7020 return Ok(vec![self.route_open_refusal_frame(
7021 ctx,
7022 &caller,
7023 &frame,
7024 &target_module_id,
7025 "admission_facts_carrier_not_permitted",
7026 "admission_facts_not_permitted",
7027 "admission facts may only be carried by the configured reserved module",
7028 )?]);
7029 }
7030
7031 let target_allowed = self
7032 .admission_facts_targets
7033 .as_ref()
7034 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
7035 if !target_allowed {
7036 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
7037 &frame,
7038 &target_module_id,
7039 "admission_facts_target_not_listed",
7040 "admission_facts_target_not_allowed",
7041 format!(
7042 "admission facts are not permitted for target module_id '{target_module_id}'"
7043 ),
7044 )?]);
7045 }
7046
7047 }
7051
7052 let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
7058 None => (None, None, false),
7059 Some(selector) => {
7060 let owner_configured = match &selector.owner {
7061 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
7067 _ => false,
7068 };
7069 let mut scopes = self
7070 .scopes
7071 .write()
7072 .unwrap_or_else(|poisoned| poisoned.into_inner());
7073 let admitted = crate::scopes::admission::admit_route(
7074 &scopes,
7075 &admission_registry,
7076 &principal,
7077 &target_module_id,
7078 Some(selector),
7079 owner_configured,
7080 crate::scopes::admission::AdmissionMode::Open,
7081 )
7082 .map(|route| {
7083 let admission = route.scope.expect("scoped admission");
7084 let bound = scopes.reserve_bind(
7085 &admission,
7086 &principal,
7087 ctx.connection_id,
7088 selector.hold_presence,
7089 );
7090 (admission, bound)
7091 });
7092 match admitted {
7093 Ok((admission, bound)) => {
7094 (Some(bound), Some(admission.stamp), admission.stamp_required)
7095 }
7096 Err(refusal) => {
7097 return Ok(vec![self.route_open_refusal_frame(
7098 ctx,
7099 &caller,
7100 &frame,
7101 &target_module_id,
7102 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7103 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7104 refusal.message,
7105 )?]);
7106 }
7107 }
7108 }
7109 };
7110 if let Some(stamp) = &scope_stamp {
7113 caller.scope = route_open_scope_label(stamp);
7114 }
7115 if scope_stamp.as_ref().is_some_and(scope_acts_for_another) {
7118 caller_capture = caller_capture.map(crate::spawn_authority::CallerCapture::relayed);
7119 }
7120
7121 timings.enter(RouteOpenPhase::ProjectRoot);
7129 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
7130 Ok(project) => project,
7131 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
7132 };
7133 timings.enter(RouteOpenPhase::Admission);
7134 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
7135 Ok(root) => root,
7136 Err(err) => {
7137 return Ok(vec![control_error_frame(
7138 &frame,
7139 "invalid_project_root",
7140 err.to_string(),
7141 )?])
7142 }
7143 };
7144 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
7150
7151 let _concurrency_guard = match self
7162 .route_bind_concurrency
7163 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
7164 {
7165 Ok(guard) => guard,
7166 Err(in_flight) => {
7167 return Ok(vec![self.route_open_target_capacity_refusal(
7168 ctx,
7169 &caller,
7170 &frame,
7171 &target_module_id,
7172 in_flight,
7173 )?]);
7174 }
7175 };
7176
7177 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
7180 RouteBindAdmission::Admitted { guard, probe } => {
7181 if probe {
7182 info!(
7183 module_id = %target_module_id,
7184 connection_id = ctx.connection_id.get(),
7185 "route.bind breaker half-open: admitting one probe"
7186 );
7187 }
7188 guard
7189 }
7190 RouteBindAdmission::Refused {
7191 consecutive_timeouts,
7192 retry_in,
7193 probe_in_flight,
7194 } => {
7195 return Ok(vec![self.route_open_breaker_refusal_frame(
7196 ctx,
7197 &caller,
7198 &frame,
7199 &target_module_id,
7200 consecutive_timeouts,
7201 retry_in,
7202 probe_in_flight,
7203 )?]);
7204 }
7205 };
7206
7207 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
7212 let relay_deadline = Instant::now() + route_bind_relay_timeout;
7213 let operator_app = self
7214 .operator_apps
7215 .lock()
7216 .unwrap_or_else(|p| p.into_inner())
7217 .get(&ctx.connection_id)
7218 .cloned();
7219 let production_peer_build =
7220 build_class.as_deref() == Some(subc_protocol::BUILD_CLASS_PRODUCTION);
7221 timings.enter(RouteOpenPhase::EgressReservation);
7222 let pending = match self
7223 .forwarding
7224 .begin_route_bind_relay_for(
7225 ctx.connection_id,
7226 ctx.egress.clone(),
7227 response_version(&frame),
7228 frame.header.corr,
7229 &target_module_id,
7230 principal.clone(),
7231 bound_scope,
7232 project_root,
7233 identity.harness.clone(),
7234 build_class,
7235 operator_app.clone(),
7236 crate::forwarding::RouteAuthority {
7237 incarnation: self.supervisor.daemon_incarnation(),
7238 caller: caller_capture
7239 .as_ref()
7240 .map(|caller| caller.authority.pair()),
7241 },
7242 relay_deadline,
7243 busy_refusals == Some(true),
7244 accepts_close_reasons,
7245 )
7246 .await
7247 {
7248 Ok(pending) => pending,
7249 Err(err) => {
7250 return Ok(vec![self.route_open_refusal_frame(
7251 ctx,
7252 &caller,
7253 &frame,
7254 &target_module_id,
7255 "relay_reservation_failed",
7256 forwarding_error_code(&err),
7257 err.to_string(),
7258 )?])
7259 }
7260 };
7261 timings.enter(RouteOpenPhase::Admission);
7262 let crate::forwarding::PendingRouteBindRelay {
7263 endpoint,
7264 module_sink,
7265 negotiated_ver,
7266 client_channel,
7267 client_epoch,
7268 module_channel,
7269 module_epoch,
7270 corr: relay_corr,
7271 receiver,
7272 } = pending;
7273 let mut reservation =
7274 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
7275
7276 let relay_registration = self
7281 .registry
7282 .route_registration_by_connection(endpoint.connection_id)
7283 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7284 let relay_capabilities = relay_registration
7285 .as_ref()
7286 .and_then(|registration| registration.capabilities.as_ref());
7287 if let Some(selector) = &scope {
7288 if let Err(refusal) = self
7291 .scopes
7292 .read()
7293 .unwrap_or_else(|poisoned| poisoned.into_inner())
7294 .check_bind_proof(&principal, selector)
7295 {
7296 reservation.release_and_disarm();
7297 return Ok(vec![self.route_open_refusal_frame(
7298 ctx,
7299 &caller,
7300 &frame,
7301 &target_module_id,
7302 refusal.code,
7303 refusal.code,
7304 refusal.message,
7305 )?]);
7306 }
7307 }
7308 if let Some(stamp) = scope_stamp.as_ref() {
7309 if let Err(refusal) = crate::scopes::check_target_required_stamp(
7310 stamp_required,
7311 &target_module_id,
7312 relay_capabilities,
7313 )
7314 .and_then(|()| {
7315 crate::scopes::check_target_flow_support(
7316 stamp,
7317 &target_module_id,
7318 relay_capabilities,
7319 )
7320 })
7321 .and_then(|()| {
7322 crate::scopes::check_target_agent_run_support(
7323 stamp,
7324 &target_module_id,
7325 relay_capabilities,
7326 )
7327 }) {
7328 reservation.release_and_disarm();
7329 return Ok(vec![self.route_open_refusal_frame(
7330 ctx,
7331 &caller,
7332 &frame,
7333 &target_module_id,
7334 refusal.code,
7335 refusal.code,
7336 refusal.message,
7337 )?]);
7338 }
7339 }
7340
7341 if scope_stamp
7344 .as_ref()
7345 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
7346 && !crate::scopes::target_supports_federated(relay_capabilities)
7347 {
7348 scope_stamp = None;
7350 }
7351 if !relay_capabilities.is_some_and(|caps| {
7352 caps.provides
7353 .iter()
7354 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
7355 }) {
7356 if let Some(stamp) = scope_stamp.as_mut() {
7357 stamp.workdir = None;
7358 stamp.project_id = None;
7359 }
7360 }
7361 let worker_run = scope_stamp
7366 .as_ref()
7367 .is_some_and(crate::scopes::is_worker_run)
7368 && relay_capabilities.is_some_and(|caps| {
7369 caps.provides
7370 .iter()
7371 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
7372 });
7373 if !worker_run
7374 && !relay_capabilities.is_some_and(|caps| {
7375 caps.provides
7376 .iter()
7377 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
7378 })
7379 {
7380 if let Some(stamp) = scope_stamp.as_mut() {
7381 stamp.attributes.on_behalf_of_agent = None;
7382 }
7383 }
7384 if !relay_capabilities.is_some_and(|caps| {
7385 caps.provides
7386 .iter()
7387 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
7388 }) {
7389 if let Some(stamp) = scope_stamp.as_mut() {
7390 stamp.attributes.flow_version = None;
7391 stamp.attributes.flow_code_hash = None;
7392 stamp.attributes.flow_install_id = None;
7393 }
7394 }
7395 let call_mode = if worker_run
7396 || relay_capabilities
7397 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
7398 {
7399 Some(derive_call_mode(scope_stamp.as_ref()))
7400 } else {
7401 if let Some(stamp) = scope_stamp.as_mut() {
7402 stamp.expires_at_ms = None;
7403 }
7404 None
7405 };
7406
7407 debug!(
7408 connection_id = ctx.connection_id.get(),
7409 client_channel,
7410 client_epoch,
7411 module_channel,
7412 module_epoch,
7413 "reserved route handle pair"
7414 );
7415 let route_principal_label = principal_label(&principal);
7418 let relay = ModuleControlRequest::RouteBind {
7419 route_channel: module_channel,
7420 epoch: module_epoch,
7421 target,
7422 identity,
7423 principal: Some(principal),
7424 consumer_capabilities,
7425 role_versions,
7426 admission_facts,
7427 scope: scope_stamp,
7428 call_mode,
7429 };
7430 let peer = RouteBindPeer::new()
7431 .with_observed_peer(
7432 relay_capabilities
7433 .is_some_and(|caps| {
7434 caps.provides
7435 .iter()
7436 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
7437 })
7438 .then(|| {
7439 self.observed_peers
7440 .lock()
7441 .unwrap_or_else(|p| p.into_inner())
7442 .get(&ctx.connection_id)
7443 .cloned()
7444 .unwrap_or_else(ObservedPeer::unobserved)
7445 }),
7446 )
7447 .with_operator_app(
7448 self.connection_operator_app(operator_app.as_ref())
7449 .filter(|_| {
7450 relay_capabilities.is_some_and(|caps| {
7451 caps.provides
7452 .iter()
7453 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
7454 })
7455 }),
7456 )
7457 .with_caller(caller_for_target(
7458 caller_capture.as_ref(),
7459 relay_capabilities,
7460 &target_module_id,
7461 ))
7462 .with_peer_build(
7463 (production_peer_build
7464 && relay_capabilities.is_some_and(|caps| {
7465 caps.provides.iter().any(|cap| cap == PEER_BUILD_CAPABILITY)
7466 }))
7467 .then(|| subc_protocol::BUILD_CLASS_PRODUCTION.to_owned()),
7468 );
7469 let relay_body = crate::bind_compat::encode_bind(
7470 &relay,
7471 &peer,
7472 &legacy_bind_project,
7473 relay_capabilities,
7474 )
7475 .map_err(|err| {
7476 RouterError::backend(
7477 0,
7478 frame.header.corr,
7479 format!("failed to encode route.bind request: {err}"),
7480 )
7481 })?;
7482 let relay_frame = Frame::build_with_version(
7483 negotiated_ver,
7484 FrameType::Request,
7485 control_flags(),
7486 0,
7487 0,
7488 relay_corr,
7489 relay_body,
7490 )
7491 .map_err(RouterError::FrameBuild)?;
7492
7493 timings.enter(RouteOpenPhase::BindSend);
7494 let sent = module_sink.send(relay_frame).await;
7495 timings.enter(RouteOpenPhase::Admission);
7496 if let Err(err) = sent {
7497 reservation.release_and_disarm();
7498 return Ok(vec![self.route_open_refusal_frame(
7499 ctx,
7500 &caller,
7501 &frame,
7502 &target_module_id,
7503 "relay_send_failed",
7504 "target_unavailable",
7505 err.to_string(),
7506 )?]);
7507 }
7508
7509 #[cfg(test)]
7510 if let Some(hold) = &self.route_bind_relay_sent_hold {
7511 hold.wait().await;
7515 hold.wait().await;
7516 }
7517
7518 if self
7523 .forwarding
7524 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
7525 .map_err(RouterError::Forwarding)?
7526 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
7527 {
7528 self.send_abandoned_route_bind_goodbye(
7529 &module_sink,
7530 negotiated_ver,
7531 module_channel,
7532 module_epoch,
7533 );
7534 }
7535
7536 timings.enter(RouteOpenPhase::BindAckWait);
7537 let outcome = timeout_at(relay_deadline, receiver).await;
7538 timings.enter(RouteOpenPhase::Admission);
7539 match outcome {
7540 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
7541 reservation.disarm();
7542 if breaker.record_accepted() {
7543 info!(
7544 module_id = %target_module_id,
7545 "route.bind breaker closed: the probe was accepted"
7546 );
7547 }
7548 self.observe_route_open_accept(
7549 ctx,
7550 &target_module_id,
7551 &route_principal_label,
7552 &caller.scope,
7553 );
7554 Ok(Vec::new())
7555 }
7556 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
7557 reservation.release_and_disarm();
7558 breaker.record_inconclusive();
7562 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
7563 }
7564 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
7565 reservation.release_and_disarm();
7566 breaker.record_inconclusive();
7567 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
7568 }
7569 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
7570 reservation.release_and_disarm();
7571 breaker.record_inconclusive();
7572 tracing::warn!(
7579 module_id = %target_module_id,
7580 "route.bind relay abandoned: {message}"
7581 );
7582 Ok(vec![self.route_open_refusal_frame(
7583 ctx,
7584 &caller,
7585 &frame,
7586 &target_module_id,
7587 "relay_abandoned",
7588 "target_unavailable",
7589 message,
7590 )?])
7591 }
7592 Ok(Err(_)) => {
7593 reservation.release_and_disarm();
7594 breaker.record_inconclusive();
7595 Ok(vec![self.route_open_refusal_frame(
7596 ctx,
7597 &caller,
7598 &frame,
7599 &target_module_id,
7600 "relay_waiter_canceled",
7601 "target_unavailable",
7602 "route.bind relay waiter was canceled before the module responded",
7603 )?])
7604 }
7605 Err(_) => {
7606 reservation.release_and_disarm();
7607 if let Some(opened) = breaker.record_timeout(
7611 self.route_bind_breaker_threshold,
7612 self.route_bind_breaker_cooldown,
7613 ) {
7614 warn!(
7615 module_id = %target_module_id,
7616 consecutive_timeouts = opened.consecutive_timeouts,
7617 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
7618 reopened_after_probe = opened.reopened_after_probe,
7619 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
7620 );
7621 }
7622 tracing::warn!(
7630 module_id = %target_module_id,
7631 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
7632 "route.bind relay timed out: module did not ack within budget"
7633 );
7634 Ok(vec![self.route_open_refusal_frame(
7635 ctx,
7636 &caller,
7637 &frame,
7638 &target_module_id,
7639 "relay_timed_out",
7640 "module_timeout",
7641 format!(
7642 "module_id '{target_module_id}' did not answer route.bind within {:?}",
7643 route_bind_relay_timeout
7644 ),
7645 )?])
7646 }
7647 }
7648 }
7649
7650 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7651 let response = ClientControlResponse::SupervisorSpawnSnapshot {
7652 snapshot: self.supervisor.spawn_snapshot(),
7653 };
7654 Ok(vec![control_response_body_frame(
7655 &frame,
7656 &response,
7657 "ClientControlResponse::SupervisorSpawnSnapshot",
7658 )?])
7659 }
7660
7661 fn handle_supervisor_spawn_subscribe(
7662 &self,
7663 ctx: &RouteCtx,
7664 frame: Frame,
7665 since: Option<SpawnCursor>,
7666 ) -> Result<Vec<Frame>, RouterError> {
7667 match self.supervisor.subscribe_spawns(
7668 ctx.connection_id,
7669 frame.header.corr,
7670 response_version(&frame),
7671 since,
7672 ctx.egress.clone(),
7673 ) {
7674 Ok(()) => Ok(Vec::new()),
7675 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
7676 Ok(vec![control_error_body_frame(
7677 &frame,
7678 ErrorBody {
7679 code: "spawn_cursor_incarnation_mismatch".to_string(),
7680 message: "spawn cursor belongs to a different daemon incarnation"
7681 .to_string(),
7682 detail: Some(serde_json::json!({
7683 "current_daemon_incarnation": current
7684 })),
7685 },
7686 )?])
7687 }
7688 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
7689 &frame,
7690 ErrorBody {
7691 code: "spawn_cursor_too_old".to_string(),
7692 message: "spawn cursor predates the retained event ring".to_string(),
7693 detail: Some(serde_json::json!({
7694 "oldest_retained_cursor": oldest
7695 })),
7696 },
7697 )?]),
7698 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
7699 0,
7700 frame.header.corr,
7701 format!("failed to open supervisor spawn subscription: {error}"),
7702 )),
7703 }
7704 }
7705
7706 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7707 let handler = self.clone();
7710 let runtime = tokio::runtime::Handle::current();
7711 let subscriber = tracing::dispatcher::get_default(Clone::clone);
7714 let corr = frame.header.corr;
7715 tokio::task::spawn_blocking(move || {
7716 runtime.block_on(
7717 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
7718 .with_subscriber(subscriber),
7719 )
7720 })
7721 .await
7722 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
7723 }
7724
7725 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7726 use crate::config_gate::runtime::ApprovedDaemonView;
7727 let publication = self
7728 .config_runtime
7729 .as_ref()
7730 .and_then(|owner| owner.polled_snapshot().ok());
7731 let approved = publication
7732 .as_ref()
7733 .map(|view| view.approved_daemon.clone())
7734 .unwrap_or_else(|| {
7735 if self.config_runtime.is_some() {
7736 ApprovedDaemonView::Unavailable
7737 } else {
7738 ApprovedDaemonView::NotStamped
7739 }
7740 });
7741 let generation = self
7742 .registry
7743 .generation()
7744 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7745 let supervised = self.supervisor.list();
7746 let mut statuses = BTreeMap::new();
7747 let mut not_running = BTreeSet::new();
7748 for module in &supervised {
7749 let status = module.status_for_control("list").map_err(|err| {
7750 RouterError::backend(
7751 0,
7752 frame.header.corr,
7753 format!("failed to read supervisor status: {err}"),
7754 )
7755 })?;
7756 if !status.enabled || !status.process_alive {
7757 not_running.insert(module.module_id().to_owned());
7758 }
7759 statuses.insert(module.module_id().to_owned(), status);
7760 }
7761 let mut process_resources = BTreeMap::new();
7764 for module in &supervised {
7765 process_resources.insert(
7766 module.module_id().to_owned(),
7767 crate::subtree_resources::read_process(module.clone()).await,
7768 );
7769 }
7770 let mut image_readings = BTreeMap::new();
7773 for module in &supervised {
7774 image_readings.insert(
7775 module.module_id().to_owned(),
7776 module.running_image_agreement().await,
7777 );
7778 }
7779 let readings = self.resource_tracker.list(supervised.clone());
7780 let mut modules = Vec::new();
7781 for module in supervised {
7782 let status = statuses
7783 .remove(module.module_id())
7784 .expect("each supervised module has a status snapshot");
7785 let (configured, _) = module.configuration().map_err(|err| {
7786 RouterError::backend(
7787 0,
7788 frame.header.corr,
7789 format!("failed to read module configuration: {err}"),
7790 )
7791 })?;
7792 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
7793 RouterError::backend(
7794 0,
7795 frame.header.corr,
7796 format!("failed to read retry time: {err}"),
7797 )
7798 })?;
7799 let image = image_readings
7800 .remove(module.module_id())
7801 .expect("each supervised module has an image snapshot");
7802 let resources = process_resources.remove(module.module_id());
7803 let mut verdict =
7804 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
7805 verdict.spec = Some(launch_spec_agreement(
7806 &configured,
7807 status.process_alive,
7808 status.launch_spec_digest.as_ref(),
7809 ));
7810 let pending_reload = Some(verdict);
7811 let pending_entry = match &approved {
7812 ApprovedDaemonView::Approved { config, .. } => config
7813 .modules
7814 .iter()
7815 .find(|m| m.module_id == module.module_id()),
7816 _ => None,
7817 };
7818 let pending = self
7819 .pending_module_fields(
7820 &module,
7821 pending_entry,
7822 false,
7823 Self::registration_channel_in_force(
7824 publication
7825 .as_ref()
7826 .and_then(|view| view.registration_declaration(module.module_id()))
7827 .as_deref(),
7828 pending_entry.map(|entry| entry.registration_channel),
7829 ),
7830 )
7831 .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
7832 let script = configured.script.as_ref().map(|entry| {
7833 let approved = self
7834 .config_runtime
7835 .as_ref()
7836 .and_then(|owner| owner.approved_script(&status.module_id, &entry.path));
7837 crate::supervise::script::status(entry, approved.as_ref())
7838 });
7839 let configured_enabled = match &approved {
7840 ApprovedDaemonView::Approved {
7841 config: approved, ..
7842 } => approved
7843 .modules
7844 .iter()
7845 .find(|module| module.module_id == status.module_id)
7846 .map(|module| module.enabled),
7847 ApprovedDaemonView::NotStamped => self
7848 .capability_evaluator
7849 .configured_enabled(&status.module_id),
7850 ApprovedDaemonView::Unavailable => None,
7851 };
7852 let admission_refused = status
7853 .admission_refused
7854 .clone()
7855 .filter(|_| status.enabled && !status.process_alive)
7856 .filter(|_| {
7857 publication
7858 .as_ref()
7859 .and_then(|view| view.registration_declaration(&status.module_id))
7860 .as_deref()
7861 == Some("undeclared")
7862 });
7863 let mut entry = SupervisorEntry::new(
7866 status.module_id,
7867 status.state.to_string(),
7868 status.enabled,
7869 status.live,
7870 status.health.status,
7871 )
7872 .with_configured_enabled(configured_enabled)
7873 .with_script(script)
7874 .with_admission_refused(admission_refused)
7875 .with_next_retry_at_ms(next_retry_at_ms)
7876 .with_launch_nonce_env(Some(!cfg!(unix)))
7877 .with_launch_nonce_source(status.launch_nonce_source)
7878 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
7879 .with_protocol(status.protocol)
7880 .with_pending_reload(pending_reload)
7881 .with_last_probe_ms(status.health.last_probe_ms)
7882 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
7883 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
7884 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
7885 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
7886 .with_restart_count(Some(status.restart_count))
7887 .with_max_restarts(Some(status.max_restarts))
7888 .with_lifetime_restarts(Some(status.lifetime_restarts))
7889 .with_spawn_generation(Some(status.spawn_generation))
7890 .with_restart_window_secs(Some(status.restart_window.as_secs()))
7891 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
7892 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
7893 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
7894 .with_resources(resources);
7895 entry.pending = pending;
7896 modules.push(entry);
7897 }
7898 for module in &mut modules {
7899 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
7900 module.tree = tree.clone();
7901 module.cpu_last_hour = hour.clone();
7902 }
7903 module.tree_unavailable = readings
7904 .reasons
7905 .get(&module.module_id)
7906 .map(|reason| (*reason).to_owned());
7907 if module.tree.is_none() {
7908 module.tree_unavailable = Some(
7909 if not_running.contains(&module.module_id) {
7910 "not_running"
7911 } else if let Some(reason) = readings.reasons.get(&module.module_id) {
7912 reason
7913 } else {
7914 "not_yet_measured"
7915 }
7916 .to_owned(),
7917 );
7918 }
7919 }
7920 let response = ClientControlResponse::SupervisorList {
7921 generation,
7922 modules,
7923 daemon: readings.daemon,
7924 agent_commands: readings.agent_commands,
7925 };
7926 Ok(vec![control_response_body_frame(
7927 &frame,
7928 &response,
7929 "ClientControlResponse::SupervisorList",
7930 )?])
7931 }
7932
7933 fn handle_supervisor_stderr_tail(
7934 &self,
7935 frame: Frame,
7936 module_id: String,
7937 max_lines: Option<u32>,
7938 max_bytes: Option<u32>,
7939 ) -> Result<Vec<Frame>, RouterError> {
7940 let Some(module) = self.supervisor.get(&module_id) else {
7941 return Ok(vec![control_error_frame(
7942 &frame,
7943 "unknown_module",
7944 format!("module_id '{module_id}' is not supervised"),
7945 )?]);
7946 };
7947
7948 let snapshot = module.stderr_tail(
7949 max_lines.map(|value| value as usize),
7950 max_bytes.map(|value| value as usize),
7951 );
7952
7953 let response = ClientControlResponse::SupervisorStderrTail {
7954 module_id,
7955 tail: StderrTail {
7956 capture: match snapshot.capture {
7957 CaptureState::Captured => StderrCaptureState::Captured,
7958 CaptureState::Incomplete { reason } => {
7959 StderrCaptureState::Incomplete { reason }
7960 }
7961 CaptureState::NotCaptured { reason } => {
7962 StderrCaptureState::NotCaptured { reason }
7963 }
7964 },
7965 entries: snapshot
7966 .entries
7967 .into_iter()
7968 .map(|entry| match entry {
7969 TailEntry::Line {
7970 text,
7971 truncated,
7972 at_ms,
7973 } => StderrTailEntry::Line {
7974 text,
7975 truncated,
7976 at_ms,
7977 },
7978 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
7979 })
7980 .collect(),
7981 dropped_lines: snapshot.dropped_lines,
7982 },
7983 };
7984 Ok(vec![control_response_body_frame(
7985 &frame,
7986 &response,
7987 "ClientControlResponse::SupervisorStderrTail",
7988 )?])
7989 }
7990
7991 async fn observe_control_wait<F: std::future::Future>(
7992 &self,
7993 ctx: &RouteCtx,
7994 target: Option<&str>,
7995 kind: &'static str,
7996 future: F,
7997 ) -> F::Output {
7998 self.counters
7999 .observe_reader_wait(
8000 || {
8001 let module = self
8002 .forwarding
8003 .module_id_for_connection(ctx.connection_id)
8004 .ok()
8005 .flatten();
8006 let principal = if module.is_some() { "module" } else { "direct" };
8007 (
8008 self.resolve_reader_wait_target(kind, target)
8009 .or(module)
8010 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
8011 principal.into(),
8012 )
8013 },
8014 kind,
8015 future,
8016 )
8017 .await
8018 }
8019
8020 pub(crate) fn resolve_reader_wait_target(
8023 &self,
8024 op: &str,
8025 selector: Option<&str>,
8026 ) -> Option<String> {
8027 let selector = selector?;
8028 match op {
8029 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
8030 .supervisor
8031 .get(selector)
8032 .map(|module| module.module_id().to_owned()),
8033 ops::SUPERVISOR_HEALTH_PROBE => self
8034 .registry
8035 .get_module(selector)
8036 .ok()
8037 .flatten()
8038 .map(|registration| registration.manifest.module_id),
8039 _ => None,
8040 }
8041 }
8042
8043 async fn handle_supervisor_terminals(
8044 &self,
8045 ctx: &RouteCtx,
8046 frame: Frame,
8047 module_id: String,
8048 ) -> Result<Vec<Frame>, RouterError> {
8049 let Some(module) = self.supervisor.get(&module_id) else {
8050 return Ok(vec![control_error_frame(
8051 &frame,
8052 "unknown_module",
8053 format!("module_id '{module_id}' is not supervised"),
8054 )?]);
8055 };
8056
8057 let terminals = self
8060 .observe_control_wait(
8061 ctx,
8062 Some(module.module_id()),
8063 ops::SUPERVISOR_TERMINALS,
8064 module.read_durable_terminal_history(),
8065 )
8066 .await
8067 .map_err(|error| {
8068 RouterError::backend(
8069 0,
8070 frame.header.corr,
8071 format!("failed to read terminal history: {error}"),
8072 )
8073 })?;
8074 let response = ClientControlResponse::SupervisorTerminals {
8075 module_id,
8076 terminals,
8077 };
8078 Ok(vec![control_response_body_frame(
8079 &frame,
8080 &response,
8081 "ClientControlResponse::SupervisorTerminals",
8082 )?])
8083 }
8084
8085 fn handle_supervisor_routes(
8086 &self,
8087 frame: Frame,
8088 module_id: Option<String>,
8089 ) -> Result<Vec<Frame>, RouterError> {
8090 let modules = self
8091 .forwarding
8092 .route_census(module_id.as_deref())
8093 .map_err(RouterError::Forwarding)?
8094 .into_iter()
8095 .map(|(module_id, routes)| SupervisorRouteModule {
8096 module_id,
8097 routes: routes
8098 .into_iter()
8099 .map(|route| SupervisorRoute {
8100 consumer: match route.principal {
8101 Principal::Reserved { module_id } => {
8102 SupervisorRouteConsumer::Reserved { module_id }
8103 }
8104 Principal::Direct | Principal::Unverified => {
8105 SupervisorRouteConsumer::Direct {
8106 connection_id: route.goodbye_target.connection_id.get(),
8107 }
8108 }
8109 },
8110 age_ms: Instant::now()
8111 .saturating_duration_since(route.bound_at)
8112 .as_millis()
8113 .try_into()
8114 .unwrap_or(u64::MAX),
8115 draining: route.draining,
8116 drain_reason: route.drain_reason,
8117 })
8118 .collect(),
8119 })
8120 .collect();
8121 let response = ClientControlResponse::SupervisorRoutes { modules };
8122 Ok(vec![control_response_body_frame(
8123 &frame,
8124 &response,
8125 "ClientControlResponse::SupervisorRoutes",
8126 )?])
8127 }
8128
8129 fn running_registration_method(
8130 &self,
8131 module: &str,
8132 ) -> Option<subc_control::RegistrationMethod> {
8133 let registration = self.registry.get_module(module).ok()??;
8134 let connection = registration.connection_id;
8135 let channel = self.supervisor.registration_spawn(connection).is_some();
8136 let nonce = if channel {
8137 None
8138 } else {
8139 self.hello_launch_nonces
8140 .lock()
8141 .ok()?
8142 .nonce(connection)
8143 .map(str::to_owned)
8144 };
8145 if !channel && nonce.is_none() {
8146 return None;
8147 }
8148 self.supervisor
8149 .registration_snapshot_is_current_live(
8150 &self.registry,
8151 module,
8152 connection,
8153 nonce.as_deref(),
8154 )
8155 .then_some(if channel {
8156 subc_control::RegistrationMethod::Channel
8157 } else {
8158 subc_control::RegistrationMethod::Nonce
8159 })
8160 }
8161
8162 async fn handle_supervisor_provenance(
8163 &self,
8164 ctx: &RouteCtx,
8165 frame: Frame,
8166 module_id: Option<String>,
8167 ) -> Result<Vec<Frame>, RouterError> {
8168 let targeted = module_id.is_some();
8169 let selected = if let Some(module_id) = module_id {
8170 let Some(module) = self.supervisor.get(&module_id) else {
8171 return Ok(vec![control_error_frame(
8172 &frame,
8173 "unknown_module",
8174 format!("module_id '{module_id}' is not supervised"),
8175 )?]);
8176 };
8177 vec![module]
8178 } else {
8179 self.supervisor.list()
8180 };
8181
8182 let target = targeted.then(|| selected[0].module_id());
8183 let declarations = if let Some(owner) = &self.config_runtime {
8184 owner
8185 .on_blocking(|owner| owner.registration_declarations())
8186 .await
8187 .ok()
8188 } else {
8189 None
8190 };
8191 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
8192 let mut modules = Vec::with_capacity(selected.len());
8193 let mut extensions = Vec::with_capacity(selected.len());
8194 for module in &selected {
8195 let (status, observed_image) = module
8196 .status_and_running_image_agreement()
8197 .await
8198 .map_err(|err| {
8199 RouterError::backend(
8200 0,
8201 frame.header.corr,
8202 format!("failed to read supervisor status: {err}"),
8203 )
8204 })?;
8205 let module_declared = self
8206 .registry
8207 .get_module(&status.module_id)
8208 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
8209 .and_then(|registration| registration.manifest.provenance)
8210 .map(|build| ModuleDeclaredProvenance::Reported { build })
8211 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
8212 #[cfg(test)]
8213 let running_image = match &self.provenance_probe_override {
8214 Some(result) => result.clone(),
8215 None => observed_image,
8216 };
8217 #[cfg(not(test))]
8218 let running_image = observed_image;
8219 let registration = self.running_registration_method(&status.module_id);
8220 let mut extension = subc_control::SupervisorModuleProvenanceExt::new()
8221 .with_registration(registration);
8222 extension.registration_declaration = declarations
8223 .as_ref()
8224 .and_then(|entries| entries.get(&status.module_id))
8225 .cloned();
8226 if status.process_alive && status.registration_declaration.is_some() {
8227 extension.registration_declaration = status.registration_declaration.clone();
8228 }
8229 extensions.push(extension);
8230 modules.push(SupervisorModuleProvenance {
8231 module_id: status.module_id,
8232 module_declared,
8233 daemon_observed: SupervisorObservedProcess {
8234 launch_nonce_source: status.launch_nonce_source,
8235 pid: status.pid,
8236 spawned_at_ms: status.spawned_at_ms,
8237 spawned_from: status.spawned_from,
8238 running_image,
8239 },
8240 });
8241 }
8242 let daemon = SupervisorDaemonProvenance {
8243 daemon_build: self.daemon_provenance.build.clone(),
8244 daemon_observed: DaemonObservedProcess {
8245 pid: self.daemon_provenance.pid,
8246 started_at_ms: self
8247 .daemon_provenance
8248 .start_clock
8249 .map(|clock| clock.started_at_ms())
8250 .or(self.daemon_provenance.started_at_ms),
8251 running_image: self
8252 .daemon_provenance
8253 .probe
8254 .observe(
8255 self.daemon_provenance.pid,
8256 self.daemon_provenance.executable_path.as_deref(),
8257 self.daemon_provenance.executable_identity.clone(),
8258 self.daemon_provenance.process_start_time,
8259 )
8260 .await,
8261 },
8262 };
8263 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
8264 let mut response = serde_json::to_value(response)
8265 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
8266 for (entry, extension) in response["modules"]
8267 .as_array_mut()
8268 .expect("provenance modules are an array")
8269 .iter_mut()
8270 .zip(extensions)
8271 {
8272 extension.write_to(entry).map_err(|error| {
8273 RouterError::backend(0, frame.header.corr, error.to_string())
8274 })?;
8275 }
8276 Ok(vec![control_response_body_frame(
8277 &frame,
8278 &response,
8279 "ClientControlResponse::SupervisorProvenance",
8280 )?])
8281 })
8282 .await
8283 }
8284
8285 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
8286 self.refresh_capability_requirements();
8287 let publication = self
8288 .config_runtime
8289 .as_ref()
8290 .and_then(|owner| owner.polled_snapshot().ok());
8291 let generation = self
8292 .registry
8293 .generation()
8294 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
8295 let modules = self
8296 .supervisor
8297 .list()
8298 .into_iter()
8299 .map(|module| {
8300 let status = module.status_for_control("health").map_err(|err| {
8301 RouterError::backend(
8302 0,
8303 frame.header.corr,
8304 format!("failed to read supervisor health: {err}"),
8305 )
8306 })?;
8307 let module_id = status.module_id;
8308 let capability_detail = self
8309 .capability_evaluator
8310 .required_problem_detail(&module_id);
8311 let declaration_warning = self.registry.get_module(&module_id).ok().flatten()
8312 .filter(|registration| registration.manifest.registration_channel == Some(false))
8313 .filter(|_| publication.as_ref().and_then(|view| view.registration_declaration(&module_id))
8314 .as_deref().map(|next| if status.process_alive { status.registration_declaration.as_deref().unwrap_or(next) } else { next }) == Some("declared"))
8315 .map(|_| format!("{module_id}: HELLO registration_channel is false despite the approved channel declaration"));
8316 let capability_detail = append_capability_problem_detail(capability_detail, declaration_warning);
8317 Ok(SupervisorHealthEntry {
8318 module_id,
8319 status: status.health.status,
8320 detail: append_capability_problem_detail(
8321 status.health.detail,
8322 capability_detail,
8323 ),
8324 metrics: status.health.metrics,
8325 consecutive_failures: status.health.consecutive_failures,
8326 late_answer_count: status.health.late_answer_count,
8327 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
8328 last_action: status.health.last_action,
8329 last_action_ms: status.health.last_action_ms,
8330 last_probe_ms: status.health.last_probe_ms,
8331 })
8332 })
8333 .collect::<Result<Vec<_>, RouterError>>()?;
8334 let response = ClientControlResponse::SupervisorHealth {
8335 generation,
8336 modules,
8337 };
8338 Ok(vec![control_response_body_frame(
8339 &frame,
8340 &response,
8341 "ClientControlResponse::SupervisorHealth",
8342 )?])
8343 }
8344
8345 async fn handle_supervisor_restart(
8346 &self,
8347 frame: Frame,
8348 module_id: String,
8349 drain_timeout_ms: Option<u64>,
8350 requester: Principal,
8351 ) -> Result<Vec<Frame>, RouterError> {
8352 let operation_lock = self.supervisor.operation_lock();
8353 let _operation_guard = operation_lock.lock().await;
8354 let Some(module) = self.supervisor.get(&module_id) else {
8355 return Ok(vec![control_error_frame(
8356 &frame,
8357 "unknown_module",
8358 format!("module_id '{module_id}' is not supervised"),
8359 )?]);
8360 };
8361
8362 self.route_outages.mark_operator_action(&module_id);
8363 if let Err(err) = module
8364 .restart_requested(drain_timeout_ms, Some(requester))
8365 .await
8366 {
8367 self.route_outages
8368 .operator_action_ended_unrefused(&module_id);
8369 let (code, message) = match err {
8370 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8371 crate::supervise::SuperviseError::Disabled { .. } => {
8372 ("module_disabled", err.to_string())
8373 }
8374 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8375 ("swap_in_progress", err.to_string())
8376 }
8377 _ => (
8378 "target_unavailable",
8379 format!("failed to restart module_id '{module_id}': {err}"),
8380 ),
8381 };
8382 return Ok(vec![control_error_frame(&frame, code, message)?]);
8383 }
8384
8385 let response = ClientControlResponse::SupervisorAck {
8386 module_id,
8387 applied: true,
8388 };
8389 Ok(vec![control_response_body_frame(
8390 &frame,
8391 &response,
8392 "ClientControlResponse::SupervisorAck",
8393 )?])
8394 }
8395
8396 async fn handle_supervisor_swap(
8400 &self,
8401 frame: Frame,
8402 module_id: String,
8403 ready_timeout_ms: Option<u64>,
8404 requester: Principal,
8405 ) -> Result<Vec<Frame>, RouterError> {
8406 let module = {
8413 let operation_lock = self.supervisor.operation_lock();
8414 let _operation_guard = operation_lock.lock().await;
8415 self.supervisor.get(&module_id)
8416 };
8417 let Some(module) = module else {
8418 return Ok(vec![control_error_frame(
8419 &frame,
8420 "unknown_module",
8421 format!("module_id '{module_id}' is not supervised"),
8422 )?]);
8423 };
8424
8425 self.route_outages.mark_operator_action(&module_id);
8426 if let Err(err) = module
8427 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
8428 .await
8429 {
8430 self.route_outages
8431 .operator_action_ended_unrefused(&module_id);
8432 use crate::supervise::SuperviseError;
8433 let message = err.to_string();
8434 let error = match err {
8435 SuperviseError::ConfigHeld { .. } => ErrorBody::new("config_gated", message),
8436 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
8437 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
8438 code: "swap_refused".to_string(),
8439 message,
8440 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
8441 },
8442 SuperviseError::SwapFailed {
8443 arm,
8444 candidate_exit,
8445 ..
8446 } => ErrorBody {
8447 code: "swap_failed".to_string(),
8448 message,
8449 detail: Some(serde_json::json!({
8450 "arm": arm.as_str(),
8451 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
8452 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
8453 })),
8454 },
8455 _ => ErrorBody::new(
8456 "target_unavailable",
8457 format!("failed to swap module_id '{module_id}': {message}"),
8458 ),
8459 };
8460 return Ok(vec![control_error_body_frame(&frame, error)?]);
8461 }
8462 self.route_outages
8466 .operator_action_ended_unrefused(&module_id);
8467
8468 let response = ClientControlResponse::SupervisorAck {
8469 module_id,
8470 applied: true,
8471 };
8472 Ok(vec![control_response_body_frame(
8473 &frame,
8474 &response,
8475 "ClientControlResponse::SupervisorAck",
8476 )?])
8477 }
8478
8479 async fn handle_supervisor_reload(
8480 &self,
8481 frame: Frame,
8482 module_id: String,
8483 requester: Principal,
8484 ) -> Result<Vec<Frame>, RouterError> {
8485 let operation_lock = self.supervisor.operation_lock();
8486 let _operation_guard = operation_lock.lock().await;
8487 let Some(module) = self.supervisor.get(&module_id) else {
8488 return Ok(vec![control_error_frame(
8489 &frame,
8490 "unknown_module",
8491 format!("module_id '{module_id}' is not supervised"),
8492 )?]);
8493 };
8494
8495 self.route_outages.mark_operator_action(&module_id);
8496 if let Err(err) = module.reload_requested(Some(requester)).await {
8497 self.route_outages
8498 .operator_action_ended_unrefused(&module_id);
8499 let (code, message) = match err {
8500 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8501 crate::supervise::SuperviseError::Disabled { .. } => {
8502 ("module_disabled", err.to_string())
8503 }
8504 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8505 ("swap_in_progress", err.to_string())
8506 }
8507 _ => (
8508 "reload_failed",
8509 format!("failed to reload module_id '{module_id}': {err}"),
8510 ),
8511 };
8512 return Ok(vec![control_error_frame(&frame, code, message)?]);
8513 }
8514
8515 let response = ClientControlResponse::SupervisorAck {
8516 module_id,
8517 applied: true,
8518 };
8519 Ok(vec![control_response_body_frame(
8520 &frame,
8521 &response,
8522 "ClientControlResponse::SupervisorAck",
8523 )?])
8524 }
8525
8526 async fn handle_supervisor_rescan(
8530 &self,
8531 frame: Frame,
8532 mode: RescanMode,
8533 requester: Principal,
8534 ) -> Result<Vec<Frame>, RouterError> {
8535 let preview = mode.previews();
8536 let mut approved_preview = None;
8537 if matches!(mode, RescanMode::Preview) {
8538 if let Some(owner) = &self.config_runtime {
8539 match owner
8540 .on_blocking(|owner| {
8541 owner.observe_file_proposal()?;
8542 owner.approved_daemon_configuration()
8543 })
8544 .await
8545 {
8546 Ok(config) => approved_preview = config,
8547 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8548 }
8549 }
8550 }
8551 let Some(context) = self.rescan.clone() else {
8552 return Ok(vec![control_error_frame(
8553 &frame,
8554 "rescan_unavailable",
8555 "the daemon was not started with a reloadable config path".to_string(),
8556 )?]);
8557 };
8558
8559 let operation_lock = self.supervisor.operation_lock();
8560 let _operation_guard = operation_lock.lock().await;
8561 let loaded = match self.load_rescan_config(&mode, &context) {
8565 Ok(config) => config,
8566 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
8567 };
8568 let Some((config, activity_config)) = loaded else {
8576 return Ok(vec![control_error_frame(
8577 &frame,
8578 "invalid_daemon_config",
8579 format!(
8580 "daemon config not found at {}; refusing to rescan (an absent config would \
8581 retire every supervised module)",
8582 context.config_path.display()
8583 ),
8584 )?]);
8585 };
8586 let (
8587 configured_port,
8588 storage_config,
8589 admission_facts_carrier_module_id,
8590 admission_facts_targets,
8591 scope_authority_owners,
8592 federated_scope_owner,
8593 privileged_operator_prompt,
8594 modules,
8595 reserved_capabilities,
8596 ) = (
8597 config.port,
8598 config.storage,
8599 config.admission_facts_carrier_module_id,
8600 config.admission_facts_targets,
8601 config.scope_authority_owners,
8602 config.federated_scope_owner,
8603 config.privileged_operator_prompt,
8604 config.modules,
8605 config.reserved_capabilities,
8606 );
8607
8608 let mut restart_required = Vec::new();
8617 for section in RestartRequiredSection::ALL {
8618 let changed = match section {
8619 RestartRequiredSection::Port => configured_port != context.configured_port,
8620 RestartRequiredSection::Storage => storage_config != context.storage_config,
8621 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
8622 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
8623 }
8624 RestartRequiredSection::AdmissionFactsTargets => {
8625 admission_facts_targets != context.admission_facts_targets
8626 }
8627 RestartRequiredSection::ScopeAuthorityOwners => {
8628 scope_authority_owners != context.scope_authority_owners
8629 }
8630 RestartRequiredSection::FederatedScopeOwner => {
8631 federated_scope_owner != context.federated_scope_owner
8632 }
8633 RestartRequiredSection::PrivilegedOperatorPrompt => {
8634 privileged_operator_prompt != context.privileged_operator_prompt
8635 }
8636 };
8637 if changed {
8638 restart_required.push(section.label().to_string());
8639 }
8640 }
8641 if !restart_required.is_empty() {
8642 warn!(
8643 config_path = %context.config_path.display(),
8644 sections = %restart_required.join(", "),
8645 "daemon config changed outside the modules section; restart the daemon to apply those changes"
8646 );
8647 }
8648
8649 for configured in &modules {
8650 if let Err(err) = validate_spec(&configured.module_spec()) {
8651 return Ok(vec![control_error_frame(
8652 &frame,
8653 "invalid_daemon_config",
8654 format!("supervisor rescan rejected daemon config: {err}"),
8655 )?]);
8656 }
8657 }
8658
8659 let configured_capabilities = modules
8660 .iter()
8661 .map(|module| (module.module_id.clone(), module.enabled))
8662 .collect::<Vec<_>>();
8663 let preview_capability_warnings = if preview {
8664 let (_, registrations) = self.runtime_capability_snapshot()?;
8665 let current_modules = self
8666 .supervisor
8667 .list()
8668 .into_iter()
8669 .map(|module| module.module_id().to_string())
8670 .collect::<BTreeSet<_>>();
8671 let resulting_modules = configured_capabilities.clone();
8672 let removed = current_modules
8673 .into_iter()
8674 .filter(|module_id| {
8675 !resulting_modules
8676 .iter()
8677 .any(|(configured_id, _)| configured_id == module_id)
8678 })
8679 .collect::<Vec<_>>();
8680 let disabled = match &mode {
8683 RescanMode::Candidate(_) => resulting_modules
8684 .iter()
8685 .filter(|(_, enabled)| !enabled)
8686 .map(|(module_id, _)| module_id.clone())
8687 .collect::<Vec<_>>(),
8688 _ => Vec::new(),
8689 };
8690 self.capability_evaluator.preview_warnings(
8691 resulting_modules,
8692 &removed,
8693 &disabled,
8694 ®istrations,
8695 )
8696 } else {
8697 Vec::new()
8698 };
8699 let reconciled = match (&mode, approved_preview.as_ref()) {
8700 (RescanMode::Preview, Some(approved)) => Ok((
8701 config_gate::preview_approved_daemon(approved, &modules),
8702 Vec::new(),
8703 )),
8704 (RescanMode::Candidate(candidate), _) => self
8705 .preview_candidate(&context, modules, candidate)
8706 .map(|result| (result, Vec::new())),
8707 _ => {
8708 self.reconcile_supervised_modules(
8709 &context.supervisor,
8710 modules,
8711 preview,
8712 requester.clone(),
8713 )
8714 .await
8715 }
8716 };
8717 let (result, changed) = match reconciled {
8718 Ok(result) => result,
8719 Err(message) => {
8720 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
8721 }
8722 };
8723 if !preview {
8724 if let Some(ledger) = &self.activity {
8725 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
8726 ledger.config_changed(&result.added, &result.removed, &changed, requester);
8727 }
8728 if let Err(code) = self.configure_activity(activity_config).await {
8729 warn!(
8730 code,
8731 "activity retention could not be applied during rescan"
8732 );
8733 }
8734 self.capability_evaluator
8735 .configure(configured_capabilities, reserved_capabilities);
8736 self.capability_evaluator.wake_deadline_loop();
8737 self.refresh_capability_requirements();
8738 }
8739 let mut result = result;
8740 result.restart_required = restart_required;
8741 result.capability_warnings = preview_capability_warnings;
8742 let response = ClientControlResponse::SupervisorRescan { result };
8743 Ok(vec![control_response_body_frame(
8744 &frame,
8745 &response,
8746 "ClientControlResponse::SupervisorRescan",
8747 )?])
8748 }
8749
8750 async fn handle_supervisor_release_reserved(
8751 &self,
8752 frame: Frame,
8753 module_id: String,
8754 ) -> Result<Vec<Frame>, RouterError> {
8755 let Some(context) = self.rescan.clone() else {
8756 return Ok(vec![control_error_frame(
8757 &frame,
8758 "release_unavailable",
8759 "reserved-id release requires a daemon started with a reloadable config path",
8760 )?]);
8761 };
8762 let operation_lock = self.supervisor.operation_lock();
8763 let _operation_guard = operation_lock.lock().await;
8764 let approved = if let Some(owner) = &self.config_runtime {
8765 match owner
8768 .on_blocking(|owner| {
8769 owner.observe_file_proposal()?;
8770 owner.approved_daemon_configuration()
8771 })
8772 .await
8773 {
8774 Ok(config) => config,
8775 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8776 }
8777 } else {
8778 None
8779 };
8780 let loaded = if let Some(approved) = approved {
8781 Ok(Some(approved))
8782 } else {
8783 crate::daemon_config::load(&context.config_path)
8784 };
8785 let loaded = match loaded {
8786 Ok(Some(config)) => config,
8787 Ok(None) => {
8788 return Ok(vec![control_error_frame(
8789 &frame,
8790 "invalid_daemon_config",
8791 format!(
8792 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
8793 context.config_path.display()
8794 ),
8795 )?])
8796 }
8797 Err(err) => {
8798 return Ok(vec![control_error_frame(
8799 &frame,
8800 "invalid_daemon_config",
8801 format!("unable to verify reserved-id release against daemon config: {err}"),
8802 )?])
8803 }
8804 };
8805 if loaded
8806 .modules
8807 .iter()
8808 .any(|configured| configured.module_id == module_id)
8809 {
8810 return Ok(vec![control_error_frame(
8811 &frame,
8812 "reserved_module_configured",
8813 format!(
8814 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
8815 ),
8816 )?]);
8817 }
8818 if !self.supervisor.release_retained_reserved_gate(&module_id) {
8819 return Ok(vec![control_error_frame(
8820 &frame,
8821 "reserved_gate_not_retained",
8822 format!(
8823 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
8824 ),
8825 )?]);
8826 }
8827
8828 let response = ClientControlResponse::SupervisorAck {
8829 module_id,
8830 applied: true,
8831 };
8832 Ok(vec![control_response_body_frame(
8833 &frame,
8834 &response,
8835 "ClientControlResponse::SupervisorAck",
8836 )?])
8837 }
8838
8839 fn plan_rescan(
8843 &self,
8844 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8845 ) -> Result<RescanPlan, String> {
8846 let mut current = BTreeMap::new();
8847 for module in self.supervisor.list() {
8848 let (spec, health) = module.configuration().map_err(|err| {
8849 format!(
8850 "failed to read configuration for module_id '{}': {err}",
8851 module.module_id()
8852 )
8853 })?;
8854 let enabled = module
8855 .status()
8856 .map_err(|err| {
8857 format!(
8858 "failed to read status for module_id '{}': {err}",
8859 module.module_id()
8860 )
8861 })?
8862 .enabled;
8863 current.insert(
8864 module.module_id().to_string(),
8865 (module, spec, health, enabled),
8866 );
8867 }
8868 let configured = configured_modules
8869 .into_iter()
8870 .map(|module| (module.module_id.clone(), module))
8871 .collect::<BTreeMap<_, _>>();
8872
8873 let added = configured
8874 .keys()
8875 .filter(|module_id| !current.contains_key(*module_id))
8876 .cloned()
8877 .collect::<Vec<_>>();
8878 let removed = current
8879 .keys()
8880 .filter(|module_id| !configured.contains_key(*module_id))
8881 .cloned()
8882 .collect::<Vec<_>>();
8883 let mut changed_pending_reload = Vec::new();
8884 let mut changed_pending = Vec::new();
8885 let mut configuration_changes = BTreeSet::new();
8886 let mut enabled_changes = BTreeSet::new();
8887 let mut unchanged = 0_u32;
8888 let publication = self
8889 .config_runtime
8890 .as_ref()
8891 .and_then(|owner| owner.polled_snapshot().ok());
8892
8893 for (module_id, configured_module) in &configured {
8894 let Some((module, current_spec, current_health, current_enabled)) =
8895 current.get(module_id)
8896 else {
8897 continue;
8898 };
8899 let launch_changed = *current_spec != configured_module.module_spec();
8903 let configuration_changed =
8904 launch_changed || *current_health != configured_module.health;
8905 let pending = self.pending_module_fields(
8906 module,
8907 Some(configured_module),
8908 configuration_changed,
8909 Self::registration_channel_in_force(
8910 publication
8911 .as_ref()
8912 .and_then(|view| view.registration_declaration(module_id))
8913 .as_deref(),
8914 publication
8915 .as_ref()
8916 .and_then(|view| match &view.approved_daemon {
8917 crate::config_gate::runtime::ApprovedDaemonView::Approved {
8918 config,
8919 ..
8920 } => config
8921 .modules
8922 .iter()
8923 .find(|entry| entry.module_id == *module_id)
8924 .map(|entry| entry.registration_channel),
8925 _ => None,
8926 }),
8927 ),
8928 )?;
8929 let has_pending = !pending.is_empty();
8930 if has_pending {
8931 changed_pending.push(subc_control::SupervisorPendingModule {
8932 module_id: module_id.clone(),
8933 fields: pending
8934 .into_iter()
8935 .map(|mut field| {
8936 field.value = None;
8937 field
8938 })
8939 .collect(),
8940 });
8941 }
8942 let enabled_changed = *current_enabled != configured_module.enabled
8945 || self
8946 .capability_evaluator
8947 .configured_enabled(module_id)
8948 .is_some_and(|enabled| enabled != configured_module.enabled);
8949 if configuration_changed {
8950 configuration_changes.insert(module_id.clone());
8951 }
8952 if launch_changed {
8953 changed_pending_reload.push(module_id.clone());
8954 }
8955 if enabled_changed {
8956 enabled_changes.insert(module_id.clone());
8957 }
8958 if !configuration_changed && !enabled_changed && !has_pending {
8959 unchanged = unchanged.saturating_add(1);
8960 }
8961 }
8962 Ok(RescanPlan {
8963 current,
8964 configured,
8965 added,
8966 removed,
8967 changed_pending_reload,
8968 changed_pending,
8969 configuration_changes,
8970 enabled_changes,
8971 unchanged,
8972 })
8973 }
8974
8975 async fn reconcile_supervised_modules(
8984 &self,
8985 supervisor: &Supervisor,
8986 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8987 preview: bool,
8988 requester: Principal,
8989 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
8990 let plan = self.plan_rescan(configured_modules)?;
8991
8992 if preview {
8998 return Ok((plan.preview_result(), Vec::new()));
8999 }
9000 let changed = plan
9001 .configuration_changes
9002 .union(&plan.enabled_changes)
9003 .cloned()
9004 .collect::<Vec<_>>();
9005 let RescanPlan {
9006 current,
9007 configured,
9008 added,
9009 removed,
9010 changed_pending_reload,
9011 changed_pending,
9012 configuration_changes,
9013 enabled_changes,
9014 unchanged,
9015 } = plan;
9016 if !(added.is_empty()
9021 && removed.is_empty()
9022 && configuration_changes.is_empty()
9023 && changed_pending.is_empty()
9024 && enabled_changes.is_empty())
9025 {
9026 if let Some(context) = &self.rescan {
9027 context
9028 .generation
9029 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
9030 }
9031 }
9032
9033 for module_id in &removed {
9034 let module = ¤t
9035 .get(module_id)
9036 .expect("removed module came from current supervisor state")
9037 .0;
9038 module
9039 .retire_requested(Some(requester.clone()))
9040 .await
9041 .map_err(|err| {
9042 format!("failed to retire module_id '{module_id}' during rescan: {err}")
9043 })?;
9044 self.supervisor.record_rescan_removal(module_id);
9078 self.supervisor.retire(module_id);
9079 self.route_outages.forget(module_id);
9080 }
9081
9082 for module_id in configured.keys() {
9083 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
9084 continue;
9085 };
9086 let configured_module = configured
9087 .get(module_id)
9088 .expect("configured module id came from configured map");
9089 if configuration_changes.contains(module_id) {
9090 module
9091 .update_configuration(
9092 configured_module.module_spec(),
9093 configured_module.health.clone(),
9094 configured_module.drain_timeout_ms,
9095 )
9096 .await
9097 .map_err(|err| {
9098 format!(
9099 "failed to update module_id '{module_id}' configuration during rescan: {err}"
9100 )
9101 })?;
9102 }
9103 if *current_enabled != configured_module.enabled {
9106 self.route_outages.mark_operator_action(module_id);
9109 module
9110 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
9111 .await
9112 .map_err(|err| {
9113 self.route_outages.operator_action_ended_unrefused(module_id);
9114 format!(
9115 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
9116 configured_module.enabled
9117 )
9118 })?;
9119 }
9120 }
9121
9122 for module_id in &added {
9123 let configured_module = configured
9124 .get(module_id)
9125 .expect("added module id came from configured map");
9126 supervisor
9127 .supervise_configured_requested_async(
9128 configured_module.module_spec(),
9129 configured_module.enabled,
9130 configured_module.health.clone(),
9131 configured_module.drain_timeout_ms,
9132 configured_module.restart,
9133 configured_module.register_timeout,
9134 Some(requester.clone()),
9135 )
9136 .await
9137 .map_err(|err| {
9138 format!("failed to add module_id '{module_id}' during rescan: {err}")
9139 })?;
9140 }
9141
9142 if let Some(context) = &self.rescan {
9143 let mut entries = context
9144 .module_entries
9145 .lock()
9146 .unwrap_or_else(|e| e.into_inner());
9147 entries.record(&configured, &added, &removed);
9148 }
9149
9150 Ok((
9151 SupervisorRescanResult {
9152 added,
9153 removed,
9154 changed_pending_reload,
9155 changed_pending,
9156 enabled_changes: enabled_changes.iter().cloned().collect(),
9157 unchanged,
9158 preview: false,
9159 restart_required: Vec::new(),
9162 capability_warnings: Vec::new(),
9163 module_actions: Vec::new(),
9164 earlier_pending: Vec::new(),
9165 expect: None,
9166 },
9167 changed,
9168 ))
9169 }
9170
9171 async fn handle_supervisor_set_enabled(
9172 &self,
9173 frame: Frame,
9174 module_id: String,
9175 enabled: bool,
9176 requester: Principal,
9177 ) -> Result<Vec<Frame>, RouterError> {
9178 let operation_lock = self.supervisor.operation_lock();
9179 let _operation_guard = operation_lock.lock().await;
9180 let Some(module) = self.supervisor.get(&module_id) else {
9181 return Ok(vec![control_error_frame(
9182 &frame,
9183 "unknown_module",
9184 format!("module_id '{module_id}' is not supervised"),
9185 )?]);
9186 };
9187
9188 self.route_outages.mark_operator_action(&module_id);
9191 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
9192 Ok(applied) => applied,
9193 Err(err) => {
9194 self.route_outages
9195 .operator_action_ended_unrefused(&module_id);
9196 return Ok(vec![control_error_frame(
9197 &frame,
9198 if err.is_config_gate_error() {
9199 "config_gated"
9200 } else {
9201 "target_unavailable"
9202 },
9203 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
9204 )?]);
9205 }
9206 };
9207 if !applied {
9208 self.route_outages
9211 .operator_action_ended_unrefused(&module_id);
9212 }
9213
9214 self.capability_evaluator.wake_deadline_loop();
9215 self.refresh_capability_requirements();
9216 let response = ClientControlResponse::SupervisorAck { module_id, applied };
9217 Ok(vec![control_response_body_frame(
9218 &frame,
9219 &response,
9220 "ClientControlResponse::SupervisorAck",
9221 )?])
9222 }
9223
9224 async fn handle_supervisor_health_probe(
9225 &self,
9226 ctx: &RouteCtx,
9227 frame: Frame,
9228 module_id: String,
9229 ) -> Result<Vec<Frame>, RouterError> {
9230 self.refresh_capability_requirements();
9231 let Some(registration) = self
9232 .registry
9233 .get_module(&module_id)
9234 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
9235 else {
9236 return Ok(vec![control_error_frame(
9237 &frame,
9238 "unknown_module",
9239 format!("module_id '{module_id}' is not registered"),
9240 )?]);
9241 };
9242
9243 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
9256 {
9257 return Ok(vec![control_error_frame(
9258 &frame,
9259 "health_not_advertised",
9260 format!("module_id '{module_id}' did not advertise health.check"),
9261 )?]);
9262 }
9263
9264 self.observe_control_wait(
9265 ctx,
9266 Some(®istration.manifest.module_id),
9267 ops::SUPERVISOR_HEALTH_PROBE,
9268 async {
9269 let deadline = Instant::now() + self.health_probe_timeout;
9270 let pending = match self.forwarding.begin_module_control_rpc_for(
9271 &module_id,
9272 MODULE_CONTROL_OP_HEALTH_CHECK,
9273 deadline,
9274 ) {
9275 Ok(pending) => pending,
9276 Err(err) => {
9277 return Ok(vec![control_error_frame(
9278 &frame,
9279 forwarding_error_code(&err),
9280 err.to_string(),
9281 )?])
9282 }
9283 };
9284
9285 let PendingModuleControlRpc {
9286 endpoint,
9287 module_sink,
9288 negotiated_ver,
9289 corr: probe_corr,
9290 receiver,
9291 } = pending;
9292 let mut guard =
9293 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
9294 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
9295 .map_err(|err| {
9296 RouterError::backend(
9297 0,
9298 frame.header.corr,
9299 format!("failed to encode health.check request: {err}"),
9300 )
9301 })?;
9302 let probe_frame = Frame::build_with_version(
9303 negotiated_ver,
9304 FrameType::Request,
9305 control_flags(),
9306 0,
9307 0,
9308 probe_corr,
9309 probe_body,
9310 )
9311 .map_err(RouterError::FrameBuild)?;
9312
9313 match timeout_at(deadline, module_sink.send(probe_frame)).await {
9316 Ok(Ok(())) => {}
9317 Ok(Err(err)) => {
9318 return Ok(vec![control_error_frame(
9319 &frame,
9320 "target_unavailable",
9321 err.to_string(),
9322 )?]);
9323 }
9324 Err(_) => {
9325 return Ok(vec![control_error_frame(
9326 &frame,
9327 "module_timeout",
9328 format!(
9329 "module_id '{module_id}' did not answer health.check within {:?}",
9330 self.health_probe_timeout
9331 ),
9332 )?]);
9333 }
9334 }
9335
9336 match timeout_at(deadline, receiver).await {
9337 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => {
9338 Ok(vec![control_error_frame(
9339 &frame,
9340 "invalid_control_body",
9341 "health.check returned an unexpected config response",
9342 )?])
9343 }
9344 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
9345 guard.disarm();
9346 let Some(report) = response.health_report() else {
9347 return Ok(vec![control_error_frame(
9348 &frame,
9349 "invalid_control_body",
9350 "health.check RPC returned a non-health response",
9351 )?]);
9352 };
9353 let HealthReport {
9358 status,
9359 detail,
9360 metrics,
9361 } = report;
9362 let capability_detail = self
9363 .capability_evaluator
9364 .required_problem_detail(&module_id);
9365 let response = ClientControlResponse::SupervisorHealthProbe {
9366 module_id,
9367 status,
9368 detail: append_capability_problem_detail(detail, capability_detail),
9369 metrics,
9370 };
9371 Ok(vec![control_response_body_frame(
9372 &frame,
9373 &response,
9374 "ClientControlResponse::SupervisorHealthProbe",
9375 )?])
9376 }
9377 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
9378 guard.disarm();
9379 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
9384 }
9385 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
9386 guard.disarm();
9387 Ok(vec![control_error_frame(
9388 &frame,
9389 "target_unavailable",
9390 message,
9391 )?])
9392 }
9393 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
9394 guard.disarm();
9395 Ok(vec![control_error_frame(
9396 &frame,
9397 "invalid_control_body",
9398 message,
9399 )?])
9400 }
9401 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
9402 guard.disarm();
9403 Ok(vec![control_error_frame(
9404 &frame,
9405 "invalid_control_body",
9406 format!("expected module-control op '{expected}', got '{actual}'"),
9407 )?])
9408 }
9409 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
9410 guard.disarm();
9411 Ok(vec![control_error_frame(
9412 &frame,
9413 "module_timeout",
9414 format!(
9415 "module_id '{module_id}' answered health.check after {:?}",
9416 self.health_probe_timeout
9417 ),
9418 )?])
9419 }
9420 Ok(Err(_)) => Ok(vec![control_error_frame(
9421 &frame,
9422 "target_unavailable",
9423 "health.check waiter was canceled before the module responded",
9424 )?]),
9425 Err(_) => Ok(vec![control_error_frame(
9426 &frame,
9427 "module_timeout",
9428 format!(
9429 "module_id '{module_id}' did not answer health.check within {:?}",
9430 self.health_probe_timeout
9431 ),
9432 )?]),
9433 }
9434 },
9435 )
9436 .await
9437 }
9438
9439 fn supervisor_status(
9440 &self,
9441 module_id: &str,
9442 corr: u64,
9443 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
9444 self.supervisor
9445 .get(module_id)
9446 .map(|module| {
9447 let warming = module.is_warming_for_control("status").map_err(|err| {
9448 RouterError::backend(
9449 0,
9450 corr,
9451 format!(
9452 "failed to read supervisor warming state for module_id '{module_id}': {err}"
9453 ),
9454 )
9455 })?;
9456 module.status_for_control("status").map_err(|err| {
9457 RouterError::backend(
9458 0,
9459 corr,
9460 format!(
9461 "failed to read supervisor status for module_id '{module_id}': {err}"
9462 ),
9463 )
9464 }).map(|status| (status, warming))
9465 })
9466 .transpose()
9467 }
9468
9469 fn guard_module_control_op(
9470 &self,
9471 frame: &Frame,
9472 module_id: &str,
9473 op: &str,
9474 ) -> Result<Option<Frame>, RouterError> {
9475 if self.module_grants_op(module_id, op, frame.header.corr)? {
9476 return Ok(None);
9477 }
9478
9479 Ok(Some(control_error_frame(
9480 frame,
9481 "op_not_allowed",
9482 format!("module_id '{module_id}' did not grant control op '{op}'"),
9483 )?))
9484 }
9485
9486 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
9487 self.registry
9488 .grants_control_op(module_id, op)
9489 .map_err(|err| RouterError::backend(0, corr, err.to_string()))
9490 }
9491
9492 fn handle_status_update(
9493 &self,
9494 endpoint: ModuleEndpointId,
9495 frame: Frame,
9496 ) -> Result<Vec<Frame>, RouterError> {
9497 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
9498 Ok(update) => update,
9499 Err(err) => {
9500 if is_known_module_push_op(&frame.body) {
9505 return Ok(vec![control_error_frame(
9506 &frame,
9507 "invalid_control_body",
9508 format!("malformed module control push body: {err}"),
9509 )?]);
9510 }
9511 return Ok(Vec::new());
9512 }
9513 };
9514
9515 match update {
9516 ModuleControlPush::RouteStatus {
9517 route_channel,
9518 route_epoch,
9519 status,
9520 } => {
9521 self.forwarding
9522 .cache_status(endpoint, route_channel, route_epoch, status)
9523 .map_err(RouterError::Forwarding)?;
9524 }
9525 }
9526 Ok(Vec::new())
9527 }
9528
9529 fn handle_route_poll(
9530 &self,
9531 ctx: &RouteCtx,
9532 frame: Frame,
9533 route_channel: u16,
9534 route_epoch: u32,
9535 kind: PollKind,
9536 ) -> Result<Vec<Frame>, RouterError> {
9537 let snapshot = self
9538 .forwarding
9539 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
9540 .map_err(RouterError::Forwarding)?;
9541 let response = match (kind, snapshot) {
9542 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
9543 ClientControlResponse::RoutePoll {
9544 route_channel,
9545 route_epoch,
9546 status,
9547 live: None,
9548 }
9549 }
9550 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9551 route_channel,
9552 route_epoch,
9553 status: None,
9554 live: None,
9555 },
9556 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
9557 let live = self
9583 .process_liveness
9584 .as_ref()
9585 .and_then(|source| source.process_live(&module_id))
9586 .unwrap_or(true);
9587 ClientControlResponse::RoutePoll {
9588 route_channel,
9589 route_epoch,
9590 status: None,
9591 live: Some(live),
9592 }
9593 }
9594 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9595 route_channel,
9596 route_epoch,
9597 status: None,
9598 live: Some(false),
9599 },
9600 };
9601
9602 Ok(vec![control_response_body_frame(
9603 &frame,
9604 &response,
9605 "ClientControlResponse::RoutePoll",
9606 )?])
9607 }
9608
9609 pub(crate) fn observe_module_control_completion(
9610 &self,
9611 completion: ModuleControlRpcCompletion,
9612 ) -> bool {
9613 match completion {
9614 ModuleControlRpcCompletion::Unknown => false,
9615 ModuleControlRpcCompletion::Settled => true,
9616 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
9617 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
9618 info!(
9619 module_id = %module_id,
9620 latency_ms,
9621 "late health.check answer proves the module is alive"
9622 );
9623 match self
9624 .supervisor
9625 .record_late_health_answer(&module_id, latency_ms)
9626 {
9627 Ok(true) => {}
9628 Ok(false) => debug!(
9629 module_id = %module_id,
9630 latency_ms,
9631 "late health.check answer has no active supervisor snapshot"
9632 ),
9633 Err(err) => warn!(
9634 module_id = %module_id,
9635 latency_ms,
9636 error = %err,
9637 "failed to record late health.check answer"
9638 ),
9639 }
9640 true
9641 }
9642 }
9643 }
9644
9645 fn refuse_to_end_module_connection_for_a_client(
9666 &self,
9667 module_connection_id: ConnectionId,
9668 corr: u64,
9669 err: ForwardingError,
9670 ) -> Result<(), RouterError> {
9671 if let ForwardingError::ConnectionClosing { connection_id } = err {
9672 if connection_id != module_connection_id {
9673 warn!(
9674 module_connection_id = module_connection_id.get(),
9675 client_connection_id = connection_id.get(),
9676 corr,
9677 "dropping a route.bind response for a closing client; the module connection keeps serving"
9678 );
9679 return Ok(());
9680 }
9681 }
9682 Err(RouterError::Forwarding(err))
9683 }
9684
9685 fn handle_module_relay_response(
9686 &self,
9687 connection_id: ConnectionId,
9688 frame: Frame,
9689 ) -> Result<Vec<Frame>, RouterError> {
9690 let mut secondary_error = None;
9691 let outcome = match frame.header.ty {
9692 FrameType::Response => match module_control_response_probe(&frame.body) {
9693 Ok(probe) if probe.op == "route.bind" => {
9694 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9695 Ok(ModuleControlResponse::RouteBindAck {}) => {
9696 RouteBindRelayOutcome::Accepted
9697 }
9698 Ok(other) => {
9699 let message =
9700 format!("route.bind response carried unexpected body: {other:?}");
9701 secondary_error = Some(control_error_frame(
9702 &frame,
9703 "invalid_control_body",
9704 message.clone(),
9705 )?);
9706 RouteBindRelayOutcome::ModuleGone(message)
9707 }
9708 Err(err) => {
9709 let message = format!("malformed route.bind response body: {err}");
9710 secondary_error = Some(control_error_frame(
9711 &frame,
9712 "invalid_control_body",
9713 message.clone(),
9714 )?);
9715 RouteBindRelayOutcome::ModuleGone(message)
9716 }
9717 }
9718 }
9719 Ok(probe) => {
9720 let outcome = if matches!(
9721 probe.op.as_str(),
9722 "config.review" | "consent/v1" | "projects.enumerate"
9723 ) {
9724 ModuleControlRpcOutcome::RawResponse(frame.body.clone())
9725 } else {
9726 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9727 Ok(response) => ModuleControlRpcOutcome::Response(response),
9728 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9729 "malformed {} response body: {err}",
9730 probe.op
9731 )),
9732 }
9733 };
9734 let completion = self
9735 .forwarding
9736 .complete_module_control_rpc(
9737 connection_id,
9738 frame.header.corr,
9739 Some(&probe.op),
9740 outcome,
9741 )
9742 .map_err(RouterError::Forwarding)?;
9743 if !self.observe_module_control_completion(completion) {
9744 debug!(
9745 connection_id = connection_id.get(),
9746 corr = frame.header.corr,
9747 op = %probe.op,
9748 "dropping late or unknown module-control RPC response"
9749 );
9750 }
9751 return Ok(Vec::new());
9752 }
9753 Err(err) => {
9754 if let Some(expected_op) = self
9755 .forwarding
9756 .pending_module_control_op(connection_id, frame.header.corr)
9757 .map_err(RouterError::Forwarding)?
9758 {
9759 let completion = self
9760 .forwarding
9761 .complete_module_control_rpc(
9762 connection_id,
9763 frame.header.corr,
9764 None,
9765 ModuleControlRpcOutcome::MalformedResponse(format!(
9766 "malformed {expected_op} response body: {err}"
9767 )),
9768 )
9769 .map_err(RouterError::Forwarding)?;
9770 if !self.observe_module_control_completion(completion) {
9771 debug!(
9772 connection_id = connection_id.get(),
9773 corr = frame.header.corr,
9774 "dropping late malformed module-control RPC response"
9775 );
9776 }
9777 return Ok(Vec::new());
9778 }
9779 let message = format!("malformed route.bind response body: {err}");
9780 secondary_error = Some(control_error_frame(
9781 &frame,
9782 "invalid_control_body",
9783 message.clone(),
9784 )?);
9785 RouteBindRelayOutcome::ModuleGone(message)
9786 }
9787 },
9788 FrameType::Error => {
9789 if self
9790 .forwarding
9791 .pending_module_control_op(connection_id, frame.header.corr)
9792 .map_err(RouterError::Forwarding)?
9793 .is_some()
9794 {
9795 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
9796 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
9797 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9798 "malformed module-control ERROR body: {err}"
9799 )),
9800 };
9801 let completion = self
9802 .forwarding
9803 .complete_module_control_rpc(
9804 connection_id,
9805 frame.header.corr,
9806 None,
9807 outcome,
9808 )
9809 .map_err(RouterError::Forwarding)?;
9810 if !self.observe_module_control_completion(completion) {
9811 debug!(
9812 connection_id = connection_id.get(),
9813 corr = frame.header.corr,
9814 "dropping late or unknown module-control RPC error"
9815 );
9816 }
9817 return Ok(Vec::new());
9818 }
9819 match serde_json::from_slice::<ErrorBody>(&frame.body) {
9820 Ok(body) => RouteBindRelayOutcome::Rejected(body),
9821 Err(err) => {
9822 let message = format!("malformed route.bind ERROR body: {err}");
9823 secondary_error = Some(control_error_frame(
9824 &frame,
9825 "invalid_control_body",
9826 message.clone(),
9827 )?);
9828 RouteBindRelayOutcome::ModuleGone(message)
9829 }
9830 }
9831 }
9832 ty => {
9833 return Ok(vec![control_error_frame(
9834 &frame,
9835 "unsupported_control_frame",
9836 format!("unsupported module channel-0 frame {ty:?}"),
9837 )?])
9838 }
9839 };
9840
9841 let settled =
9842 self.forwarding
9843 .complete_pending_relay(connection_id, frame.header.corr, outcome);
9844 let completion = match settled {
9845 Ok(completion) => completion,
9846 Err(err) => {
9847 self.refuse_to_end_module_connection_for_a_client(
9848 connection_id,
9849 frame.header.corr,
9850 err,
9851 )?;
9852 return Ok(secondary_error.into_iter().collect());
9853 }
9854 };
9855 if let Some(target) = completion.abandoned.as_ref() {
9856 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
9857 }
9858 if !completion.settled {
9859 debug!(
9860 connection_id = connection_id.get(),
9861 corr = frame.header.corr,
9862 frame_type = ?frame.header.ty,
9863 "dropping late or unknown route.bind relay response"
9864 );
9865 }
9866 Ok(secondary_error.into_iter().collect())
9867 }
9868
9869 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
9870 debug!(connection_id = connection_id.get(), "handling GOODBYE");
9871 self.cleanup_connection_with_end_reason(
9875 connection_id,
9876 RegistrationEndReason::ExplicitGoodbye,
9877 )
9878 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
9879 Ok(Vec::new())
9880 }
9881}
9882
9883impl Default for ControlHandler {
9884 fn default() -> Self {
9885 Self::new(Arc::new(Registry::default()))
9886 }
9887}
9888
9889impl crate::supervise::SwapPromotionObserver for ControlHandler {
9890 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
9891 self.apply_registration_capabilities(registration);
9892 }
9893}
9894
9895fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
9896 CapabilityRequirementStatus {
9897 consumer: status.consumer,
9898 capability: status.capability,
9899 need: match status.need {
9900 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
9901 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
9902 },
9903 verdict: status.verdict.as_str().to_string(),
9904 episode_seq: status.episode_seq,
9905 config_satisfiable: status.config_satisfiable,
9906 runtime_available: status.runtime_available,
9907 detail: status.detail,
9908 }
9909}
9910
9911fn append_capability_problem_detail(
9912 detail: Option<String>,
9913 capability_detail: Option<String>,
9914) -> Option<String> {
9915 match (detail, capability_detail) {
9916 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
9917 (Some(detail), None) => Some(detail),
9918 (None, Some(capability_detail)) => Some(capability_detail),
9919 (None, None) => None,
9920 }
9921}
9922
9923fn subc_ops() -> Vec<String> {
9924 SUBC_CONTROL_OPS
9925 .iter()
9926 .map(|op| (*op).to_string())
9927 .collect()
9928}
9929
9930fn module_subc_ops() -> Vec<String> {
9931 SUBC_CONTROL_OPS
9932 .iter()
9933 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
9934 .map(|op| (*op).to_string())
9935 .collect()
9936}
9937
9938#[cfg(test)]
9939fn module_baseline_control_ops() -> Vec<String> {
9940 MODULE_BASELINE_CONTROL_OPS
9941 .iter()
9942 .map(|op| (*op).to_string())
9943 .collect()
9944}
9945
9946fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
9947 let mut seen = HashSet::new();
9948 let mut effective = Vec::new();
9949 for op in MODULE_BASELINE_CONTROL_OPS {
9950 if seen.insert((*op).to_string()) {
9951 effective.push((*op).to_string());
9952 }
9953 }
9954 for op in declared.unwrap_or_default() {
9955 if seen.insert(op.clone()) {
9956 effective.push(op);
9957 }
9958 }
9959 effective
9960}
9961
9962fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
9963 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
9964}
9965
9966fn target_module_id(target: &RouteTarget) -> &str {
9967 match target {
9968 RouteTarget::ToolProvider { module_id }
9969 | RouteTarget::ManagementSurface { module_id }
9970 | RouteTarget::InternalService { module_id, .. } => module_id,
9971 }
9972}
9973
9974fn target_has_required_role(
9975 target: &RouteTarget,
9976 roles: &[crate::registry::AdmissionRole],
9977) -> bool {
9978 use crate::registry::AdmissionRole;
9979 roles.iter().any(|role| match (target, role) {
9980 (RouteTarget::ToolProvider { .. }, AdmissionRole::ToolProvider) => true,
9981 (RouteTarget::ManagementSurface { .. }, AdmissionRole::ManagementSurface) => true,
9982 (
9983 RouteTarget::InternalService { service_id, .. },
9984 AdmissionRole::InternalService(provided),
9985 ) => service_id == provided,
9986 _ => false,
9987 })
9988}
9989
9990fn is_routable_role(role: &ProviderRole) -> bool {
9991 matches!(
9992 role,
9993 ProviderRole::ToolProvider { .. }
9994 | ProviderRole::ManagementSurface { .. }
9995 | ProviderRole::InternalService { .. }
9996 )
9997}
9998
9999#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10000enum ControlRequestBodyError {
10001 UnknownOp,
10002 InvalidBody,
10003}
10004
10005#[derive(Debug, Deserialize)]
10006struct ControlOpProbe {
10007 op: String,
10008}
10009
10010fn module_control_response_probe(body: &[u8]) -> Result<ControlOpProbe, serde_json::Error> {
10011 serde_json::from_slice(body).or_else(|original| {
10012 #[derive(Deserialize)]
10013 struct ConsentProbe {
10014 protocol: String,
10015 }
10016 match serde_json::from_slice::<ConsentProbe>(body) {
10017 Ok(probe) if probe.protocol == subc_protocol::consent::CONSENT_V1 => {
10018 Ok(ControlOpProbe { op: probe.protocol })
10019 }
10020 _ => Err(original),
10021 }
10022 })
10023}
10024
10025const MODULE_PUSH_OPS: &[&str] = &["route.status"];
10028
10029fn is_known_module_push_op(body: &[u8]) -> bool {
10030 serde_json::from_slice::<ControlOpProbe>(body)
10031 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
10032 .unwrap_or(false)
10033}
10034
10035fn is_known_module_request_op(body: &[u8]) -> bool {
10036 serde_json::from_slice::<ControlOpProbe>(body)
10037 .map(|probe| is_module_to_subc_op(&probe.op))
10038 .unwrap_or(false)
10039}
10040
10041fn is_module_to_subc_op(op: &str) -> bool {
10042 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
10043}
10044
10045fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
10046 debug!(
10047 op = %op,
10048 connection_id = connection_id.get(),
10049 corr,
10050 "control dispatch"
10051 );
10052}
10053
10054fn log_slow_control_dispatch(
10055 dispatch_started_at: Option<StdInstant>,
10056 op: &'static str,
10057 connection_id: ConnectionId,
10058 corr: u64,
10059 route_open_timings: Option<&RouteOpenTimings>,
10060) {
10061 let Some(dispatch_started_at) = dispatch_started_at else {
10062 return;
10063 };
10064 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
10066 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
10067 if let Some(timings) = route_open_timings {
10068 warn!(
10069 op = %op,
10070 connection_id = connection_id.get(),
10071 corr,
10072 elapsed_ms = elapsed.as_millis() as u64,
10073 module_id = ?timings.module_id,
10074 principal = %timings.principal,
10075 admission_ms = timings.admission.as_millis() as u64,
10076 project_root_ms = timings.project_root.as_millis() as u64,
10077 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
10078 bind_send_ms = timings.bind_send.as_millis() as u64,
10079 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
10080 "slow control dispatch"
10081 );
10082 return;
10083 }
10084 warn!(
10085 op = %op,
10086 connection_id = connection_id.get(),
10087 corr,
10088 elapsed_ms = elapsed.as_millis() as u64,
10089 "slow control dispatch"
10090 );
10091 }
10092}
10093
10094#[cfg(test)]
10096type TestCallerLookup =
10097 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
10098
10099fn privileged_operation(request: &impl PrivilegedRequest) -> Option<PrivilegedOperation<'_>> {
10101 request.privileged_operation()
10102}
10103
10104trait PrivilegedRequest {
10105 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>>;
10106}
10107
10108impl PrivilegedRequest for ClientControlRequest {
10109 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>> {
10110 let request = self;
10111 match request {
10112 ClientControlRequest::SupervisorRestart { module_id, .. } => {
10113 Some(PrivilegedOperation::Restart { module_id })
10114 }
10115 ClientControlRequest::SupervisorSwap { module_id, .. } => {
10116 Some(PrivilegedOperation::Swap { module_id })
10117 }
10118 ClientControlRequest::SupervisorReload { module_id } => {
10119 Some(PrivilegedOperation::Reload { module_id })
10120 }
10121 ClientControlRequest::SupervisorRescan { preview } => {
10122 if *preview {
10123 None
10124 } else {
10125 Some(PrivilegedOperation::Rescan)
10126 }
10127 }
10128 ClientControlRequest::SupervisorRescanChecked {
10133 preview,
10134 candidate,
10135 expect,
10136 } => (!*preview && candidate.is_none() && expect.is_some())
10137 .then_some(PrivilegedOperation::Rescan),
10138 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
10139 Some(PrivilegedOperation::SetEnabled {
10140 module_id,
10141 enabled: *enabled,
10142 })
10143 }
10144 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
10145 Some(PrivilegedOperation::ReleaseReserved { module_id })
10146 }
10147 ClientControlRequest::ServerDescribe {}
10148 | ClientControlRequest::DaemonShutdownIntent { .. }
10151 | ClientControlRequest::CatalogList { .. }
10152 | ClientControlRequest::RouteOpen { .. }
10153 | ClientControlRequest::RoutePoll { .. }
10154 | ClientControlRequest::SupervisorList {}
10155 | ClientControlRequest::SupervisorHealth {}
10156 | ClientControlRequest::SupervisorHealthProbe { .. }
10157 | ClientControlRequest::SupervisorRoutes { .. }
10158 | ClientControlRequest::SupervisorProvenance { .. }
10159 | ClientControlRequest::SupervisorStderrTail { .. }
10160 | ClientControlRequest::SupervisorTerminals { .. }
10161 | ClientControlRequest::SupervisorSpawnSnapshot {}
10162 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
10163 }
10164 }
10165}
10166
10167fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
10168 match request {
10169 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
10170 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
10171 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
10172 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
10173 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
10174 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
10175 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
10176 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
10177 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
10178 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
10179 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
10180 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
10181 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
10182 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
10183 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
10184 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
10185 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
10186 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
10187 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
10188 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
10189 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
10190 }
10191}
10192
10193struct RouteOpenCaller {
10201 principal: String,
10205 scope: String,
10208 harness: String,
10211}
10212
10213impl RouteOpenCaller {
10214 fn new(
10215 principal: Option<&Principal>,
10216 scope: Option<&ScopeSelector>,
10217 identity: &BindIdentity,
10218 ) -> Self {
10219 Self {
10220 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
10221 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
10222 harness: bounded_log_text(&identity.harness),
10223 }
10224 }
10225
10226 fn unknown() -> Self {
10229 Self {
10230 principal: "unknown".to_string(),
10231 scope: "unknown".to_string(),
10232 harness: "unknown".to_string(),
10233 }
10234 }
10235}
10236
10237const MAX_LOGGED_TEXT_CHARS: usize = 64;
10240
10241fn bounded_log_text(value: &str) -> String {
10245 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
10246 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
10247 bounded.push('…');
10248 }
10249 format!("{bounded:?}")
10250}
10251
10252fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
10258 use subc_protocol::scope::ScopeKind;
10259 let kind = match stamp.kind {
10260 ScopeKind::Head => "head",
10261 ScopeKind::Worker => "worker",
10262 ScopeKind::Ephemeral => "ephemeral",
10263 ScopeKind::Federated => "federated",
10264 _ => "unknown",
10265 };
10266 format!(
10267 "{kind}/{}/{}@{}",
10268 principal_label(&stamp.owner),
10269 bounded_log_text(&stamp.scope_ref),
10270 stamp.scope_epoch
10271 )
10272}
10273
10274fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
10281 let epoch = selector
10282 .scope_epoch
10283 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
10284 format!(
10285 "requested/{}/{}@{epoch}",
10286 bounded_log_text(&principal_label(&selector.owner)),
10287 bounded_log_text(&selector.scope_ref),
10288 )
10289}
10290
10291fn principal_label(principal: &Principal) -> String {
10292 match principal {
10293 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
10294 Principal::Direct => "direct".to_string(),
10295 other => format!("{other:?}"),
10296 }
10297}
10298
10299fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
10300 match request {
10301 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
10302 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
10303 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
10304 }
10305 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
10306 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
10307 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
10308 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
10309 ModuleControlRequestFromModule::RouteAuthorize { .. } => {
10310 subc_protocol::scope::ROUTE_AUTHORIZE_OP
10311 }
10312 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
10313 _ => "unsupported",
10314 }
10315}
10316
10317fn parse_client_control_request(
10318 body: &[u8],
10319) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
10320 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
10321 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10322 Ok(probe)
10323 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
10324 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
10325 {
10326 ControlRequestBodyError::InvalidBody
10327 }
10328 Ok(_) => ControlRequestBodyError::UnknownOp,
10329 Err(_) => ControlRequestBodyError::InvalidBody,
10330 };
10331 (err, classification)
10332 })
10333}
10334
10335fn parse_module_control_request_from_module(
10336 body: &[u8],
10337) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
10338 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
10339 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10340 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
10341 Ok(_) => ControlRequestBodyError::UnknownOp,
10342 Err(_) => ControlRequestBodyError::InvalidBody,
10343 };
10344 (err, classification)
10345 })
10346}
10347
10348#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
10349enum ProviderRoleKind {
10350 ToolProvider,
10351 PipelineStage,
10352 ManagementSurface,
10353 InternalService,
10354}
10355
10356fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
10357 match role {
10358 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
10359 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
10360 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
10361 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
10362 }
10363}
10364
10365fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
10366 roles.iter().map(provider_role_kind).collect()
10367}
10368
10369const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
10372
10373#[derive(Debug, Default, PartialEq, Eq)]
10375struct ScopeOutcomeCounts {
10376 created: usize,
10377 replaced: usize,
10378 updated: usize,
10379 unchanged: usize,
10380 refused: usize,
10381}
10382
10383impl ScopeOutcomeCounts {
10384 fn of(results: &[ScopeRecordResult]) -> Self {
10385 let mut counts = Self::default();
10386 for result in results {
10387 let slot = match result.outcome {
10388 ScopeRecordOutcome::Created => &mut counts.created,
10389 ScopeRecordOutcome::Replaced => &mut counts.replaced,
10390 ScopeRecordOutcome::Updated => &mut counts.updated,
10391 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
10392 ScopeRecordOutcome::Refused => &mut counts.refused,
10393 };
10394 *slot += 1;
10395 }
10396 counts
10397 }
10398}
10399
10400#[cfg(test)]
10401mod scope_outcome_count_tests {
10402 use super::*;
10403
10404 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
10405 ScopeRecordResult {
10406 scope_ref: "r".to_string(),
10407 scope_epoch: 1,
10408 outcome,
10409 code: None,
10410 message: None,
10411 version: None,
10412 parent_state: None,
10413 }
10414 }
10415
10416 #[test]
10419 fn every_outcome_is_counted_in_its_own_field() {
10420 let results = [
10421 result(ScopeRecordOutcome::Created),
10422 result(ScopeRecordOutcome::Created),
10423 result(ScopeRecordOutcome::Replaced),
10424 result(ScopeRecordOutcome::Updated),
10425 result(ScopeRecordOutcome::Unchanged),
10426 result(ScopeRecordOutcome::Refused),
10427 result(ScopeRecordOutcome::Refused),
10428 result(ScopeRecordOutcome::Refused),
10429 ];
10430 assert_eq!(
10431 ScopeOutcomeCounts::of(&results),
10432 ScopeOutcomeCounts {
10433 created: 2,
10434 replaced: 1,
10435 updated: 1,
10436 unchanged: 1,
10437 refused: 3,
10438 }
10439 );
10440 }
10441}
10442
10443fn capability_census_trigger(
10447 old: Option<&CapabilityDeclarations>,
10448 new: Option<&CapabilityDeclarations>,
10449) -> bool {
10450 let old_provides = old
10451 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
10452 .unwrap_or_default();
10453 let old_denies = old
10454 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
10455 .unwrap_or_default();
10456 let new = new.cloned().unwrap_or(CapabilityDeclarations {
10457 provides: Vec::new(),
10458 requires: Vec::new(),
10459 must_never_reach: Vec::new(),
10460 });
10461
10462 new.provides
10463 .iter()
10464 .any(|capability| !old_provides.contains(capability))
10465 || new
10466 .must_never_reach
10467 .iter()
10468 .any(|capability| !old_denies.contains(capability))
10469}
10470
10471fn catalog_update_frozen_field_message(
10472 registered: &ModuleManifest,
10473 provides: &[ProviderRole],
10474) -> Option<String> {
10475 let old_has_provides = !registered.provides.is_empty();
10476 let new_has_provides = !provides.is_empty();
10477 if old_has_provides != new_has_provides {
10478 return Some(format!(
10479 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
10480 registered.module_id
10481 ));
10482 }
10483
10484 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
10485 return Some(format!(
10486 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
10487 registered.module_id
10488 ));
10489 }
10490
10491 let registered_concurrency = manifest_concurrency(registered);
10492 let mut candidate = registered.clone();
10493 candidate.provides = provides.to_vec();
10494 let candidate_concurrency = manifest_concurrency(&candidate);
10495 if candidate_concurrency != registered_concurrency {
10496 return Some(format!(
10497 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
10498 registered.module_id, registered_concurrency, candidate_concurrency
10499 ));
10500 }
10501
10502 None
10505}
10506
10507fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
10508 manifest.provides.iter().any(is_routable_role)
10509}
10510
10511fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
10517 manifest
10518 .provides
10519 .iter()
10520 .find_map(|provider| match provider {
10521 ProviderRole::ToolProvider { concurrency, .. }
10522 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
10523 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
10524 })
10525 .unwrap_or(Concurrency::ModuleManaged)
10526}
10527
10528fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
10535 let has_management_surface = manifest
10536 .provides
10537 .iter()
10538 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
10539 if !has_management_surface {
10540 return false;
10541 }
10542 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
10543 return false;
10544 };
10545 let Some(provides) = raw
10546 .get("manifest")
10547 .and_then(|manifest| manifest.get("provides"))
10548 .and_then(serde_json::Value::as_array)
10549 else {
10550 return false;
10551 };
10552 provides.iter().any(|role| {
10557 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
10558 && role.get("concurrency").is_none()
10559 })
10560}
10561
10562fn negotiate_version(peer_version: u8) -> Result<u8, String> {
10563 if peer_version != PROTOCOL_VERSION {
10564 return Err(format!(
10565 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
10566 ));
10567 }
10568 Ok(PROTOCOL_VERSION)
10569}
10570
10571fn pong(frame: &Frame) -> Result<Frame, RouterError> {
10572 Frame::build_with_version(
10573 response_version(frame),
10574 FrameType::Pong,
10575 frame.header.flags,
10576 0,
10577 0,
10578 frame.header.corr,
10579 Vec::new(),
10580 )
10581 .map_err(RouterError::FrameBuild)
10582}
10583
10584pub(crate) fn control_error_frame(
10585 frame: &Frame,
10586 code: &'static str,
10587 message: impl Into<String>,
10588) -> Result<Frame, RouterError> {
10589 control_error_body_frame(
10590 frame,
10591 ErrorBody {
10592 code: code.to_string(),
10593 message: message.into(),
10594 detail: None,
10595 },
10596 )
10597}
10598
10599pub(crate) fn control_error_body_frame(
10607 frame: &Frame,
10608 error: ErrorBody,
10609) -> Result<Frame, RouterError> {
10610 control_error_frame_with_flags(frame, error, daemon_error_flags())
10611}
10612
10613fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
10619 control_error_frame_with_flags(frame, error, control_flags())
10620}
10621
10622fn control_error_frame_with_flags(
10623 frame: &Frame,
10624 error: ErrorBody,
10625 flags: Flags,
10626) -> Result<Frame, RouterError> {
10627 let body = serde_json::to_vec(&error).map_err(|err| {
10628 RouterError::backend(
10629 0,
10630 frame.header.corr,
10631 format!("failed to encode control ERROR: {err}"),
10632 )
10633 })?;
10634
10635 Frame::build_with_version(
10636 response_version(frame),
10637 FrameType::Error,
10638 flags,
10639 0,
10640 0,
10641 frame.header.corr,
10642 body,
10643 )
10644 .map_err(RouterError::FrameBuild)
10645}
10646
10647pub(crate) fn control_response_body_frame<T: Serialize>(
10648 frame: &Frame,
10649 reply: &T,
10650 label: &'static str,
10651) -> Result<Frame, RouterError> {
10652 let body = serde_json::to_vec(reply).map_err(|err| {
10653 RouterError::backend(
10654 0,
10655 frame.header.corr,
10656 format!("failed to encode {label}: {err}"),
10657 )
10658 })?;
10659
10660 Frame::build_with_version(
10661 response_version(frame),
10662 FrameType::Response,
10663 control_flags(),
10664 0,
10665 0,
10666 frame.header.corr,
10667 body,
10668 )
10669 .map_err(RouterError::FrameBuild)
10670}
10671
10672struct HelloAuthority {
10705 record: Arc<crate::spawn_authority::SpawnAuthorityRecord>,
10706 requested: ProviderDeclarations,
10707 froze: std::sync::atomic::AtomicBool,
10710}
10711
10712fn forwarding_error_code(err: &ForwardingError) -> &'static str {
10713 match err {
10714 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
10715 ForwardingError::NoModuleConnection => "target_unavailable",
10716 ForwardingError::ModuleReloading { .. } => "module_reloading",
10717 ForwardingError::ClientRouteChannelExhausted { .. }
10718 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
10719 ForwardingError::StaleModuleEndpoint
10720 | ForwardingError::UnknownReservation { .. }
10721 | ForwardingError::ConnectionClosing { .. }
10722 | ForwardingError::ClientEgressClosed { .. }
10723 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
10724 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
10727 ForwardingError::RelayCorrelationExhausted
10728 | ForwardingError::RouteOpenBuild(_)
10729 | ForwardingError::Poisoned => "forwarding_error",
10730 }
10731}
10732
10733fn response_version(frame: &Frame) -> u8 {
10734 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
10735 frame.header.ver
10736 } else {
10737 PROTOCOL_VERSION
10738 }
10739}
10740
10741pub(crate) fn control_flags() -> Flags {
10742 Flags::new(false, Priority::Passive, false)
10743}
10744
10745fn send_goodbye_target_best_effort(
10750 counters: &DaemonCounters,
10751 target: &GoodbyeTarget,
10752 context: &'static str,
10753) {
10754 let Ok(frame) = Frame::build_with_version(
10755 target.negotiated_ver,
10756 FrameType::Goodbye,
10757 control_flags(),
10758 target.channel,
10759 target.epoch,
10760 0,
10761 Vec::new(),
10762 ) else {
10763 return;
10764 };
10765 crate::forwarding::send_module_route_goodbye(
10766 counters,
10767 &target.sink,
10768 frame,
10769 target.module_id.as_deref(),
10770 context,
10771 );
10772}
10773
10774fn route_close_reason_for_opener(
10775 reason: RouteCloseReason,
10776 accepts_close_reasons: Option<&[String]>,
10777) -> RouteCloseReason {
10778 if reason == RouteCloseReason::AuthorityChanged
10779 && !accepts_close_reasons
10780 .is_some_and(|reasons| reasons.iter().any(|reason| reason == "authority_changed"))
10781 {
10782 RouteCloseReason::Reload
10785 } else {
10786 reason
10787 }
10788}
10789
10790pub(crate) fn send_route_control_pushes(
10791 forwarding: &ForwardingTable,
10792 routes: Vec<EndpointRoute>,
10793 push: ClientControlPush,
10794) {
10795 let reason = match &push {
10796 ClientControlPush::RouteClosing { reason, .. }
10797 | ClientControlPush::RouteClosed { reason, .. } => *reason,
10798 };
10799 let mut targets: Vec<(GoodbyeTarget, RouteCloseReason, Vec<u16>)> = Vec::new();
10800 for route in routes {
10801 let recipient_reason =
10802 route_close_reason_for_opener(reason, route.accepts_close_reasons.as_deref());
10803 let target = route.goodbye_target;
10804 if let Some((existing, _, channels)) = targets.iter_mut().find(|(existing, reason, _)| {
10805 existing.connection_id == target.connection_id && *reason == recipient_reason
10806 }) {
10807 debug_assert_eq!(
10808 existing.negotiated_ver, target.negotiated_ver,
10809 "one connection cannot negotiate multiple frame versions"
10810 );
10811 if !channels.contains(&target.channel) {
10812 channels.push(target.channel);
10813 }
10814 continue;
10815 }
10816 let channel = target.channel;
10817 targets.push((target, recipient_reason, vec![channel]));
10818 }
10819 for (target, recipient_reason, mut channels) in targets {
10820 channels.sort_unstable();
10821 let mut push = push.clone();
10822 match &mut push {
10823 ClientControlPush::RouteClosing {
10824 channels: covered,
10825 reason,
10826 ..
10827 }
10828 | ClientControlPush::RouteClosed {
10829 channels: covered,
10830 reason,
10831 ..
10832 } => {
10833 *covered = channels;
10834 *reason = recipient_reason;
10835 }
10836 }
10837 let body = match serde_json::to_vec(&push) {
10838 Ok(body) => body,
10839 Err(err) => {
10840 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
10841 continue;
10842 }
10843 };
10844 let frame = match Frame::build_with_version(
10845 target.negotiated_ver,
10846 FrameType::Push,
10847 control_flags(),
10848 0,
10849 0,
10850 0,
10851 body.clone(),
10852 ) {
10853 Ok(frame) => frame,
10854 Err(err) => {
10855 warn!(
10856 route_channel = target.channel,
10857 error = %err,
10858 "failed to build route lifecycle control PUSH frame"
10859 );
10860 continue;
10861 }
10862 };
10863 if let Err(err) = target.sink.try_send(frame) {
10864 if target.close_on_delivery_failure() {
10865 warn!(
10866 target_connection_id = target.connection_id.get(),
10867 route_channel = target.channel,
10868 error = %err,
10869 "route lifecycle control PUSH was not delivered to client; closing target connection"
10870 );
10871 let _ = forwarding.escalate_client_delivery_failure(
10872 target.connection_id,
10873 target.channel,
10874 target.epoch,
10875 CloseReason::new(
10876 "route_lifecycle_push_delivery_failed",
10877 format!(
10878 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
10879 target.channel
10880 ),
10881 ),
10882 crate::forwarding::UndeliveredFrame {
10883 module_id: target.module_id.as_deref(),
10884 sink: &target.sink,
10885 },
10886 );
10887 }
10888 }
10889 }
10890}
10891
10892#[cfg(test)]
10893mod tests {
10894 include!("control/agent_commands_tests.rs");
10895 include!("control/resource_revision_tests.rs");
10896 include!("control/route_open_recovery_tests.rs");
10897 #[tokio::test]
10898 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
10899 let registry = Arc::new(Registry::default());
10900 let handler = ControlHandler::new(registry);
10901 assert_eq!(
10902 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10903 None
10904 );
10905 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
10906 hello_via_sink(
10907 &handler,
10908 &ctx,
10909 &mut rx,
10910 hello_frame("daemon", PROTOCOL_VERSION, 1),
10911 )
10912 .await;
10913 assert_eq!(
10914 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10915 Some("daemon".into())
10916 );
10917 assert_eq!(
10918 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
10919 None
10920 );
10921 assert_eq!(
10922 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
10923 None,
10924 "a registered module is not necessarily supervised"
10925 );
10926 assert_eq!(
10927 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
10928 None,
10929 "aggregate provenance has no single target"
10930 );
10931 assert_eq!(
10932 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
10933 None
10934 );
10935 }
10936
10937 mod peer_manifest {
10938 include!("peer_manifest_tests.rs");
10939 }
10940 mod catalog_declarations {
10941 include!("control/catalog_tests.rs");
10942 }
10943 mod activity_read {
10944 include!("activity_read_tests.rs");
10945 }
10946
10947 mod lifecycle_activity {
10948 include!("control_lifecycle_tests.rs");
10949 }
10950 #[cfg(unix)]
10951 #[tokio::test]
10952 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
10953 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
10954 let path = dir.join("subc.jsonc");
10955 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
10956 "program":"/bin/sleep","args":["60"],"protocol":"none",
10957 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
10958 }}}).to_string()).unwrap();
10959 let mut configured = crate::daemon_config::load(&path)
10960 .unwrap()
10961 .unwrap()
10962 .modules
10963 .pop()
10964 .unwrap();
10965 let registry = std::sync::Arc::new(crate::Registry::default());
10966 let handle = crate::SupervisorHandle::new();
10967 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
10968 .with_handle(handle.clone());
10969 let module = supervisor
10970 .supervise_configured_with_health(
10971 configured.module_spec(),
10972 true,
10973 configured.health.clone(),
10974 None,
10975 configured.restart,
10976 )
10977 .unwrap();
10978 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
10979 let before = module.status().unwrap().pid;
10980 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
10981 configured.health.cadence = std::time::Duration::from_secs(3600);
10982 let health_only = handler
10983 .reconcile_supervised_modules(
10984 &supervisor,
10985 vec![configured.clone()],
10986 false,
10987 Principal::Direct,
10988 )
10989 .await
10990 .unwrap();
10991 assert!(
10992 health_only.0.changed_pending_reload.is_empty(),
10993 "health policy is already applied live"
10994 );
10995 assert_eq!(module.status().unwrap().pid, before);
10996 assert_eq!(
10997 module.configuration().unwrap().1.http,
10998 configured.health.http
10999 );
11000 configured.args = vec!["61".into()];
11001 let launch = handler
11002 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
11003 .await
11004 .unwrap();
11005 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
11006 assert_eq!(
11007 module.status().unwrap().pid,
11008 before,
11009 "a launch edit is stored until reload"
11010 );
11011 module.drain().await.unwrap();
11012 }
11013 use cortexkit_test_support::ScratchDir;
11014 use std::{
11015 collections::BTreeMap,
11016 fmt,
11017 path::PathBuf,
11018 sync::{Arc, Mutex},
11019 time::Duration,
11020 };
11021
11022 use serde_json::{json, Value};
11023 use subc_protocol::{
11024 manifest::{
11025 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
11026 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
11027 },
11028 session::HealthStatus,
11029 FrameType,
11030 };
11031
11032 use super::*;
11033 use crate::{
11034 forwarding::{DataRoute, DataRouteState},
11035 registry::ChannelState,
11036 router::FrameSink,
11037 stderr_tail::DEFAULT_MAX_LINE_BYTES,
11038 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
11039 RouteCtx, Router,
11040 };
11041 use tokio::{
11042 sync::mpsc,
11043 time::{sleep, Instant},
11044 };
11045 use tracing::{
11046 field::{Field, Visit},
11047 Event, Subscriber,
11048 };
11049 use tracing_subscriber::{layer::Context, prelude::*, Layer};
11050
11051 fn fake_aft_stub_path() -> PathBuf {
11070 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
11071 assert!(
11072 path.exists(),
11073 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11074 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11075 path.display()
11076 );
11077 path
11078 }
11079
11080 fn client_retries(code: &str) -> bool {
11085 subc_protocol::error_codes::is_retryable_route_open(code)
11086 }
11087
11088 #[test]
11099 fn retryability_of_forwarding_codes_matches_the_failure() {
11100 let transient = [
11103 ForwardingError::NoModuleConnection,
11104 ForwardingError::ModuleReloading {
11105 module_id: "m".into(),
11106 },
11107 ForwardingError::StaleModuleEndpoint,
11108 ForwardingError::UnknownReservation {
11109 client_channel: 1,
11110 module_channel: 1,
11111 },
11112 ForwardingError::ConnectionClosing {
11113 connection_id: ConnectionId::new(1),
11114 },
11115 ForwardingError::ClientEgressClosed {
11116 connection_id: ConnectionId::new(1),
11117 },
11118 ForwardingError::ModuleEgressUnavailable {
11119 connection_id: ConnectionId::new(1),
11120 },
11121 ];
11122 for err in transient {
11123 let code = forwarding_error_code(&err);
11124 assert!(
11125 client_retries(code),
11126 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
11127 );
11128 }
11129
11130 let permanent = [
11135 ForwardingError::ConnectionRoleConflict {
11136 connection_id: ConnectionId::new(1),
11137 },
11138 ForwardingError::ClientRouteChannelExhausted {
11139 connection_id: ConnectionId::new(1),
11140 },
11141 ForwardingError::ModuleRouteChannelExhausted {
11142 endpoint: ModuleEndpointId {
11143 connection_id: ConnectionId::new(1),
11144 generation: 1,
11145 },
11146 },
11147 ForwardingError::RelayCorrelationExhausted,
11148 ForwardingError::RouteOpenBuild("x".into()),
11149 ForwardingError::Poisoned,
11150 ];
11151 for err in permanent {
11152 let code = forwarding_error_code(&err);
11153 assert!(
11154 !client_retries(code),
11155 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
11156 );
11157 }
11158 }
11159
11160 #[tokio::test]
11188 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
11189 let handler = ControlHandler::default();
11190 let frame =
11191 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
11192
11193 let stamped = handler
11197 .route_open_principal(ConnectionId::new(1), &frame, None)
11198 .unwrap()
11199 .unwrap();
11200 assert_eq!(
11201 stamped,
11202 Principal::Direct,
11203 "a caller that proved nothing must not be stamped as a supervised module"
11204 );
11205
11206 let forged = handler
11211 .route_open_principal(
11212 ConnectionId::new(1),
11213 &frame,
11214 Some(ConsumerIdentity {
11215 module_id: "aft".to_string(),
11216 launch_nonce: "not-a-real-nonce".to_string(),
11217 }),
11218 )
11219 .unwrap();
11220 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
11221 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
11222 }
11223
11224 #[test]
11241 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
11242 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
11243 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
11244 let ClientControlRequest::RouteOpen {
11245 consumer_identity, ..
11246 } = parsed
11247 else {
11248 panic!("route.open body must parse as RouteOpen");
11249 };
11250 assert_eq!(
11251 consumer_identity,
11252 Some(ConsumerIdentity {
11253 module_id: "aft".to_string(),
11254 launch_nonce: "n".to_string(),
11255 }),
11256 "the wire field name must reach the value route_open_principal reads"
11257 );
11258 }
11259
11260 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
11261 ModuleManifest::builder(module_id, "0.1.0")
11262 .protocol_ver(protocol_ver)
11263 .provides(vec![ProviderRole::ToolProvider {
11264 tools: vec![Tool {
11265 name: "read".to_string(),
11266 description: None,
11267 execution_mode: ExecutionMode::Pure,
11268 schema: json!({"type": "object"}),
11269 }],
11270 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
11271 concurrency: Concurrency::ModuleManaged,
11272 emits_push: true,
11273 sub_supervises: true,
11274 }])
11275 .build()
11276 }
11277
11278 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
11279 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
11280 }
11281
11282 fn hello_frame_with_control_ops(
11283 module_id: &str,
11284 protocol_ver: u8,
11285 corr: u64,
11286 control_ops: Option<Vec<String>>,
11287 ) -> Frame {
11288 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
11289 }
11290
11291 fn hello_frame_with_nonce(
11292 module_id: &str,
11293 protocol_ver: u8,
11294 corr: u64,
11295 launch_nonce: Option<&str>,
11296 ) -> Frame {
11297 hello_frame_full(
11298 module_id,
11299 protocol_ver,
11300 corr,
11301 None,
11302 launch_nonce.map(ToOwned::to_owned),
11303 )
11304 }
11305
11306 fn hello_frame_full(
11307 module_id: &str,
11308 protocol_ver: u8,
11309 corr: u64,
11310 control_ops: Option<Vec<String>>,
11311 launch_nonce: Option<String>,
11312 ) -> Frame {
11313 let body = serde_json::to_vec(&ModuleHelloBody {
11314 manifest: manifest(module_id, protocol_ver),
11315 protocol_ver,
11316 control_ops,
11317 launch_nonce,
11318 })
11319 .unwrap();
11320 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11321 }
11322
11323 fn hello_frame_with_rich_ui(
11324 module_id: &str,
11325 protocol_ver: u8,
11326 corr: u64,
11327 launch_nonce: Option<&str>,
11328 rich_ui: Value,
11329 ) -> Frame {
11330 let mut body = serde_json::to_value(ModuleHelloBody {
11331 manifest: manifest(module_id, protocol_ver),
11332 protocol_ver,
11333 control_ops: None,
11334 launch_nonce: launch_nonce.map(str::to_string),
11335 })
11336 .unwrap();
11337 body["manifest"]["rich_ui"] = rich_ui;
11338 Frame::build(
11339 FrameType::Hello,
11340 control_flags(),
11341 0,
11342 0,
11343 corr,
11344 serde_json::to_vec(&body).unwrap(),
11345 )
11346 .unwrap()
11347 }
11348
11349 fn non_routable_hello_frame_with_control_ops(
11350 module_id: &str,
11351 corr: u64,
11352 control_ops: Option<Vec<String>>,
11353 ) -> Frame {
11354 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11355 manifest.provides.clear();
11356 let body = serde_json::to_vec(&ModuleHelloBody {
11357 manifest,
11358 protocol_ver: PROTOCOL_VERSION,
11359 control_ops,
11360 launch_nonce: None,
11361 })
11362 .unwrap();
11363 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11364 }
11365
11366 fn capability_grammar_hello_frame(
11367 capabilities: Value,
11368 runtime_computed: Option<Value>,
11369 corr: u64,
11370 ) -> Frame {
11371 let mut body = serde_json::to_value(ModuleHelloBody {
11372 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
11373 protocol_ver: PROTOCOL_VERSION,
11374 control_ops: None,
11375 launch_nonce: None,
11376 })
11377 .expect("HELLO body serializes");
11378 body["manifest"]["capabilities"] = capabilities;
11379 if let Some(runtime_computed) = runtime_computed {
11380 body["runtime_computed"] = runtime_computed;
11381 }
11382 Frame::build(
11383 FrameType::Hello,
11384 control_flags(),
11385 0,
11386 0,
11387 corr,
11388 serde_json::to_vec(&body).expect("HELLO body reserializes"),
11389 )
11390 .expect("HELLO frame builds")
11391 }
11392
11393 fn channel_request(channel: u16, corr: u64) -> Frame {
11394 Frame::build(
11395 FrameType::Request,
11396 Flags::new(true, Priority::Interactive, false),
11397 channel,
11398 0,
11399 corr,
11400 b"opaque".to_vec(),
11401 )
11402 .unwrap()
11403 }
11404
11405 fn route_ctx(
11406 connection_id: ConnectionId,
11407 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11408 let (tx, rx) = mpsc::channel(8);
11409 (
11410 RouteCtx {
11411 connection_id,
11412 egress: FrameSink::new(tx),
11413 },
11414 rx,
11415 )
11416 }
11417
11418 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
11419 serde_json::from_slice(&frame.body).unwrap()
11420 }
11421
11422 async fn hello_via_sink(
11427 handler: &ControlHandler,
11428 ctx: &RouteCtx,
11429 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11430 hello: Frame,
11431 ) -> Frame {
11432 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
11433 assert!(
11434 replies.is_empty(),
11435 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
11436 );
11437 let ack = rx
11438 .try_recv()
11439 .expect("HELLO_ACK is queued on the module sink")
11440 .frame;
11441 assert_eq!(ack.header.ty, FrameType::HelloAck);
11442 ack
11443 }
11444
11445 fn parse_error(frame: &Frame) -> Value {
11446 serde_json::from_slice(&frame.body).unwrap()
11447 }
11448
11449 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
11450 serde_json::from_slice(&frame.body).unwrap()
11451 }
11452
11453 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
11454 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
11455 route_channel,
11456 route_epoch: 0,
11457 kind,
11458 })
11459 .unwrap();
11460 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11461 }
11462
11463 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
11464 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
11465 module_id: module_id.to_string(),
11466 })
11467 .unwrap();
11468 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11469 }
11470
11471 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
11472 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
11473 }
11474
11475 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
11477 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
11478 let mut frame = route_open_frame(corr, module_id, root);
11479 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
11480 body["identity"]["project_root"] = serde_json::json!(path.as_path());
11481 frame.body = serde_json::to_vec(&body).unwrap();
11482 frame
11483 }
11484
11485 fn route_open_frame_with_consumer_capabilities(
11486 corr: u64,
11487 module_id: &str,
11488 _project_root: ScratchDir,
11489 consumer_capabilities: Option<Vec<String>>,
11490 ) -> Frame {
11491 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11492 busy_refusals: None,
11493 target: RouteTarget::ToolProvider {
11494 module_id: module_id.to_string(),
11495 },
11496 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
11497 consumer_identity: None,
11498 consumer_capabilities,
11499 role_versions: None,
11500 admission_facts: None,
11501 scope: None,
11502 })
11503 .unwrap();
11504 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11505 }
11506
11507 fn route_open_frame_with_role_versions(
11508 corr: u64,
11509 module_id: &str,
11510 _project_root: ScratchDir,
11511 role_versions: Option<BTreeMap<String, String>>,
11512 ) -> Frame {
11513 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11514 busy_refusals: None,
11515 target: RouteTarget::ToolProvider {
11516 module_id: module_id.to_string(),
11517 },
11518 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11519 consumer_identity: None,
11520 consumer_capabilities: None,
11521 role_versions,
11522 admission_facts: None,
11523 scope: None,
11524 })
11525 .unwrap();
11526 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11527 }
11528
11529 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
11530 entries
11531 .iter()
11532 .map(|(role, version)| (role.to_string(), version.to_string()))
11533 .collect()
11534 }
11535
11536 fn route_open_frame_with_admission_facts(
11537 corr: u64,
11538 module_id: &str,
11539 _project_root: ScratchDir,
11540 consumer_identity: Option<subc_control::ConsumerIdentity>,
11541 facts: Option<Value>,
11542 ) -> Frame {
11543 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11544 busy_refusals: None,
11545 target: RouteTarget::ToolProvider {
11546 module_id: module_id.to_string(),
11547 },
11548 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11549 consumer_identity,
11550 consumer_capabilities: None,
11551 role_versions: None,
11552 admission_facts: facts,
11553 scope: None,
11554 })
11555 .unwrap();
11556 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11557 }
11558
11559 #[derive(Clone, Default)]
11560 struct EventCapture {
11561 events: Arc<Mutex<Vec<CapturedEvent>>>,
11562 }
11563
11564 #[derive(Clone, Debug)]
11565 struct CapturedEvent {
11566 target: String,
11567 level: tracing::Level,
11568 fields: BTreeMap<String, String>,
11569 }
11570
11571 impl EventCapture {
11572 fn events(&self) -> Vec<CapturedEvent> {
11573 self.events.lock().unwrap().clone()
11574 }
11575 }
11576
11577 impl<S> Layer<S> for EventCapture
11578 where
11579 S: Subscriber,
11580 {
11581 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
11582 let mut visitor = EventFieldVisitor::default();
11583 event.record(&mut visitor);
11584 self.events.lock().unwrap().push(CapturedEvent {
11585 target: event.metadata().target().to_string(),
11586 level: *event.metadata().level(),
11587 fields: visitor.fields,
11588 });
11589 }
11590 }
11591
11592 #[derive(Default)]
11593 struct EventFieldVisitor {
11594 fields: BTreeMap<String, String>,
11595 }
11596
11597 impl Visit for EventFieldVisitor {
11598 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
11599 self.fields
11600 .insert(field.name().to_string(), format!("{value:?}"));
11601 }
11602 }
11603
11604 fn health_response(corr: u64, status: HealthStatus) -> Frame {
11605 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
11606 status,
11607 detail: Some("warming".to_string()),
11608 metrics: Some(json!({"queue_depth": 3})),
11609 })
11610 .unwrap();
11611 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11612 }
11613
11614 fn route_bind_ack(corr: u64) -> Frame {
11615 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
11616 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11617 }
11618
11619 fn unique_project_root(label: &str) -> ScratchDir {
11620 ScratchDir::new(label)
11621 }
11622
11623 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
11624 match parse_route_poll(frame) {
11625 ClientControlResponse::RoutePoll {
11626 status: None,
11627 live: Some(live),
11628 ..
11629 } => assert_eq!(live, expected_live),
11630 other => panic!("unexpected route.poll response: {other:?}"),
11631 }
11632 }
11633
11634 fn bind_liveness_route(
11635 registry: &Registry,
11636 forwarding: &ForwardingTable,
11637 module_id: &str,
11638 ) -> (RouteCtx, u16, u32) {
11639 let module_connection = ConnectionId::new(101);
11640 let client_connection = ConnectionId::new(202);
11641 let registration = registry
11642 .register_with_control_ops(
11643 manifest(module_id, PROTOCOL_VERSION),
11644 PROTOCOL_VERSION,
11645 module_connection,
11646 module_baseline_control_ops(),
11647 )
11648 .unwrap();
11649 let (module_tx, _module_rx) = mpsc::channel(8);
11650 let endpoint = forwarding
11651 .register_module_connection(
11652 module_connection,
11653 module_id.to_string(),
11654 PROTOCOL_VERSION,
11655 manifest_concurrency(®istration.manifest),
11656 FrameSink::new(module_tx),
11657 )
11658 .unwrap();
11659 let (client_ctx, _client_rx) = route_ctx(client_connection);
11660 let pending = forwarding
11661 .begin_route_bind_relay_for_test(
11662 client_connection,
11663 client_ctx.egress.clone(),
11664 1,
11665 module_id,
11666 )
11667 .unwrap();
11668 assert_eq!(pending.endpoint, endpoint);
11669 let route_channel = pending.client_channel;
11670 let route_epoch = pending.client_epoch;
11671 forwarding
11672 .complete_pending_relay(
11673 module_connection,
11674 pending.corr,
11675 RouteBindRelayOutcome::Accepted,
11676 )
11677 .unwrap();
11678 (client_ctx, route_channel, route_epoch)
11679 }
11680
11681 struct FakeProcessLiveness {
11682 live: Option<bool>,
11683 }
11684
11685 impl ModuleProcessLiveness for FakeProcessLiveness {
11686 fn process_live(&self, _module_id: &str) -> Option<bool> {
11687 self.live
11688 }
11689 }
11690
11691 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11692 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
11693 {
11694 let registry = Arc::new(Registry::default());
11695 let supervisor_handle = SupervisorHandle::new();
11696 let supervisor = Supervisor::new_for_test(
11697 Arc::clone(®istry),
11698 RestartPolicy::new(1, Duration::from_millis(10)),
11699 )
11700 .with_handle(supervisor_handle.clone());
11701 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
11702 let module = supervisor
11703 .spawn(ModuleSpec {
11704 script: None,
11705 carry_containment: false,
11706 module_id: "stderr-tail-wire".to_string(),
11707 program: fake_aft_stub_path(),
11708 args: Vec::new(),
11709 env: vec![
11710 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
11711 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
11712 ],
11713 reserved: false,
11714 reserved_prefixes: Vec::new(),
11715 protocol: ModuleProtocol::Subc,
11716 overlap: Default::default(),
11717 })
11718 .unwrap();
11719
11720 let deadline = Instant::now() + Duration::from_secs(5);
11721 loop {
11722 let tail = module.stderr_tail(None, None);
11723 if tail
11724 .entries
11725 .iter()
11726 .any(|entry| matches!(entry, TailEntry::ProcessStart))
11727 && tail.entries.iter().any(|entry| {
11728 matches!(
11729 entry,
11730 TailEntry::Line {
11731 truncated: true,
11732 ..
11733 }
11734 )
11735 })
11736 {
11737 break;
11738 }
11739 assert!(
11740 Instant::now() < deadline,
11741 "module did not produce a truncated line and restart boundary: {tail:?}"
11742 );
11743 sleep(Duration::from_millis(10)).await;
11744 }
11745
11746 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11747 let request = ClientControlRequest::SupervisorStderrTail {
11748 module_id: "stderr-tail-wire".to_string(),
11749 max_lines: None,
11750 max_bytes: None,
11751 };
11752 let frame = Frame::build(
11753 FrameType::Request,
11754 control_flags(),
11755 0,
11756 0,
11757 1,
11758 serde_json::to_vec(&request).unwrap(),
11759 )
11760 .unwrap();
11761 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11762 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11763 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
11764 serde_json::from_slice(&responses[0].body).unwrap()
11765 else {
11766 panic!("expected supervisor.stderr_tail response");
11767 };
11768
11769 assert!(
11770 tail.entries
11771 .iter()
11772 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
11773 "the control response lost the restart boundary"
11774 );
11775 let Some(StderrTailEntry::Line {
11776 text,
11777 truncated,
11778 at_ms,
11779 }) = tail.entries.iter().find(|entry| {
11780 matches!(
11781 entry,
11782 StderrTailEntry::Line {
11783 truncated: true,
11784 ..
11785 }
11786 )
11787 })
11788 else {
11789 panic!("the control response lost the truncated line");
11790 };
11791 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
11792 assert!(*truncated);
11793 assert!(
11794 at_ms.is_some(),
11795 "the control response lost the line's capture time"
11796 );
11797 }
11798
11799 #[tokio::test(flavor = "current_thread")]
11804 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
11805 let dir = ScratchDir::new("terminals-off-worker");
11806 let journal_path = dir.join("terminals.jsonl");
11807 let registry = Arc::new(Registry::default());
11808 let supervisor_handle = SupervisorHandle::new();
11809 let supervisor =
11810 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11811 .with_handle(supervisor_handle.clone())
11812 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
11813 let module = supervisor
11814 .spawn(ModuleSpec {
11815 script: None,
11816 carry_containment: false,
11817 module_id: "terminal-off-worker".to_string(),
11818 program: fake_aft_stub_path(),
11819 args: Vec::new(),
11820 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11821 reserved: false,
11822 reserved_prefixes: Vec::new(),
11823 protocol: ModuleProtocol::Subc,
11824 overlap: Default::default(),
11825 })
11826 .unwrap();
11827 let deadline = Instant::now() + Duration::from_secs(5);
11828 while module.terminal_history().entries.len() != 2 {
11829 assert!(Instant::now() < deadline, "module did not record two exits");
11830 sleep(Duration::from_millis(10)).await;
11831 }
11832
11833 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
11834 let handler =
11835 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
11836 let frame = Frame::build(
11837 FrameType::Request,
11838 control_flags(),
11839 0,
11840 0,
11841 1,
11842 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
11843 module_id: "terminal-off-worker".to_string(),
11844 })
11845 .unwrap(),
11846 )
11847 .unwrap();
11848 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11849 let spawned_at = std::time::Instant::now();
11850 let read = tokio::spawn({
11851 let handler = Arc::clone(&handler);
11852 async move { handler.handle_control_frame(&ctx, frame).await }
11853 });
11854 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
11857 .await
11858 .unwrap()
11859 .expect("the history read reached its pause");
11860 let elapsed = spawned_at.elapsed();
11861 assert!(
11862 elapsed < Duration::from_secs(2) && !read.is_finished(),
11863 "this task could not run while the history read was paused \
11864 (resumed after {elapsed:?}, read finished: {})",
11865 read.is_finished()
11866 );
11867
11868 drop(release);
11869 let responses = read.await.unwrap().unwrap();
11870 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11871 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
11872 panic!("expected supervisor.terminals response");
11873 };
11874 assert_eq!(terminals.entries.len(), 2);
11875 assert_eq!(terminals.journal_skipped_lines, 0);
11876 assert_eq!(terminals.journal_read_errors, 0);
11877 }
11878
11879 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11880 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
11881 let registry = Arc::new(Registry::default());
11882 let supervisor_handle = SupervisorHandle::new();
11883 let supervisor =
11884 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11885 .with_handle(supervisor_handle.clone());
11886 let module = supervisor
11887 .spawn(ModuleSpec {
11888 script: None,
11889 carry_containment: false,
11890 module_id: "terminal-golden".to_string(),
11891 program: fake_aft_stub_path(),
11892 args: Vec::new(),
11893 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11894 reserved: false,
11895 reserved_prefixes: Vec::new(),
11896 protocol: ModuleProtocol::Subc,
11897 overlap: Default::default(),
11898 })
11899 .unwrap();
11900
11901 let deadline = Instant::now() + Duration::from_secs(5);
11902 while module.terminal_history().entries.len() != 2 {
11903 assert!(
11904 Instant::now() < deadline,
11905 "module did not retain two terminal exits: {:?}",
11906 module.terminal_history()
11907 );
11908 sleep(Duration::from_millis(10)).await;
11909 }
11910
11911 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11912 let request = ClientControlRequest::SupervisorTerminals {
11913 module_id: "terminal-golden".to_string(),
11914 };
11915 let frame = Frame::build(
11916 FrameType::Request,
11917 control_flags(),
11918 0,
11919 0,
11920 1,
11921 serde_json::to_vec(&request).unwrap(),
11922 )
11923 .unwrap();
11924 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11925 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11926 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11927 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
11928 panic!("expected supervisor.terminals response");
11929 };
11930 assert_eq!(terminals.entries.len(), 2);
11931 assert_eq!(terminals.dropped, 0);
11932
11933 let mut rendered = serde_json::to_value(response).unwrap();
11934 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
11937 for (index, entry) in rendered["entries"]
11938 .as_array_mut()
11939 .expect("terminal response entries array")
11940 .iter_mut()
11941 .enumerate()
11942 {
11943 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
11944 }
11945
11946 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11947 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
11948 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
11949 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11950 std::fs::write(&golden_path, &serialized).unwrap();
11951 }
11952 let expected: Value =
11953 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
11954 assert_eq!(rendered, expected);
11955 }
11956
11957 #[test]
11958 fn hello_registers_manifest_and_returns_ack() {
11959 let registry = Arc::new(Registry::default());
11960 let handler = ControlHandler::new(Arc::clone(®istry));
11961 let conn = ConnectionId::new(1);
11962
11963 let responses = handler
11964 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
11965 .unwrap();
11966
11967 assert_eq!(responses.len(), 1);
11968 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11969 assert_eq!(responses[0].header.channel, 0);
11970 assert_eq!(responses[0].header.corr, 7);
11971 let ack = parse_ack(&responses[0]);
11972 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11973 assert!(ack
11974 .subc_capabilities
11975 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
11976 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
11977 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
11978 assert!(ack
11979 .subc_ops
11980 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
11981 assert!(ack
11982 .subc_ops
11983 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
11984 assert!(ack
11985 .subc_ops
11986 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
11987
11988 let registration = registry.get_module("aft").unwrap().unwrap();
11989 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
11990 assert_eq!(registration.state, ChannelState::Active);
11991 assert_eq!(registration.connection_id, conn);
11992 assert_eq!(registration.control_ops, module_baseline_control_ops());
11993 assert!(registration.manifest.display_name.is_none());
11994 }
11995
11996 #[test]
11997 fn hello_display_name_refuses_invalid_manifest_by_field() {
11998 let registry = Arc::new(Registry::default());
11999 let handler = ControlHandler::new(Arc::clone(®istry));
12000 for (index, invalid) in [
12001 json!(""),
12002 json!("a".repeat(41)),
12003 json!(" Routing"),
12004 json!("a\nb"),
12005 json!("a\u{202e}b"),
12006 json!(7),
12007 ]
12008 .into_iter()
12009 .enumerate()
12010 {
12011 let module_id = format!("bad-display-name-{index}");
12012 let mut body = serde_json::to_value(ModuleHelloBody {
12013 manifest: manifest(&module_id, PROTOCOL_VERSION),
12014 protocol_ver: PROTOCOL_VERSION,
12015 control_ops: None,
12016 launch_nonce: None,
12017 })
12018 .unwrap();
12019 body["manifest"]["display_name"] = invalid;
12020 let hello = Frame::build(
12021 FrameType::Hello,
12022 control_flags(),
12023 0,
12024 0,
12025 7,
12026 serde_json::to_vec(&body).unwrap(),
12027 )
12028 .unwrap();
12029 let response = handler
12030 .handle_control(ConnectionId::new(100 + index as u64), hello)
12031 .unwrap();
12032 assert_eq!(response[0].header.ty, FrameType::Error);
12033 let error = parse_error(&response[0]);
12034 assert_eq!(error["code"], "invalid_manifest");
12035 assert!(error["message"].as_str().unwrap().contains("display_name"));
12036 assert!(registry.get_module(&module_id).unwrap().is_none());
12037 }
12038 }
12039
12040 #[test]
12041 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
12042 let invalid_identifiers = [
12043 ("case_change", "credentials-Provider/v1"),
12044 ("leading_zero", "credentials-provider/v01"),
12045 ("trailing_hyphen", "credentials-provider-/v1"),
12046 ("consecutive_hyphens", "credentials--provider/v1"),
12047 ("uppercase", "Credentials-provider/v1"),
12048 ("missing_v", "credentials-provider/1"),
12049 ("whitespace", "credentials provider/v1"),
12050 ("zero_version", "credentials-provider/v0"),
12051 ("out_of_range_version", "credentials-provider/v4294967296"),
12052 (
12053 "overlength_name",
12054 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
12055 ),
12056 ];
12057 let mut cases = invalid_identifiers
12058 .into_iter()
12059 .map(|(name, identifier)| {
12060 (
12061 format!("identifier_{name}"),
12062 "capabilities.provides[0]".to_string(),
12063 identifier.to_string(),
12064 json!({ "provides": [identifier] }),
12065 None,
12066 )
12067 })
12068 .collect::<Vec<_>>();
12069 cases.extend([
12070 (
12071 "unknown_need".to_string(),
12072 "capabilities.requires[0].need".to_string(),
12073 "deferred".to_string(),
12074 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
12075 None,
12076 ),
12077 (
12078 "duplicate_provides".to_string(),
12079 "capabilities.provides[1]".to_string(),
12080 "credentials-provider/v1".to_string(),
12081 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
12082 None,
12083 ),
12084 (
12085 "duplicate_must_never_reach".to_string(),
12086 "capabilities.must_never_reach[1]".to_string(),
12087 "credentials-provider/v1".to_string(),
12088 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
12089 None,
12090 ),
12091 (
12092 "duplicate_requires_same_need".to_string(),
12093 "capabilities.requires[1]".to_string(),
12094 "credentials-provider/v1".to_string(),
12095 json!({ "requires": [
12096 { "capability": "credentials-provider/v1", "need": "required" },
12097 { "capability": "credentials-provider/v1", "need": "required" }
12098 ] }),
12099 None,
12100 ),
12101 (
12102 "duplicate_requires_conflicting_need".to_string(),
12103 "capabilities.requires[1]".to_string(),
12104 "credentials-provider/v1".to_string(),
12105 json!({ "requires": [
12106 { "capability": "credentials-provider/v1", "need": "required" },
12107 { "capability": "credentials-provider/v1", "need": "optional" }
12108 ] }),
12109 None,
12110 ),
12111 (
12112 "capabilities_root_pointer".to_string(),
12113 "runtime_computed[0]".to_string(),
12114 "/capabilities".to_string(),
12115 json!({}),
12116 Some(json!(["/capabilities"])),
12117 ),
12118 (
12119 "capabilities_descendant_pointer".to_string(),
12120 "runtime_computed[0]".to_string(),
12121 "/capabilities/provides".to_string(),
12122 json!({}),
12123 Some(json!(["/capabilities/provides"])),
12124 ),
12125 (
12126 "malformed_pointer_without_leading_slash".to_string(),
12127 "runtime_computed[0]".to_string(),
12128 "capabilities".to_string(),
12129 json!({}),
12130 Some(json!(["capabilities"])),
12131 ),
12132 (
12133 "malformed_pointer_escape".to_string(),
12134 "runtime_computed[0]".to_string(),
12135 "/roles/~2/tools".to_string(),
12136 json!({}),
12137 Some(json!(["/roles/~2/tools"])),
12138 ),
12139 (
12140 "unknown_capabilities_field".to_string(),
12141 "capabilities.future".to_string(),
12142 "<array>".to_string(),
12143 json!({ "future": [] }),
12144 None,
12145 ),
12146 ]);
12147
12148 for (index, (name, field, value, capabilities, runtime_computed)) in
12149 cases.into_iter().enumerate()
12150 {
12151 let registry = Arc::new(Registry::default());
12152 let handler = ControlHandler::new(Arc::clone(®istry));
12153 let response = handler
12154 .handle_control(
12155 ConnectionId::new((index + 1) as u64),
12156 capability_grammar_hello_frame(
12157 capabilities,
12158 runtime_computed,
12159 index as u64 + 1,
12160 ),
12161 )
12162 .expect("invalid HELLO returns a refusal");
12163
12164 assert_eq!(response.len(), 1, "{name} must emit one refusal");
12165 let error = parse_error(&response[0]);
12166 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
12167 let message = error["message"]
12168 .as_str()
12169 .expect("error message is a string");
12170 assert!(
12171 message.contains(&field),
12172 "{name}: field missing from {message}"
12173 );
12174 assert!(
12175 message.contains(&value),
12176 "{name}: value missing from {message}"
12177 );
12178 assert_eq!(
12179 registry
12180 .active_registration_count()
12181 .expect("registry reads"),
12182 0,
12183 "{name}: refused HELLO must not create a catalog entry"
12184 );
12185 }
12186 }
12187
12188 #[test]
12189 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
12190 let registry = Arc::new(Registry::default());
12191 let handler = ControlHandler::new(Arc::clone(®istry));
12192 let capabilities = json!({
12193 "provides": ["credentials-provider/v1"],
12194 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
12195 "must_never_reach": ["federation-transport/v1"]
12196 });
12197 let response = handler
12198 .handle_control(
12199 ConnectionId::new(99),
12200 capability_grammar_hello_frame(
12201 capabilities.clone(),
12202 Some(json!(["/roles/0/tools"])),
12203 99,
12204 ),
12205 )
12206 .expect("valid HELLO registers");
12207 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12208
12209 let request = Frame::build(
12210 FrameType::Request,
12211 control_flags(),
12212 0,
12213 0,
12214 100,
12215 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12216 .expect("catalog request serializes"),
12217 )
12218 .expect("catalog request frame builds");
12219 let response = handler
12220 .handle_catalog_list(request, None)
12221 .expect("catalog list succeeds");
12222 let ClientControlResponse::CatalogList { modules, .. } =
12223 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
12224 else {
12225 panic!("catalog request must return catalog.list");
12226 };
12227 assert_eq!(modules.len(), 1);
12228 assert_eq!(
12229 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
12230 capabilities
12231 );
12232 }
12233
12234 #[test]
12235 fn catalog_list_mirrors_management_operation_description() {
12236 let registry = Arc::new(Registry::default());
12237 let handler = ControlHandler::new(Arc::clone(®istry));
12238 let description = "List managed records and return their identifiers and metadata.";
12239 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
12240 manifest.provides = vec![ProviderRole::ManagementSurface {
12241 operations: vec![ManagementOperation {
12242 name: "records.list".to_string(),
12243 kind: ManagementOperationKind::Query,
12244 description: Some(description.to_string()),
12245 }],
12246 config_schema: json!({"type": "object"}),
12247 observability: vec![ObservabilitySurface {
12248 name: "records.stats".to_string(),
12249 kind: ObservabilityKind::Snapshot,
12250 }],
12251 identity_scope: vec![IdentityScope::Project],
12252 concurrency: Concurrency::ModuleManaged,
12253 }];
12254 registry
12255 .register_with_control_ops(
12256 manifest,
12257 PROTOCOL_VERSION,
12258 ConnectionId::new(99),
12259 Vec::new(),
12260 )
12261 .expect("described management manifest registers");
12262
12263 let request = Frame::build(
12264 FrameType::Request,
12265 control_flags(),
12266 0,
12267 0,
12268 100,
12269 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12270 .expect("catalog request serializes"),
12271 )
12272 .expect("catalog request frame builds");
12273 let response = handler
12274 .handle_catalog_list(request, None)
12275 .expect("catalog list succeeds");
12276 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
12277 assert_eq!(
12278 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
12279 "catalog.list must preserve the declared operation description verbatim"
12280 );
12281 }
12282
12283 #[test]
12284 fn catalog_list_mirrors_registered_operation_delivery() {
12285 let registry = Arc::new(Registry::default());
12286 let handler = ControlHandler::new(Arc::clone(®istry));
12287 let delivery =
12288 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
12289 let mut body = serde_json::to_value(ModuleHelloBody {
12290 manifest: manifest("watch-management", PROTOCOL_VERSION),
12291 protocol_ver: PROTOCOL_VERSION,
12292 control_ops: None,
12293 launch_nonce: None,
12294 })
12295 .unwrap();
12296 body["manifest"]["provides"] = json!([{
12297 "role": "management_surface",
12298 "operations": [
12299 {"name": "status", "kind": "query"},
12300 {"name": "future", "kind": "query"},
12301 {"name": "reset", "kind": "mutate"}
12302 ],
12303 "config_schema": {}, "observability": [], "identity_scope": []
12304 }]);
12305 body["manifest"]["operation_delivery"] = delivery.clone();
12306 let hello = Frame::build(
12307 FrameType::Hello,
12308 control_flags(),
12309 0,
12310 0,
12311 99,
12312 serde_json::to_vec(&body).unwrap(),
12313 )
12314 .unwrap();
12315 let response = handler
12316 .handle_control(ConnectionId::new(99), hello)
12317 .unwrap();
12318 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12319
12320 let (_, registrations) = registry.list_modules().unwrap();
12321 let registered = ®istrations[0].manifest;
12322 assert!(registered.is_watch_operation("status"));
12323 assert!(!registered.is_watch_operation("future"));
12324 assert!(!registered.is_watch_operation("reset"));
12325 assert!(!registered.is_watch_operation("missing"));
12326
12327 let request = Frame::build(
12328 FrameType::Request,
12329 control_flags(),
12330 0,
12331 0,
12332 100,
12333 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12334 )
12335 .unwrap();
12336 let response = handler.handle_catalog_list(request, None).unwrap();
12337 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
12338 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
12339 let ClientControlResponse::CatalogList { modules, .. } =
12340 serde_json::from_slice(&response[0].body).unwrap()
12341 else {
12342 panic!("expected catalog.list");
12343 };
12344 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
12345 }
12346
12347 #[test]
12348 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
12349 let registry = Arc::new(Registry::default());
12350 let supervisor = SupervisorHandle::new();
12351 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12352 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12353 let descriptor = json!({
12354 "schema": "ckios.richui.manifest/1",
12355 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12356 "bytes": 4096,
12357 "chunks": 4
12358 });
12359 let connection = ConnectionId::new(201);
12360 let hello = handler
12361 .handle_control(
12362 connection,
12363 hello_frame_with_rich_ui(
12364 "vault",
12365 PROTOCOL_VERSION,
12366 201,
12367 Some("the-real-nonce"),
12368 descriptor.clone(),
12369 ),
12370 )
12371 .unwrap();
12372 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12373
12374 let request =
12375 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
12376 let provides = registry
12377 .get_module_by_connection(connection)
12378 .unwrap()
12379 .unwrap()
12380 .manifest
12381 .provides;
12382 handler
12383 .handle_catalog_update(connection, request, provides, None, Some(false))
12384 .unwrap();
12385
12386 let request = Frame::build(
12387 FrameType::Request,
12388 control_flags(),
12389 0,
12390 0,
12391 203,
12392 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12393 )
12394 .unwrap();
12395 let response = handler.handle_catalog_list(request, None).unwrap();
12396 let ClientControlResponse::CatalogList { modules, .. } =
12397 serde_json::from_slice(&response[0].body).unwrap()
12398 else {
12399 panic!("expected catalog.list");
12400 };
12401 assert_eq!(
12402 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
12403 descriptor
12404 );
12405 assert!(modules[0].nonce_verified);
12406 }
12407
12408 fn hello_with_role_declarations(declarations: Value) -> Frame {
12409 let mut body = serde_json::to_value(ModuleHelloBody {
12410 manifest: manifest("role-test", PROTOCOL_VERSION),
12411 protocol_ver: PROTOCOL_VERSION,
12412 control_ops: None,
12413 launch_nonce: None,
12414 })
12415 .unwrap();
12416 body["manifest"]["role_declarations"] = declarations;
12417 Frame::build(
12418 FrameType::Hello,
12419 control_flags(),
12420 0,
12421 0,
12422 301,
12423 serde_json::to_vec(&body).unwrap(),
12424 )
12425 .unwrap()
12426 }
12427
12428 fn update_role_declarations_frame(declarations: Value) -> Frame {
12429 Frame::build(
12430 FrameType::Request,
12431 control_flags(),
12432 0,
12433 0,
12434 303,
12435 serde_json::to_vec(&json!({
12436 "op": "catalog.update_role_declarations",
12437 "role_declarations": declarations,
12438 }))
12439 .unwrap(),
12440 )
12441 .unwrap()
12442 }
12443
12444 #[tokio::test]
12445 async fn role_declarations_update_replaces_then_clears_catalog() {
12446 let registry = Arc::new(Registry::default());
12447 let handler = ControlHandler::new(Arc::clone(®istry));
12448 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12449 let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
12450 hello_via_sink(
12451 &handler,
12452 &ctx,
12453 &mut rx,
12454 hello_with_role_declarations(initial.clone()),
12455 )
12456 .await;
12457 assert_eq!(
12458 role_declarations_catalog(&handler).0["role_declarations"],
12459 initial
12460 );
12461 let before = registry.get_module("role-test").unwrap().unwrap();
12462 for declarations in [
12463 json!({"media-provider/v1": {"engines": ["new"]}}),
12464 json!({}),
12465 ] {
12466 let response = handler
12467 .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
12468 .await
12469 .unwrap();
12470 assert_eq!(response[0].header.ty, FrameType::Response);
12471 assert_eq!(
12472 serde_json::from_slice::<Value>(&response[0].body).unwrap(),
12473 json!({"op": "catalog.update_role_declarations"})
12474 );
12475 let (raw, entry) = role_declarations_catalog(&handler);
12476 assert_eq!(
12477 serde_json::to_value(entry.role_declarations).unwrap(),
12478 declarations
12479 );
12480 if declarations.as_object().unwrap().is_empty() {
12481 assert!(raw.get("role_declarations").is_none());
12482 } else {
12483 assert_eq!(raw["role_declarations"], declarations);
12484 }
12485 let after = registry.get_module("role-test").unwrap().unwrap();
12486 let mut expected = before.manifest.clone();
12487 expected.role_declarations = serde_json::from_value(declarations).unwrap();
12488 assert_eq!(after.manifest, expected);
12489 assert_eq!(after.ready, before.ready);
12490 }
12491 }
12492
12493 #[tokio::test]
12494 async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
12495 let registry = Arc::new(Registry::default());
12496 let handler = ControlHandler::new(registry);
12497 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12498 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12499 hello_via_sink(
12500 &handler,
12501 &ctx,
12502 &mut rx,
12503 hello_with_role_declarations(initial.clone()),
12504 )
12505 .await;
12506 for (invalid, reason) in [
12507 (json!(null), "role_declarations must be an object"),
12508 (json!([]), "role_declarations must be an object"),
12509 (
12510 json!({"bad key": {}}),
12511 "role_declarations key must use capability-name grammar",
12512 ),
12513 (
12514 json!({"media-provider/v1": []}),
12515 "role_declarations values must be objects",
12516 ),
12517 (
12518 json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
12519 "role_declarations exceeds 16384 compact JSON bytes",
12520 ),
12521 ] {
12522 let response = handler
12523 .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
12524 .await
12525 .unwrap();
12526 let error = parse_error(&response[0]);
12527 assert_eq!(error["code"], "invalid_role_declarations");
12528 assert_eq!(error["message"], reason);
12529 assert_eq!(
12530 role_declarations_catalog(&handler).0["role_declarations"],
12531 initial
12532 );
12533 }
12534 }
12535
12536 #[tokio::test]
12537 async fn role_declarations_update_is_owned_by_the_registered_connection() {
12538 let registry = Arc::new(Registry::default());
12539 let handler = ControlHandler::new(Arc::clone(®istry));
12540 let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
12541 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12542 hello_via_sink(
12543 &handler,
12544 &owner,
12545 &mut owner_rx,
12546 hello_with_role_declarations(initial.clone()),
12547 )
12548 .await;
12549 let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
12550 let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
12551 let response = handler
12552 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12553 .await
12554 .unwrap();
12555 assert_eq!(parse_error(&response[0])["code"], "not_registered");
12556 hello_via_sink(
12557 &handler,
12558 &other,
12559 &mut other_rx,
12560 hello_frame("other", PROTOCOL_VERSION, 304),
12561 )
12562 .await;
12563 let response = handler
12564 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12565 .await
12566 .unwrap();
12567 assert_eq!(response[0].header.ty, FrameType::Response);
12568 assert_eq!(
12569 serde_json::to_value(
12570 registry
12571 .get_module("role-test")
12572 .unwrap()
12573 .unwrap()
12574 .manifest
12575 .role_declarations
12576 )
12577 .unwrap(),
12578 initial
12579 );
12580 assert_eq!(
12581 serde_json::to_value(
12582 registry
12583 .get_module("other")
12584 .unwrap()
12585 .unwrap()
12586 .manifest
12587 .role_declarations
12588 )
12589 .unwrap(),
12590 replacement
12591 );
12592 }
12593
12594 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
12595 let request = Frame::build(
12596 FrameType::Request,
12597 control_flags(),
12598 0,
12599 0,
12600 302,
12601 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12602 )
12603 .unwrap();
12604 let response = handler.handle_catalog_list(request, None).unwrap();
12605 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
12606 let ClientControlResponse::CatalogList { mut modules, .. } =
12607 serde_json::from_slice(&response[0].body).unwrap()
12608 else {
12609 panic!("expected catalog.list");
12610 };
12611 (raw["modules"][0].clone(), modules.remove(0))
12612 }
12613
12614 #[test]
12615 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
12616 let registry = Arc::new(Registry::default());
12617 let handler = ControlHandler::new(Arc::clone(®istry));
12618 let declarations = json!({
12619 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
12620 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
12621 });
12622 let connection = ConnectionId::new(301);
12623 let response = handler
12624 .handle_control(
12625 connection,
12626 hello_with_role_declarations(declarations.clone()),
12627 )
12628 .unwrap();
12629 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12630 let registered = registry
12631 .get_module_by_connection(connection)
12632 .unwrap()
12633 .unwrap()
12634 .manifest;
12635 assert_eq!(
12639 serde_json::to_value(®istered.role_declarations).unwrap(),
12640 declarations
12641 );
12642 let (raw, entry) = role_declarations_catalog(&handler);
12643 assert_eq!(raw["role_declarations"], declarations);
12644 assert_eq!(entry.role_declarations, registered.role_declarations);
12645
12646 let request =
12647 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
12648 handler
12649 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
12650 .unwrap();
12651 let (raw, entry) = role_declarations_catalog(&handler);
12652 assert_eq!(raw["role_declarations"], declarations);
12653 assert_eq!(entry.role_declarations, registered.role_declarations);
12654 }
12655
12656 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
12657 let registry = Arc::new(Registry::default());
12658 let handler = ControlHandler::new(Arc::clone(®istry));
12659 let capture = EventCapture::default();
12660 let _subscriber =
12661 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12662 let response = handler
12663 .handle_control(
12664 ConnectionId::new(301),
12665 hello_with_role_declarations(declarations),
12666 )
12667 .unwrap();
12668 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12669 assert!(registry
12670 .get_module_by_connection(ConnectionId::new(301))
12671 .unwrap()
12672 .unwrap()
12673 .manifest
12674 .role_declarations
12675 .is_empty());
12676 let (raw, entry) = role_declarations_catalog(&handler);
12677 assert!(raw.get("role_declarations").is_none());
12678 assert!(entry.role_declarations.is_empty());
12679 let warnings: Vec<_> = capture
12680 .events()
12681 .into_iter()
12682 .filter(|event| {
12683 event.fields.get("message").map(String::as_str)
12684 == Some("dropping whole role_declarations map from HELLO")
12685 })
12686 .collect();
12687 assert_eq!(warnings.len(), 1);
12688 assert_eq!(warnings[0].level, tracing::Level::WARN);
12689 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
12690 assert_eq!(warnings[0].fields["reason"], reason);
12691 }
12692
12693 #[test]
12694 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
12695 for value in [
12696 json!(null),
12697 json!(false),
12698 json!(0),
12699 json!("opaque"),
12700 json!([]),
12701 ] {
12702 assert_hello_drops_role_declarations_with_warning(
12703 json!({"routing/v1": {}, "broken/v1": value}),
12704 "role_declarations values must be objects",
12705 );
12706 }
12707 assert_hello_drops_role_declarations_with_warning(
12708 json!([]),
12709 "role_declarations must be an object",
12710 );
12711 }
12712
12713 #[test]
12714 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
12715 assert_hello_drops_role_declarations_with_warning(
12716 json!({"routing/v1": {}, "bad key": {}}),
12717 "role_declarations key must use capability-name grammar",
12718 );
12719 }
12720
12721 #[test]
12722 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
12723 assert_hello_drops_role_declarations_with_warning(
12724 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
12725 "role_declarations exceeds 16384 compact JSON bytes",
12726 );
12727 }
12728
12729 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
12730 let mut body = serde_json::to_value(ModuleHelloBody {
12731 manifest: manifest("flow-ops", PROTOCOL_VERSION),
12732 protocol_ver: PROTOCOL_VERSION,
12733 control_ops: None,
12734 launch_nonce: None,
12735 })
12736 .unwrap();
12737 body["manifest"]["provides"] = json!([{
12738 "role": "management_surface",
12739 "operations": [
12740 {"name": "diagnostics.findings", "kind": "query"},
12741 {"name": "reset", "kind": "mutate"}
12742 ],
12743 "config_schema": {}, "observability": [], "identity_scope": []
12744 }]);
12745 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
12746 if let Some(flow_operations) = flow_operations {
12747 body["manifest"]["flow_operations"] = flow_operations;
12748 }
12749 Frame::build(
12750 FrameType::Hello,
12751 control_flags(),
12752 0,
12753 0,
12754 401,
12755 serde_json::to_vec(&body).unwrap(),
12756 )
12757 .unwrap()
12758 }
12759
12760 #[test]
12761 fn catalog_list_mirrors_flow_operations_unchanged() {
12762 for (capability, flow_operations) in [
12765 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
12766 ("call-modes/v1", json!(["diagnostics.findings"])),
12767 ("flow-scopes/v1", json!([])),
12768 ] {
12769 let registry = Arc::new(Registry::default());
12770 let handler = ControlHandler::new(Arc::clone(®istry));
12771 let connection = ConnectionId::new(401);
12772 let response = handler
12773 .handle_control(
12774 connection,
12775 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
12776 )
12777 .unwrap();
12778 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12779 let registered = registry
12780 .get_module_by_connection(connection)
12781 .unwrap()
12782 .unwrap()
12783 .manifest;
12784 assert_eq!(
12785 serde_json::to_value(®istered.flow_operations).unwrap(),
12786 flow_operations
12787 );
12788 let (raw, entry) = role_declarations_catalog(&handler);
12789 assert_eq!(raw["flow_operations"], flow_operations);
12790 assert_eq!(entry.flow_operations, registered.flow_operations);
12791 }
12792
12793 let registry = Arc::new(Registry::default());
12794 let handler = ControlHandler::new(Arc::clone(®istry));
12795 let response = handler
12796 .handle_control(
12797 ConnectionId::new(401),
12798 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
12799 )
12800 .unwrap();
12801 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12802 let (raw, entry) = role_declarations_catalog(&handler);
12803 assert!(raw.get("flow_operations").is_none());
12804 assert_eq!(entry.flow_operations, None);
12805 }
12806
12807 #[test]
12808 fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
12809 let oversized: serde_json::Map<String, Value> = (0..257)
12810 .map(|n| (format!("tool:t{n}"), json!(true)))
12811 .collect();
12812 for (capabilities, map, valid) in [
12813 (
12814 json!(["scope-federated/v1"]),
12815 json!({"tool:read":true,"management:reset":true}),
12816 true,
12817 ),
12818 (json!(["scope-federated/v1"]), json!({}), true),
12819 (json!([]), json!({"tool:read":true}), false),
12820 (
12821 json!(["scope-federated/v1"]),
12822 json!({"tool:valid":true,"tool:bad":false}),
12823 false,
12824 ),
12825 (
12826 json!(["scope-federated/v1"]),
12827 json!({"tool:valid":true,"management:missing":true}),
12828 false,
12829 ),
12830 (
12831 json!(["scope-federated/v1"]),
12832 Value::Object(oversized),
12833 false,
12834 ),
12835 ] {
12836 let capture = EventCapture::default();
12837 let _subscriber = tracing::subscriber::set_default(
12838 tracing_subscriber::registry().with(capture.clone()),
12839 );
12840 let registry = Arc::new(Registry::default());
12841 let handler = ControlHandler::new(Arc::clone(®istry));
12842 let hello = hello_with_flow_operations(capabilities, None);
12843 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12844 body["manifest"]["operation_deferral"] = map.clone();
12845 let hello = Frame::build(
12846 FrameType::Hello,
12847 control_flags(),
12848 0,
12849 0,
12850 401,
12851 serde_json::to_vec(&body).unwrap(),
12852 )
12853 .unwrap();
12854 let response = handler
12855 .handle_control(ConnectionId::new(401), hello)
12856 .unwrap();
12857 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12858 let (raw, entry) = role_declarations_catalog(&handler);
12859 if valid {
12860 assert_eq!(raw["operation_deferral"], map);
12861 assert_eq!(
12862 serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
12863 map
12864 );
12865 } else {
12866 assert!(entry.operation_deferral.is_none());
12867 assert!(raw.get("operation_deferral").is_none());
12868 assert_eq!(
12869 capture
12870 .events()
12871 .iter()
12872 .filter(|event| event.fields.get("message").map(String::as_str)
12873 == Some("dropping operation_deferral from HELLO"))
12874 .count(),
12875 1
12876 );
12877 }
12878 }
12879 }
12880
12881 fn assert_hello_drops_flow_operations_with_warning(
12882 capabilities: Value,
12883 flow_operations: Value,
12884 reason: &str,
12885 ) {
12886 let registry = Arc::new(Registry::default());
12887 let handler = ControlHandler::new(Arc::clone(®istry));
12888 let capture = EventCapture::default();
12889 let _subscriber =
12890 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12891 let response = handler
12892 .handle_control(
12893 ConnectionId::new(401),
12894 hello_with_flow_operations(capabilities, Some(flow_operations)),
12895 )
12896 .unwrap();
12897 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12898 let registered = registry
12899 .get_module_by_connection(ConnectionId::new(401))
12900 .unwrap()
12901 .expect("the module still registers")
12902 .manifest;
12903 assert_eq!(registered.module_id, "flow-ops");
12904 assert_eq!(registered.flow_operations, None);
12905 let (raw, entry) = role_declarations_catalog(&handler);
12906 assert_eq!(raw["module_id"], "flow-ops");
12907 assert!(raw.get("flow_operations").is_none());
12908 assert_eq!(entry.flow_operations, None);
12909 let warnings: Vec<_> = capture
12910 .events()
12911 .into_iter()
12912 .filter(|event| {
12913 event.fields.get("message").map(String::as_str)
12914 == Some("dropping flow_operations from HELLO")
12915 })
12916 .collect();
12917 assert_eq!(warnings.len(), 1);
12918 assert_eq!(warnings[0].level, tracing::Level::WARN);
12919 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
12920 assert_eq!(warnings[0].fields["reason"], reason);
12921 }
12922
12923 #[test]
12924 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
12925 assert_hello_drops_flow_operations_with_warning(
12926 json!(["flow-scopes/v1"]),
12927 json!(["diagnostics.findings", "missing"]),
12928 "flow_operations names 'missing', which is not a declared management operation",
12929 );
12930 }
12931
12932 #[test]
12933 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
12934 assert_hello_drops_flow_operations_with_warning(
12935 json!([]),
12936 json!(["diagnostics.findings"]),
12937 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12938 );
12939 assert_hello_drops_flow_operations_with_warning(
12941 json!(["agent-run-scopes/v1"]),
12942 json!([]),
12943 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12944 );
12945 }
12946
12947 #[test]
12948 fn hello_malformed_flow_operations_is_dropped_with_warning() {
12949 assert_hello_drops_flow_operations_with_warning(
12950 json!(["flow-scopes/v1"]),
12951 json!(["reset", "reset"]),
12952 "flow_operations names 'reset' more than once",
12953 );
12954 assert_hello_drops_flow_operations_with_warning(
12955 json!(["flow-scopes/v1"]),
12956 json!(vec!["reset"; 257]),
12957 "flow_operations lists 257 names, more than 256",
12958 );
12959 assert_hello_drops_flow_operations_with_warning(
12960 json!(["flow-scopes/v1"]),
12961 json!("reset"),
12962 "flow_operations must be an array of operation names",
12963 );
12964 }
12965
12966 #[test]
12967 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
12968 let registry = Arc::new(Registry::default());
12969 let handler = ControlHandler::new(registry.clone());
12970 let descriptor = json!({
12971 "schema": "ckios.richui.manifest/1",
12972 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12973 "bytes": 128,
12974 "chunks": 2
12975 });
12976 let malformed = [
12977 ("schema", json!("")),
12978 ("schema", json!("a".repeat(65))),
12979 ("schema", json!("Bad schema")),
12980 (
12981 "manifest_digest",
12982 json!(format!("sha256:{}", "A".repeat(64))),
12983 ),
12984 ("manifest_digest", json!("sha256:abc")),
12985 ("bytes", json!(0)),
12986 ("bytes", json!(16_777_217)),
12987 ("bytes", json!("128")),
12988 ("chunks", json!(0)),
12989 ("chunks", json!(1025)),
12990 ("chunks", json!("2")),
12991 ];
12992 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
12993 let mut invalid = descriptor.clone();
12994 invalid[field] = invalid_value;
12995 let response = handler
12996 .handle_control(
12997 ConnectionId::new(210 + index as u64),
12998 hello_frame_with_rich_ui(
12999 &format!("bad-rich-ui-{index}"),
13000 PROTOCOL_VERSION,
13001 210 + index as u64,
13002 None,
13003 invalid,
13004 ),
13005 )
13006 .unwrap();
13007 assert_eq!(response[0].header.ty, FrameType::Error);
13008 let error = parse_error(&response[0]);
13009 assert_eq!(error["code"], "invalid_manifest");
13010 assert!(error["message"]
13011 .as_str()
13012 .unwrap()
13013 .contains(&format!("rich_ui.{field}")));
13014 assert!(registry
13015 .get_module(&format!("bad-rich-ui-{index}"))
13016 .unwrap()
13017 .is_none());
13018 }
13019 }
13020
13021 #[test]
13022 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
13023 let registry = Arc::new(Registry::default());
13024 let handler = ControlHandler::new(Arc::clone(®istry));
13025 let connection = ConnectionId::new(219);
13026 registry
13027 .register_with_control_ops(
13028 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
13029 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
13030 "bad schema",
13031 format!("sha256:{}", "a".repeat(64)),
13032 128,
13033 2,
13034 )))
13035 .build(),
13036 PROTOCOL_VERSION,
13037 connection,
13038 Vec::new(),
13039 )
13040 .unwrap();
13041 let request =
13042 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
13043 let response = handler
13044 .handle_catalog_update(connection, request, Vec::new(), None, None)
13045 .unwrap();
13046 assert_eq!(response[0].header.ty, FrameType::Error);
13047 let error = parse_error(&response[0]);
13048 assert_eq!(error["code"], "invalid_manifest");
13049 assert!(error["message"]
13050 .as_str()
13051 .unwrap()
13052 .contains("rich_ui.schema"));
13053 }
13054
13055 #[test]
13056 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
13057 let registry = Arc::new(Registry::default());
13058 let supervisor = SupervisorHandle::new();
13059 supervisor.open_swap("provider", "candidate-nonce".to_string());
13060 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13061 let connection = ConnectionId::new(218);
13062 let hello = handler
13063 .handle_control(
13064 connection,
13065 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
13066 )
13067 .unwrap();
13068 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13069 assert!(registry.list_modules().unwrap().1.is_empty());
13070
13071 registry.promote_candidate("provider").unwrap().unwrap();
13072 let request = Frame::build(
13073 FrameType::Request,
13074 control_flags(),
13075 0,
13076 0,
13077 219,
13078 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13079 )
13080 .unwrap();
13081 let response = handler.handle_catalog_list(request, None).unwrap();
13082 let ClientControlResponse::CatalogList { modules, .. } =
13083 serde_json::from_slice(&response[0].body).unwrap()
13084 else {
13085 panic!("expected catalog.list");
13086 };
13087 assert_eq!(modules.len(), 1);
13088 assert!(modules[0].nonce_verified);
13089 }
13090
13091 #[test]
13092 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
13093 for provenance_json in [
13094 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
13095 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
13096 ] {
13097 let registry = Arc::new(Registry::default());
13098 let handler = ControlHandler::new(registry);
13099 let hello = Frame::build(
13100 FrameType::Hello,
13101 control_flags(),
13102 0,
13103 0,
13104 220,
13105 serde_json::to_vec(&ModuleHelloBody {
13106 manifest: ModuleManifest::builder("built", "0.1.0")
13107 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
13108 .build(),
13109 protocol_ver: PROTOCOL_VERSION,
13110 control_ops: None,
13111 launch_nonce: None,
13112 })
13113 .unwrap(),
13114 )
13115 .unwrap();
13116 assert_eq!(
13117 handler
13118 .handle_control(ConnectionId::new(220), hello)
13119 .unwrap()[0]
13120 .header
13121 .ty,
13122 FrameType::HelloAck
13123 );
13124 let request = Frame::build(
13125 FrameType::Request,
13126 control_flags(),
13127 0,
13128 0,
13129 221,
13130 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13131 )
13132 .unwrap();
13133 let response = handler.handle_catalog_list(request, None).unwrap();
13134 let ClientControlResponse::CatalogList { modules, .. } =
13135 serde_json::from_slice(&response[0].body).unwrap()
13136 else {
13137 panic!("expected catalog.list");
13138 };
13139 assert_eq!(modules.len(), 1);
13140 assert_eq!(
13141 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13142 format!(
13143 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
13144 )
13145 .as_bytes()
13146 );
13147 }
13148 }
13149
13150 #[test]
13151 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
13152 let registry = Arc::new(Registry::default());
13153 let handler = ControlHandler::new(Arc::clone(®istry));
13154 let hello = Frame::build(
13155 FrameType::Hello,
13156 control_flags(),
13157 0,
13158 0,
13159 220,
13160 serde_json::to_vec(&ModuleHelloBody {
13161 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
13162 protocol_ver: PROTOCOL_VERSION,
13163 control_ops: None,
13164 launch_nonce: Some("unverified".to_string()),
13165 })
13166 .unwrap(),
13167 )
13168 .unwrap();
13169 assert_eq!(
13170 handler
13171 .handle_control(ConnectionId::new(220), hello)
13172 .unwrap()[0]
13173 .header
13174 .ty,
13175 FrameType::HelloAck
13176 );
13177 let request = Frame::build(
13178 FrameType::Request,
13179 control_flags(),
13180 0,
13181 0,
13182 221,
13183 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13184 )
13185 .unwrap();
13186 let response = handler.handle_catalog_list(request, None).unwrap();
13187 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
13188 assert!(wire["modules"][0].get("provenance").is_none());
13189 let ClientControlResponse::CatalogList { modules, .. } =
13190 serde_json::from_slice(&response[0].body).unwrap()
13191 else {
13192 panic!("expected catalog.list");
13193 };
13194 assert_eq!(
13195 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13196 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
13197 );
13198 assert!(!modules[0].nonce_verified);
13199 assert!(modules[0].provenance.is_none());
13200 }
13201
13202 fn catalog_legacy_projection(mut entry: CatalogEntry) -> CatalogEntry {
13204 assert_eq!(entry.first_party, Some(false));
13205 entry.incarnation = None;
13206 entry.spawn = None;
13207 entry.first_party = None;
13208 entry.origin = None;
13209 entry.requested_declarations_digest = None;
13210 entry.approved_declarations_digest = None;
13211 entry
13212 }
13213
13214 #[test]
13215 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
13216 let registry = Arc::new(Registry::default());
13217 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13218 [("vault".to_string(), true), ("squatter".to_string(), true)],
13219 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13220 );
13221 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
13222 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13223 provides: vec!["credentials-provider/v1".to_string()],
13224 requires: Vec::new(),
13225 must_never_reach: Vec::new(),
13226 });
13227 let frame = Frame::build(
13228 FrameType::Hello,
13229 control_flags(),
13230 0,
13231 0,
13232 77,
13233 serde_json::to_vec(&ModuleHelloBody {
13234 manifest: squatter,
13235 protocol_ver: PROTOCOL_VERSION,
13236 control_ops: None,
13237 launch_nonce: None,
13238 })
13239 .expect("HELLO serializes"),
13240 )
13241 .expect("HELLO frame builds");
13242 let response = handler
13243 .handle_control(ConnectionId::new(77), frame)
13244 .expect("reserved claim receives a typed refusal");
13245 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13246 assert_eq!(
13247 registry
13248 .active_registration_count()
13249 .expect("registry reads"),
13250 0,
13251 "a reserved capability refusal must not leave a catalog entry"
13252 );
13253 }
13254
13255 #[test]
13256 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
13257 for settlement in ["timeout", "inconclusive", "drop"] {
13258 let breakers = RouteBindBreakers::default();
13259 let RouteBindAdmission::Admitted {
13260 guard: mut old,
13261 probe: false,
13262 } = breakers.admit("prov")
13263 else {
13264 panic!("ordinary relay admitted")
13265 };
13266 let RouteBindAdmission::Admitted {
13267 guard: mut opener, ..
13268 } = breakers.admit("prov")
13269 else {
13270 panic!("second relay admitted")
13271 };
13272 assert!(
13273 !opener
13274 .record_timeout(1, Duration::ZERO)
13275 .unwrap()
13276 .reopened_after_probe
13277 );
13278 let RouteBindAdmission::Admitted {
13279 guard: mut probe,
13280 probe: true,
13281 } = breakers.admit("prov")
13282 else {
13283 panic!("one cooldown probe admitted")
13284 };
13285 match settlement {
13286 "timeout" => assert!(
13287 !old.record_timeout(1, Duration::ZERO)
13288 .unwrap()
13289 .reopened_after_probe
13290 ),
13291 "inconclusive" => old.record_inconclusive(),
13292 "drop" => drop(old),
13293 _ => unreachable!(),
13294 }
13295 assert!(
13296 matches!(
13297 breakers.admit("prov"),
13298 RouteBindAdmission::Refused {
13299 probe_in_flight: true,
13300 ..
13301 }
13302 ),
13303 "{settlement} of a pre-open relay cannot release the real probe"
13304 );
13305 assert!(
13306 probe
13307 .record_timeout(1, Duration::ZERO)
13308 .unwrap()
13309 .reopened_after_probe
13310 );
13311 assert!(matches!(
13312 breakers.admit("prov"),
13313 RouteBindAdmission::Admitted { probe: true, .. }
13314 ));
13315 }
13316 let breakers = RouteBindBreakers::default();
13317 let admit = || match breakers.admit("prov") {
13318 RouteBindAdmission::Admitted { guard, .. } => guard,
13319 _ => panic!("relay admitted"),
13320 };
13321 admit().record_timeout(1, Duration::ZERO);
13322 let mut old_probe = admit();
13323 breakers.reset_for_new_module_connection("prov");
13324 admit().record_timeout(1, Duration::ZERO);
13325 let _new_probe = admit();
13326 old_probe.record_inconclusive();
13327 assert!(matches!(
13328 breakers.admit("prov"),
13329 RouteBindAdmission::Refused {
13330 probe_in_flight: true,
13331 ..
13332 }
13333 ));
13334 }
13335
13336 #[tokio::test]
13337 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
13338 for candidate in [false, true] {
13339 let registry = Arc::new(Registry::default());
13340 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13341 [("vault".to_string(), true), ("squatter".to_string(), true)],
13342 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13343 );
13344 let conn = ConnectionId::new(77);
13345 let (ctx, mut rx) = route_ctx(conn);
13346 let initial = capability_manifest("squatter", &[], &[]);
13347 if candidate {
13348 registry
13349 .register_candidate_with_control_ops(
13350 initial.clone(),
13351 PROTOCOL_VERSION,
13352 conn,
13353 module_baseline_control_ops(),
13354 )
13355 .unwrap();
13356 handler
13357 .forwarding
13358 .register_candidate_module_connection(
13359 conn,
13360 "squatter".to_string(),
13361 PROTOCOL_VERSION,
13362 manifest_concurrency(&initial),
13363 ctx.egress.clone(),
13364 )
13365 .unwrap();
13366 } else {
13367 hello_via_sink(
13368 &handler,
13369 &ctx,
13370 &mut rx,
13371 hello_frame_with_manifest(initial.clone(), 1),
13372 )
13373 .await;
13374 }
13375 let response = handler
13376 .handle_control_frame(
13377 &ctx,
13378 catalog_update_with_capabilities_frame(
13379 2,
13380 capability_manifest("squatter", &["credentials-provider/v1"], &[])
13381 .capabilities
13382 .unwrap(),
13383 ),
13384 )
13385 .await
13386 .unwrap();
13387 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13388 assert_eq!(
13389 registry
13390 .get_module_by_connection(conn)
13391 .unwrap()
13392 .unwrap()
13393 .manifest,
13394 initial
13395 );
13396 }
13397 }
13398
13399 #[test]
13400 #[cfg(not(feature = "dev-desktop-approval"))]
13401 fn server_describe_default_build_advertises_no_config_capability() {
13402 let assert_default = |handler: &ControlHandler| {
13403 let request = Frame::build(
13404 FrameType::Request,
13405 control_flags(),
13406 0,
13407 0,
13408 79,
13409 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13410 )
13411 .unwrap();
13412 let responses = handler.handle_server_describe(request).unwrap();
13413 assert_eq!(responses.len(), 1);
13414 let response: Value = serde_json::from_slice(&responses[0].body).unwrap();
13415 assert_eq!(response["op"], "server.describe");
13416 let capabilities = response["capabilities"]
13417 .as_array()
13418 .expect("capability array");
13419 assert!(
13420 !capabilities.is_empty(),
13421 "exercise an actual describe response, not an empty fixture"
13422 );
13423 assert!(
13424 capabilities
13425 .iter()
13426 .all(|capability| !capability.as_str().unwrap().starts_with("config.")),
13427 "a default build must not announce configuration authority: {capabilities:?}"
13428 );
13429 };
13430 assert_default(&ControlHandler::new(Arc::new(Registry::default())));
13431 #[cfg(unix)]
13432 {
13433 let (_rig, _owner, _supervisor, handler, _spec) =
13437 crate::config_gate::runtime::RuntimeOwner::unactivated_supervisor_fixture_for_test(
13438 );
13439 assert_default(&handler);
13440 }
13441 }
13442
13443 #[test]
13444 fn server_describe_surfaces_required_capability_verdict_fields() {
13445 let registry = Arc::new(Registry::default());
13446 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13447 [
13448 ("consumer".to_string(), true),
13449 ("provider".to_string(), false),
13450 ],
13451 BTreeMap::new(),
13452 );
13453 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13454 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13455 provides: Vec::new(),
13456 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13457 capability: "credentials-provider/v1".to_string(),
13458 need: subc_protocol::manifest::CapabilityNeed::Required,
13459 }],
13460 must_never_reach: Vec::new(),
13461 });
13462 let hello = Frame::build(
13463 FrameType::Hello,
13464 control_flags(),
13465 0,
13466 0,
13467 78,
13468 serde_json::to_vec(&ModuleHelloBody {
13469 manifest: consumer,
13470 protocol_ver: PROTOCOL_VERSION,
13471 control_ops: None,
13472 launch_nonce: None,
13473 })
13474 .expect("HELLO serializes"),
13475 )
13476 .expect("HELLO frame builds");
13477 handler
13478 .handle_control(ConnectionId::new(78), hello)
13479 .expect("consumer registers");
13480 let describe = Frame::build(
13481 FrameType::Request,
13482 control_flags(),
13483 0,
13484 0,
13485 79,
13486 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
13487 .expect("request serializes"),
13488 )
13489 .expect("describe frame builds");
13490 let response = handler
13491 .handle_server_describe(describe)
13492 .expect("server.describe succeeds");
13493 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
13494 let requirement = &rendered["capability_requirements"][0];
13495 assert_eq!(requirement["consumer"], "consumer");
13496 assert_eq!(requirement["verdict"], "never_provided");
13497 assert_eq!(requirement["episode_seq"], 1);
13498 assert_eq!(requirement["config_satisfiable"], false);
13499 assert_eq!(requirement["runtime_available"], false);
13500 assert!(requirement["detail"]
13501 .as_str()
13502 .expect("detail string")
13503 .contains("credentials-provider/v1"));
13504 }
13505
13506 fn deregister_required_provider(
13512 daemon_shutdown_begun: bool,
13513 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
13514 const CAPABILITY: &str = "project-identity/v1";
13515 let registry = Arc::new(Registry::default());
13516 let daemon_shutdown = DaemonShutdownFlag::default();
13517 let handler = ControlHandler::new(Arc::clone(®istry))
13518 .with_capability_config(
13519 [
13520 ("consumer".to_string(), true),
13521 ("provider".to_string(), true),
13522 ],
13523 BTreeMap::new(),
13524 )
13525 .with_daemon_shutdown_flag(daemon_shutdown.clone());
13526 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13527 consumer.capabilities = Some(CapabilityDeclarations {
13528 provides: Vec::new(),
13529 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13530 capability: CAPABILITY.to_string(),
13531 need: subc_protocol::manifest::CapabilityNeed::Required,
13532 }],
13533 must_never_reach: Vec::new(),
13534 });
13535 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
13536 let provider_conn = ConnectionId::new(182);
13537 for (conn, manifest, corr) in [
13538 (ConnectionId::new(181), consumer, 181),
13539 (provider_conn, provider, 182),
13540 ] {
13541 let hello = handler
13542 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
13543 .expect("module registers");
13544 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13545 }
13546 assert_eq!(
13547 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13548 Some(CapabilityVerdict::Provided),
13549 "the registered provider must satisfy the requirement first"
13550 );
13551 if daemon_shutdown_begun {
13552 daemon_shutdown.set_for_test();
13553 }
13554
13555 let capture = EventCapture::default();
13556 let _subscriber =
13557 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13558 handler
13559 .cleanup_connection(provider_conn)
13560 .expect("provider deregisters");
13561 let events = capture
13562 .events()
13563 .into_iter()
13564 .filter(|event| {
13565 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
13566 })
13567 .collect();
13568 (
13569 events,
13570 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13571 )
13572 }
13573
13574 #[test]
13575 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
13576 let (events, verdict) = deregister_required_provider(true);
13577 assert_eq!(
13578 verdict,
13579 Some(CapabilityVerdict::NeverProvided),
13580 "shutdown must not change the verdict route.open refusals read"
13581 );
13582 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13583 let event = &events[0];
13584 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
13585 assert_eq!(
13586 event.fields.get("cause").map(String::as_str),
13587 Some("\"daemon_shutdown\""),
13588 "{event:?}"
13589 );
13590 assert_eq!(
13591 event.fields.get("verdict").map(String::as_str),
13592 Some("\"never_provided\""),
13593 "{event:?}"
13594 );
13595 }
13596
13597 #[test]
13598 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
13599 let (events, verdict) = deregister_required_provider(false);
13600 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
13601 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13602 let event = &events[0];
13603 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
13604 assert_eq!(event.fields.get("cause"), None, "{event:?}");
13605 assert_eq!(
13606 event.fields.get("verdict").map(String::as_str),
13607 Some("\"never_provided\""),
13608 "{event:?}"
13609 );
13610 }
13611
13612 #[test]
13613 fn catalog_list_omits_capabilities_for_legacy_manifest() {
13614 let registry = Arc::new(Registry::default());
13615 let handler = ControlHandler::new(Arc::clone(®istry));
13616 let hello = handler
13617 .handle_control(
13618 ConnectionId::new(101),
13619 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
13620 )
13621 .expect("legacy HELLO registers");
13622 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13623
13624 let request = Frame::build(
13625 FrameType::Request,
13626 control_flags(),
13627 0,
13628 0,
13629 102,
13630 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
13631 .expect("catalog request serializes"),
13632 )
13633 .expect("catalog request frame builds");
13634 let response = handler
13635 .handle_catalog_list(request, None)
13636 .expect("catalog list succeeds");
13637 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
13638 assert!(
13639 body["modules"][0].get("capabilities").is_none(),
13640 "legacy manifest must retain an absent capabilities field on catalog.list"
13641 );
13642 }
13643
13644 #[test]
13645 fn hello_ack_omits_storage_when_no_storage_config() {
13646 let registry = Arc::new(Registry::default());
13647 let handler = ControlHandler::new(Arc::clone(®istry));
13648 let responses = handler
13649 .handle_control(
13650 ConnectionId::new(1),
13651 hello_frame("aft", PROTOCOL_VERSION, 7),
13652 )
13653 .unwrap();
13654 let ack = parse_ack(&responses[0]);
13655 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
13656 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
13657 }
13658
13659 #[tokio::test]
13660 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
13661 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
13662 let registry = Arc::new(Registry::default());
13663 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
13664 let responses = handler
13665 .handle_control(
13666 ConnectionId::new(1),
13667 hello_frame("aft", PROTOCOL_VERSION, 7),
13668 )
13669 .unwrap();
13670 let ack = parse_ack(&responses[0]);
13671 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
13672
13673 let described = handler
13674 .handle_control_frame(
13675 &route_ctx(ConnectionId::new(2)).0,
13676 Frame::build(
13677 FrameType::Request,
13678 control_flags(),
13679 0,
13680 0,
13681 9,
13682 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13683 )
13684 .unwrap(),
13685 )
13686 .await
13687 .unwrap();
13688 let ClientControlResponse::ServerDescribe { machine_id, .. } =
13689 serde_json::from_slice(&described[0].body).unwrap()
13690 else {
13691 panic!("server.describe answered with another shape");
13692 };
13693 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
13694 }
13695
13696 #[test]
13697 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
13698 let registry = Arc::new(Registry::default());
13701 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
13702 crate::daemon_config::StorageConfig::Sqlite {
13703 data_home: std::path::PathBuf::from("/data"),
13704 },
13705 ));
13706
13707 let responses = handler
13708 .handle_control(
13709 ConnectionId::new(1),
13710 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
13711 )
13712 .unwrap();
13713 let ack = parse_ack(&responses[0]);
13714 assert_eq!(
13715 ack.storage,
13716 Some(serde_json::json!({
13717 "module_id": "alfonso-routing",
13718 "storage_namespace": "default",
13719 "isolation": { "kind": "module" },
13720 "backend": {
13721 "backend": "sqlite",
13722 "path": "/data/cortexkit/alfonso-routing/store.db"
13723 }
13724 })),
13725 "the delivered descriptor is the module's own sqlite store path"
13726 );
13727 }
13728
13729 #[test]
13730 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
13731 let registry = Arc::new(Registry::default());
13732 let handler = ControlHandler::new(Arc::clone(®istry));
13733 let conn = ConnectionId::new(1);
13734 let responses = handler
13735 .handle_control(
13736 conn,
13737 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13738 )
13739 .unwrap();
13740 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
13741 let registration = registry.get_module("aft").unwrap().unwrap();
13742 assert_eq!(registration.control_ops, module_baseline_control_ops());
13743
13744 let frame =
13745 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
13746 assert!(handler
13747 .guard_module_control_op(&frame, "aft", "route.bind")
13748 .unwrap()
13749 .is_none());
13750 let error = handler
13751 .guard_module_control_op(&frame, "aft", "test.synthetic")
13752 .unwrap()
13753 .expect("synthetic ungranted op should be rejected");
13754 assert_eq!(error.header.ty, FrameType::Error);
13755 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
13756 }
13757
13758 #[test]
13759 fn hello_control_ops_some_adds_optional_grants() {
13760 let registry = Arc::new(Registry::default());
13761 let handler = ControlHandler::new(Arc::clone(®istry));
13762 handler
13763 .handle_control(
13764 ConnectionId::new(1),
13765 hello_frame_with_control_ops(
13766 "aft",
13767 PROTOCOL_VERSION,
13768 7,
13769 Some(vec![
13770 "future.synthetic".to_string(),
13771 "route.bind".to_string(),
13772 ]),
13773 ),
13774 )
13775 .unwrap();
13776 let registration = registry.get_module("aft").unwrap().unwrap();
13777 assert_eq!(
13778 registration.control_ops,
13779 vec![
13780 "route.bind".to_string(),
13781 "route.status".to_string(),
13782 "future.synthetic".to_string(),
13783 ]
13784 );
13785 let frame =
13786 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
13787 assert!(handler
13788 .guard_module_control_op(&frame, "aft", "future.synthetic")
13789 .unwrap()
13790 .is_none());
13791 }
13792
13793 #[tokio::test]
13794 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
13795 let registry = Arc::new(Registry::default());
13796 let forwarding = Arc::new(ForwardingTable::default());
13797 let handler =
13798 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13799 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
13800 hello_via_sink(
13801 &handler,
13802 &module_ctx,
13803 &mut module_rx,
13804 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13805 )
13806 .await;
13807
13808 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
13809 let responses = handler
13810 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
13811 .await
13812 .unwrap();
13813 assert_eq!(responses.len(), 1);
13814 assert_eq!(responses[0].header.ty, FrameType::Error);
13815 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
13816 assert!(module_rx.try_recv().is_err());
13817 }
13818
13819 #[tokio::test]
13820 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
13821 let registry = Arc::new(Registry::default());
13822 let forwarding = Arc::new(ForwardingTable::default());
13823 let handler =
13824 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13825 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
13826 hello_via_sink(
13827 &handler,
13828 &module_ctx,
13829 &mut module_rx,
13830 hello_frame_with_control_ops(
13831 "aft",
13832 PROTOCOL_VERSION,
13833 7,
13834 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13835 ),
13836 )
13837 .await;
13838
13839 let project_root = unique_project_root("demux");
13840 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
13841 let route_handler = handler.clone();
13842 let route_task = tokio::spawn(async move {
13843 route_handler
13844 .handle_control_frame(
13845 &route_client_ctx,
13846 route_open_frame(100, "aft", project_root),
13847 )
13848 .await
13849 .unwrap()
13850 });
13851 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13852 .await
13853 .unwrap()
13854 .unwrap();
13855 assert!(matches!(
13856 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
13857 ModuleControlRequest::RouteBind { .. }
13858 ));
13859
13860 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
13861 let health_handler = handler.clone();
13862 let health_task = tokio::spawn(async move {
13863 health_handler
13864 .handle_control_frame(
13865 &health_client_ctx,
13866 supervisor_health_probe_frame(101, "aft"),
13867 )
13868 .await
13869 .unwrap()
13870 });
13871 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13872 .await
13873 .unwrap()
13874 .unwrap();
13875 assert_eq!(
13876 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13877 ModuleControlRequest::HealthCheck {}
13878 );
13879
13880 handler
13881 .handle_control_frame(
13882 &module_ctx,
13883 health_response(health_frame.header.corr, HealthStatus::Degraded),
13884 )
13885 .await
13886 .unwrap();
13887 let health_response = health_task.await.unwrap();
13888 assert_eq!(health_response.len(), 1);
13889 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13890 ClientControlResponse::SupervisorHealthProbe {
13891 module_id,
13892 status,
13893 detail,
13894 metrics,
13895 } => {
13896 assert_eq!(module_id, "aft");
13897 assert_eq!(status, HealthStatus::Degraded);
13898 assert_eq!(detail.as_deref(), Some("warming"));
13899 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
13900 }
13901 other => panic!("unexpected health response: {other:?}"),
13902 }
13903
13904 handler
13905 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13906 .await
13907 .unwrap();
13908 let route_response = route_task.await.unwrap();
13909 assert!(route_response.is_empty());
13910 let published = route_client_rx.recv().await.unwrap();
13911 assert!(matches!(
13912 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13913 ClientControlResponse::RouteOpen { .. }
13914 ));
13915 }
13916
13917 async fn relay_route_open(
13922 handler: &ControlHandler,
13923 client_connection: ConnectionId,
13924 client_egress: &FrameSink,
13925 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13926 corr: u64,
13927 module_id: &str,
13928 project_root_label: &str,
13929 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
13930 let ctx = RouteCtx {
13931 connection_id: client_connection,
13932 egress: client_egress.clone(),
13933 };
13934 let handler = handler.clone();
13935 let project_root = unique_project_root(project_root_label);
13936 let module_id = module_id.to_string();
13937 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
13938 let task = tokio::spawn(async move {
13939 let _guard = tracing::dispatcher::set_default(&dispatch);
13940 handler
13941 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
13942 .await
13943 .unwrap()
13944 });
13945 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13946 .await
13947 .expect("module receives the relayed route.bind")
13948 .expect("module egress is open");
13949 (task, bind.frame)
13950 }
13951
13952 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
13953 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
13954 ModuleControlRequest::RouteBind {
13955 route_channel,
13956 epoch,
13957 ..
13958 } => (route_channel, epoch),
13959 other => panic!("expected a route.bind request, got {other:?}"),
13960 }
13961 }
13962
13963 fn published_route(frame: &Frame) -> (u16, u32) {
13964 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
13965 ClientControlResponse::RouteOpen {
13966 route_channel,
13967 route_epoch,
13968 ..
13969 } => (route_channel, route_epoch),
13970 other => panic!("expected a route.open response, got {other:?}"),
13971 }
13972 }
13973
13974 #[tokio::test]
13994 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
13995 let registry = Arc::new(Registry::default());
13996 let forwarding = Arc::new(ForwardingTable::default());
13997 let handler =
13998 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13999
14000 let module_connection = ConnectionId::new(30);
14001 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14002 hello_via_sink(
14003 &handler,
14004 &module_ctx,
14005 &mut module_rx,
14006 hello_frame("aft", PROTOCOL_VERSION, 7),
14007 )
14008 .await;
14009
14010 let dying_client = ConnectionId::new(31);
14011 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
14012
14013 let (first_task, first_bind) = relay_route_open(
14016 &handler,
14017 dying_client,
14018 &dying_ctx.egress,
14019 &mut module_rx,
14020 100,
14021 "aft",
14022 "closing-first",
14023 )
14024 .await;
14025 handler
14026 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
14027 .await
14028 .unwrap();
14029 assert!(first_task.await.unwrap().is_empty());
14030 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
14031
14032 let (second_task, second_bind) = relay_route_open(
14034 &handler,
14035 dying_client,
14036 &dying_ctx.egress,
14037 &mut module_rx,
14038 101,
14039 "aft",
14040 "closing-second",
14041 )
14042 .await;
14043 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
14044
14045 assert!(forwarding
14048 .escalate_client_delivery_failure(
14049 dying_client,
14050 first_channel,
14051 first_epoch,
14052 CloseReason::new(
14053 "module_to_client_delivery_failed",
14054 "client egress refused a module frame",
14055 ),
14056 crate::forwarding::UndeliveredFrame {
14057 module_id: None,
14058 sink: &dying_ctx.egress,
14059 },
14060 )
14061 .unwrap());
14062 assert!(!dying_ctx.egress.is_closed());
14063
14064 let ack = handler
14066 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
14067 .await;
14068 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14069 if module_loop_error.is_some() {
14070 handler.cleanup_connection(module_connection).unwrap();
14074 }
14075 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14079 .await
14080 .ok()
14081 .flatten();
14082
14083 assert!(
14085 registry
14086 .get_module_by_connection(module_connection)
14087 .unwrap()
14088 .is_some(),
14089 "one client's closing connection ended the shared module connection: \
14090 {module_loop_error:?}"
14091 );
14092 let cotenant = ConnectionId::new(32);
14094 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14095 let (cotenant_task, cotenant_bind) = relay_route_open(
14096 &handler,
14097 cotenant,
14098 &cotenant_ctx.egress,
14099 &mut module_rx,
14100 102,
14101 "aft",
14102 "closing-cotenant",
14103 )
14104 .await;
14105 handler
14106 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
14107 .await
14108 .unwrap();
14109 assert!(cotenant_task.await.unwrap().is_empty());
14110 let (cotenant_channel, cotenant_epoch) =
14111 published_route(&cotenant_rx.recv().await.unwrap());
14112 assert!(matches!(
14113 forwarding
14114 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14115 .unwrap(),
14116 DataRoute::Client(DataRouteState::Bound(_))
14117 ));
14118
14119 let goodbye = post_ack_module_frame
14122 .expect("module receives a GOODBYE for the abandoned route channel");
14123 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14124 assert_eq!(goodbye.header.channel, abandoned_channel);
14125 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14126
14127 assert!(dying_rx.try_recv().is_err());
14131 let second_response = second_task.await.unwrap();
14132 assert_eq!(second_response.len(), 1);
14133 assert_eq!(
14134 parse_error(&second_response[0])["code"],
14135 "target_unavailable"
14136 );
14137 }
14138
14139 async fn catalog_snapshot(
14142 handler: &ControlHandler,
14143 client_ctx: &RouteCtx,
14144 corr: u64,
14145 ) -> (u64, Vec<(String, bool)>) {
14146 let body =
14147 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
14148 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
14149 let reply = handler
14150 .handle_control_frame(client_ctx, request)
14151 .await
14152 .unwrap()
14153 .pop()
14154 .unwrap();
14155 let ClientControlResponse::CatalogList {
14156 generation,
14157 modules,
14158 ..
14159 } = serde_json::from_slice(&reply.body).unwrap()
14160 else {
14161 panic!("not a catalog.list reply");
14162 };
14163 (
14164 generation,
14165 modules
14166 .into_iter()
14167 .map(|entry| (entry.module_id, entry.ready))
14168 .collect(),
14169 )
14170 }
14171
14172 fn hello_held_before_forwarding() -> (
14178 ControlHandler,
14179 Arc<std::sync::Barrier>,
14180 std::thread::JoinHandle<Vec<Frame>>,
14181 RouteCtx,
14182 mpsc::Receiver<crate::router::OutboundFrame>,
14183 ) {
14184 let hold = Arc::new(std::sync::Barrier::new(2));
14185 let handler = ControlHandler::with_forwarding(
14186 Arc::new(Registry::default()),
14187 Arc::new(ForwardingTable::default()),
14188 )
14189 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14190 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14191 let hello = {
14192 let handler = handler.clone();
14193 let module_ctx = module_ctx.clone();
14194 std::thread::spawn(move || {
14195 handler
14196 .handle_hello(
14197 module_ctx.connection_id,
14198 Some(module_ctx.egress.clone()),
14199 hello_frame("aft", PROTOCOL_VERSION, 7),
14200 )
14201 .unwrap()
14202 })
14203 };
14204 (handler, hold, hello, module_ctx, module_rx)
14205 }
14206
14207 fn lookup_during_hello_publication<T: Send + 'static>(
14213 registry: &Registry,
14214 hold: &std::sync::Barrier,
14215 lookup: impl FnOnce() -> T + Send + 'static,
14216 ) -> T {
14217 let attempted = registry.observe_next_lock_attempt();
14218 let reader = std::thread::spawn(lookup);
14219 if attempted.recv().unwrap() {
14220 let result = reader.join().unwrap();
14221 hold.wait();
14222 result
14223 } else {
14224 hold.wait();
14225 reader.join().unwrap()
14226 }
14227 }
14228
14229 #[test]
14230 fn catalog_read_after_hello_ack_waits_for_publication() {
14231 let registry = Arc::new(Registry::default());
14232 let hold = Arc::new(std::sync::Barrier::new(2));
14233 let mut handler = ControlHandler::with_forwarding(
14234 Arc::clone(®istry),
14235 Arc::new(ForwardingTable::default()),
14236 );
14237 handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
14238 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14239 let hello = {
14240 let handler = handler.clone();
14241 std::thread::spawn(move || {
14242 handler
14243 .handle_hello(
14244 module_ctx.connection_id,
14245 Some(module_ctx.egress),
14246 hello_frame("aft", PROTOCOL_VERSION, 7),
14247 )
14248 .unwrap()
14249 })
14250 };
14251 hold.wait();
14252 assert_eq!(
14253 module_rx.try_recv().unwrap().frame.header.ty,
14254 FrameType::HelloAck
14255 );
14256 let replies = lookup_during_hello_publication(®istry, &hold, move || {
14257 let request = Frame::build(
14260 FrameType::Request,
14261 control_flags(),
14262 0,
14263 0,
14264 300,
14265 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
14266 )
14267 .unwrap();
14268 handler.handle_catalog_list(request, None).unwrap()
14269 });
14270 assert!(hello.join().unwrap().is_empty());
14271 let ClientControlResponse::CatalogList {
14272 generation,
14273 modules,
14274 ..
14275 } = serde_json::from_slice(&replies[0].body).unwrap()
14276 else {
14277 panic!("not a catalog.list reply");
14278 };
14279 assert_eq!(
14280 modules
14281 .iter()
14282 .map(|entry| entry.module_id.as_str())
14283 .collect::<Vec<_>>(),
14284 vec!["aft"]
14285 );
14286 assert_eq!(generation, 1);
14287 }
14288
14289 #[tokio::test]
14298 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
14299 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
14300 let client = ConnectionId::new(41);
14301 let (client_ctx, mut client_rx) = route_ctx(client);
14302
14303 hold.wait();
14304 assert!(handler.registry.get_module("aft").unwrap().is_some());
14306 assert!(!handler
14307 .forwarding
14308 .has_live_module_connection("aft")
14309 .unwrap());
14310 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
14311 assert!(
14312 listed.is_empty(),
14313 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
14314 );
14315 let refused = handler
14316 .handle_control_frame(
14317 &client_ctx,
14318 route_open_frame(301, "aft", unique_project_root("listed-window")),
14319 )
14320 .await
14321 .unwrap();
14322 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
14323 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
14324 let duplicate = handler
14325 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
14326 .await
14327 .unwrap();
14328 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14329
14330 hold.wait();
14331 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14332 let ack = module_rx.try_recv().unwrap().frame;
14333 assert_eq!(ack.header.ty, FrameType::HelloAck);
14334 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14335 assert_eq!(listed, vec![("aft".to_string(), true)]);
14336
14337 let (open_task, bind) = relay_route_open(
14338 &handler,
14339 client,
14340 &client_ctx.egress,
14341 &mut module_rx,
14342 303,
14343 "aft",
14344 "listed-after",
14345 )
14346 .await;
14347 assert!(handler
14348 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14349 .await
14350 .unwrap()
14351 .is_empty());
14352 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
14353 published_route(&client_rx.try_recv().unwrap().frame);
14354 }
14355
14356 #[tokio::test]
14363 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
14364 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
14365 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14366 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14367
14368 hold.wait();
14369 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
14370 assert!(listed.is_empty());
14371 assert_eq!(
14372 during, before,
14373 "the generation moved while the module was not listed"
14374 );
14375
14376 hold.wait();
14377 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14378 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14379 assert_eq!(listed, vec![("aft".to_string(), true)]);
14380 assert_eq!(
14381 after,
14382 before + 1,
14383 "the module appeared without exactly one generation bump"
14384 );
14385 }
14386
14387 #[tokio::test]
14392 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
14393 let registry = Arc::new(Registry::default());
14394 let handler = ControlHandler::with_forwarding(
14395 Arc::clone(®istry),
14396 Arc::new(ForwardingTable::default()),
14397 );
14398 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14399 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14400 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14401 drop(module_rx);
14402
14403 let replies = handler
14404 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
14405 .await
14406 .unwrap();
14407 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
14408 assert!(registry.get_module("aft").unwrap().is_none());
14409 assert!(registry
14410 .get_module_by_connection(module_ctx.connection_id)
14411 .unwrap()
14412 .is_none());
14413 assert_eq!(
14414 catalog_snapshot(&handler, &client_ctx, 301).await,
14415 (before, Vec::new())
14416 );
14417 }
14418
14419 async fn route_open_held_after_bind_send() -> (
14423 ControlHandler,
14424 Arc<tokio::sync::Barrier>,
14425 RouteCtx,
14426 mpsc::Receiver<crate::router::OutboundFrame>,
14427 ) {
14428 let hold = Arc::new(tokio::sync::Barrier::new(2));
14429 let handler = ControlHandler::with_forwarding(
14430 Arc::new(Registry::default()),
14431 Arc::new(ForwardingTable::default()),
14432 )
14433 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
14434 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14435 hello_via_sink(
14436 &handler,
14437 &module_ctx,
14438 &mut module_rx,
14439 hello_frame("aft", PROTOCOL_VERSION, 7),
14440 )
14441 .await;
14442 (handler, hold, module_ctx, module_rx)
14443 }
14444
14445 #[tokio::test]
14452 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
14453 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14454 let client = ConnectionId::new(41);
14455 let (client_ctx, mut client_rx) = route_ctx(client);
14456
14457 let (open_task, bind) = relay_route_open(
14458 &handler,
14459 client,
14460 &client_ctx.egress,
14461 &mut module_rx,
14462 200,
14463 "aft",
14464 "acked-before-mark",
14465 )
14466 .await;
14467 let (module_channel, module_epoch) = route_bind_channel(&bind);
14468 hold.wait().await;
14470
14471 assert!(handler
14473 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14474 .await
14475 .unwrap()
14476 .is_empty());
14477 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
14478
14479 hold.wait().await;
14480 assert!(
14481 open_task.await.unwrap().is_empty(),
14482 "route.open answers accepted"
14483 );
14484
14485 match module_rx.try_recv() {
14488 Err(mpsc::error::TryRecvError::Empty) => {}
14489 Ok(outbound) => panic!(
14490 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
14491 outbound.frame.header.ty,
14492 outbound.frame.header.channel,
14493 outbound.frame.header.epoch
14494 ),
14495 Err(err) => panic!("module egress closed: {err}"),
14496 }
14497
14498 let router = Router::with_control_handler(Arc::new(handler.clone()));
14501 let request = Frame::build(
14502 FrameType::Request,
14503 Flags::new(true, Priority::Interactive, false),
14504 client_channel,
14505 client_epoch,
14506 201,
14507 b"first request".to_vec(),
14508 )
14509 .unwrap();
14510 router
14511 .route_for_connection(&client_ctx, request)
14512 .await
14513 .unwrap();
14514 let delivered = module_rx.try_recv().unwrap().frame;
14515 assert_eq!(delivered.header.ty, FrameType::Request);
14516 assert_eq!(
14517 (delivered.header.channel, delivered.header.epoch),
14518 (module_channel, module_epoch)
14519 );
14520 assert_eq!(delivered.body, b"first request");
14521 }
14522
14523 #[tokio::test]
14529 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
14530 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14531 let client = ConnectionId::new(42);
14532 let (client_ctx, _client_rx) = route_ctx(client);
14533
14534 let (open_task, bind) = relay_route_open(
14535 &handler,
14536 client,
14537 &client_ctx.egress,
14538 &mut module_rx,
14539 300,
14540 "aft",
14541 "released-before-mark",
14542 )
14543 .await;
14544 let (module_channel, module_epoch) = route_bind_channel(&bind);
14545 hold.wait().await;
14546
14547 handler.cleanup_connection(client).unwrap();
14548 assert!(
14549 module_rx.try_recv().is_err(),
14550 "client teardown must not GOODBYE a bind route.open is still queueing"
14551 );
14552
14553 hold.wait().await;
14554 let response = open_task.await.unwrap();
14555 assert_eq!(response.len(), 1);
14556 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14557
14558 let goodbye = module_rx
14559 .try_recv()
14560 .expect("module is told to drop the binding the daemon abandoned")
14561 .frame;
14562 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14563 assert_eq!(
14564 (goodbye.header.channel, goodbye.header.epoch),
14565 (module_channel, module_epoch)
14566 );
14567 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14568 }
14569
14570 #[tokio::test]
14575 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
14576 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14577 let client = ConnectionId::new(43);
14578 let (client_ctx, client_rx) = route_ctx(client);
14579
14580 let (open_task, bind) = relay_route_open(
14581 &handler,
14582 client,
14583 &client_ctx.egress,
14584 &mut module_rx,
14585 400,
14586 "aft",
14587 "gone-client-before-mark",
14588 )
14589 .await;
14590 let (module_channel, module_epoch) = route_bind_channel(&bind);
14591 hold.wait().await;
14592
14593 drop(client_rx);
14594 assert!(client_ctx.egress.is_closed());
14595 handler
14596 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14597 .await
14598 .unwrap();
14599
14600 hold.wait().await;
14601 let response = open_task.await.unwrap();
14602 assert_eq!(response.len(), 1);
14603 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14604
14605 let goodbye = module_rx
14606 .try_recv()
14607 .expect("module is told to drop the binding it acked")
14608 .frame;
14609 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14610 assert_eq!(
14611 (goodbye.header.channel, goodbye.header.epoch),
14612 (module_channel, module_epoch)
14613 );
14614 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14615 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14616 }
14617
14618 #[test]
14625 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
14626 let handler = ControlHandler::default();
14627 let module_connection = ConnectionId::new(30);
14628 let client_connection = ConnectionId::new(31);
14629
14630 handler
14631 .refuse_to_end_module_connection_for_a_client(
14632 module_connection,
14633 77,
14634 ForwardingError::ConnectionClosing {
14635 connection_id: client_connection,
14636 },
14637 )
14638 .expect("a closing client must never end the module connection");
14639
14640 assert!(matches!(
14641 handler.refuse_to_end_module_connection_for_a_client(
14642 module_connection,
14643 78,
14644 ForwardingError::ConnectionClosing {
14645 connection_id: module_connection,
14646 },
14647 ),
14648 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
14649 connection_id
14650 })) if connection_id == module_connection
14651 ));
14652 assert!(matches!(
14653 handler.refuse_to_end_module_connection_for_a_client(
14654 module_connection,
14655 79,
14656 ForwardingError::Poisoned,
14657 ),
14658 Err(RouterError::Forwarding(ForwardingError::Poisoned))
14659 ));
14660 assert!(matches!(
14661 handler.refuse_to_end_module_connection_for_a_client(
14662 module_connection,
14663 80,
14664 ForwardingError::StaleModuleEndpoint,
14665 ),
14666 Err(RouterError::Forwarding(
14667 ForwardingError::StaleModuleEndpoint
14668 ))
14669 ));
14670 }
14671
14672 #[tokio::test]
14685 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
14686 let registry = Arc::new(Registry::default());
14687 let forwarding = Arc::new(ForwardingTable::default());
14688 let supervisor = SupervisorHandle::new();
14689 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14690 let handler =
14691 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14692 .with_supervisor(supervisor);
14693
14694 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14695 hello_via_sink(
14696 &handler,
14697 &target_ctx,
14698 &mut target_rx,
14699 hello_frame("target", PROTOCOL_VERSION, 1),
14700 )
14701 .await;
14702
14703 let wrong_nonce = handler
14707 .handle_control_frame(
14708 &route_ctx(ConnectionId::new(91)).0,
14709 route_open_frame_with_admission_facts(
14710 20,
14711 "target",
14712 unique_project_root("admission-facts"),
14713 Some(subc_control::ConsumerIdentity {
14714 module_id: "fed".to_string(),
14715 launch_nonce: "not-the-real-nonce".to_string(),
14716 }),
14717 None,
14718 ),
14719 )
14720 .await
14721 .unwrap();
14722 assert_eq!(
14723 parse_error(&wrong_nonce[0])["code"],
14724 "bad_consumer_identity",
14725 "a mismatched launch nonce must be refused, not stamped Reserved"
14726 );
14727
14728 let never_spawned = handler
14732 .handle_control_frame(
14733 &route_ctx(ConnectionId::new(92)).0,
14734 route_open_frame_with_admission_facts(
14735 21,
14736 "target",
14737 unique_project_root("admission-facts"),
14738 Some(subc_control::ConsumerIdentity {
14739 module_id: "never-spawned".to_string(),
14740 launch_nonce: "any-nonce".to_string(),
14741 }),
14742 None,
14743 ),
14744 )
14745 .await
14746 .unwrap();
14747 assert_eq!(
14748 parse_error(&never_spawned[0])["code"],
14749 "bad_consumer_identity",
14750 "an unspawned module_id must be refused rather than accepted for lack of a record"
14751 );
14752 }
14753
14754 #[tokio::test]
14777 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
14778 let registry = Arc::new(Registry::default());
14779 let forwarding = Arc::new(ForwardingTable::default());
14780 let supervisor = SupervisorHandle::new();
14781 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14782 let handler =
14783 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14784 .with_supervisor(supervisor);
14785
14786 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
14787 hello_via_sink(
14788 &handler,
14789 &target_ctx,
14790 &mut target_rx,
14791 hello_frame("target", PROTOCOL_VERSION, 1),
14792 )
14793 .await;
14794
14795 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
14796 let route_handler = handler.clone();
14797 let route_task = tokio::spawn(async move {
14798 route_handler
14799 .handle_control_frame(
14800 &client_ctx,
14801 route_open_frame_with_admission_facts(
14802 30,
14803 "target",
14804 unique_project_root("admission-facts"),
14805 Some(subc_control::ConsumerIdentity {
14806 module_id: "fed".to_string(),
14807 launch_nonce: "fed-nonce".to_string(),
14808 }),
14809 None,
14810 ),
14811 )
14812 .await
14813 .unwrap()
14814 });
14815
14816 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14823 .await
14824 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
14825 .expect("module control channel closed before route.bind");
14826 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14827 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14828 panic!("expected route.bind")
14829 };
14830 assert_eq!(
14831 principal,
14832 Some(Principal::Reserved {
14833 module_id: "fed".to_string()
14834 }),
14835 "a correctly attested consumer must be stamped Reserved for its own id"
14836 );
14837
14838 handler
14839 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14840 .await
14841 .unwrap();
14842 assert!(route_task.await.unwrap().is_empty());
14843 assert!(
14844 matches!(
14845 serde_json::from_slice::<ClientControlResponse>(
14846 &client_rx.recv().await.unwrap().body
14847 )
14848 .unwrap(),
14849 ClientControlResponse::RouteOpen { .. }
14850 ),
14851 "the route must actually open, not merely avoid an error"
14852 );
14853 }
14854
14855 #[tokio::test(start_paused = true)]
14856 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
14857 let registry = Arc::new(Registry::default());
14858 let forwarding = Arc::new(ForwardingTable::default());
14859 let supervisor = SupervisorHandle::new();
14860 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14861 let handler =
14862 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14863 .with_supervisor(supervisor);
14864
14865 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
14866 hello_via_sink(
14867 &handler,
14868 &target_ctx,
14869 &mut target_rx,
14870 hello_frame("target", PROTOCOL_VERSION, 1),
14871 )
14872 .await;
14873
14874 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
14875 let direct_handler = handler.clone();
14876 let direct_open = tokio::spawn(async move {
14877 direct_handler
14878 .handle_control_frame(
14879 &direct_ctx,
14880 route_open_frame(2, "target", unique_project_root("route-census-direct")),
14881 )
14882 .await
14883 .unwrap()
14884 });
14885 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14886 .await
14887 .expect("no direct route.bind within 5s")
14888 .expect("target control channel closed before direct route.bind");
14889 handler
14890 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
14891 .await
14892 .unwrap();
14893 assert!(direct_open.await.unwrap().is_empty());
14894 let _ = direct_rx.recv().await.unwrap();
14895
14896 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
14897 let reserved_handler = handler.clone();
14898 let reserved_open = tokio::spawn(async move {
14899 reserved_handler
14900 .handle_control_frame(
14901 &reserved_ctx,
14902 route_open_frame_with_admission_facts(
14903 3,
14904 "target",
14905 unique_project_root("admission-facts"),
14906 Some(ConsumerIdentity {
14907 module_id: "fed".to_string(),
14908 launch_nonce: "fed-nonce".to_string(),
14909 }),
14910 None,
14911 ),
14912 )
14913 .await
14914 .unwrap()
14915 });
14916 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14917 .await
14918 .expect("no reserved route.bind within 5s")
14919 .expect("target control channel closed before reserved route.bind");
14920 handler
14921 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
14922 .await
14923 .unwrap();
14924 assert!(reserved_open.await.unwrap().is_empty());
14925 let _ = reserved_rx.recv().await.unwrap();
14926
14927 forwarding
14928 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
14929 .unwrap();
14930 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
14931 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
14932 module_id: Some("target".to_string()),
14933 })
14934 .unwrap();
14935 let census_frame =
14936 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
14937 let response = handler
14938 .handle_control_frame(&census_ctx, census_frame)
14939 .await
14940 .unwrap()
14941 .pop()
14942 .unwrap();
14943 let actual: Value = serde_json::from_slice(&response.body).unwrap();
14944 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
14945 assert!(matches!(
14946 decoded,
14947 ClientControlResponse::SupervisorRoutes { .. }
14948 ));
14949 let routes = actual["modules"][0]["routes"].as_array().unwrap();
14950 assert_eq!(routes.len(), 2);
14951 assert!(routes.iter().all(|route| route["draining"] == true));
14952 assert!(
14955 routes.iter().all(|route| route["drain_reason"] == "reload"),
14956 "draining routes must name the drain's reason: {routes:?}"
14957 );
14958 assert!(routes.iter().any(|route| {
14959 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
14960 }));
14961 assert!(routes.iter().any(|route| {
14962 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
14963 }));
14964
14965 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14966 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
14967 if std::env::var_os("UPDATE_GOLDEN").is_some() {
14968 std::fs::write(
14969 &golden_path,
14970 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
14971 )
14972 .unwrap();
14973 }
14974 let expected: Value =
14975 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
14976 assert_eq!(actual, expected);
14977 }
14978
14979 async fn query_live_roots(
14980 handler: &ControlHandler,
14981 module_ctx: &RouteCtx,
14982 ) -> ModuleControlResponseToModule {
14983 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
14984 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
14985 let response = handler
14986 .handle_control_frame(module_ctx, frame)
14987 .await
14988 .unwrap()
14989 .pop()
14990 .unwrap();
14991 serde_json::from_slice(&response.body).unwrap()
14992 }
14993
14994 #[tokio::test(start_paused = true)]
14995 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
14996 let registry = Arc::new(Registry::default());
14997 let forwarding = Arc::new(ForwardingTable::default());
14998 let handler = ControlHandler::with_forwarding(registry, forwarding);
14999 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
15000 hello_via_sink(
15001 &handler,
15002 &target_ctx,
15003 &mut target_rx,
15004 hello_frame("target", PROTOCOL_VERSION, 1),
15005 )
15006 .await;
15007 let root = unique_project_root("live-roots-known");
15008 let path = ProjectRootId::from_path_allowing_missing(root.path())
15009 .unwrap()
15010 .as_path()
15011 .to_path_buf();
15012 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
15013 let open_handler = handler.clone();
15014 let opened = tokio::spawn(async move {
15015 open_handler
15016 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
15017 .await
15018 .unwrap()
15019 });
15020 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
15021 .await
15022 .unwrap()
15023 .unwrap();
15024 handler
15025 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
15026 .await
15027 .unwrap();
15028 assert!(opened.await.unwrap().is_empty());
15029 let _ = client_rx.recv().await.unwrap();
15030
15031 let root = unique_project_root("live-roots-pending");
15032 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
15033 .unwrap()
15034 .as_path()
15035 .to_path_buf();
15036 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
15037 let open_handler = handler.clone();
15038 let pending = tokio::spawn(async move {
15039 open_handler
15040 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
15041 .await
15042 .unwrap()
15043 });
15044 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
15045 .await
15046 .unwrap()
15047 .unwrap();
15048 let actual = query_live_roots(&handler, &target_ctx).await;
15049 let ModuleControlResponseToModule::LiveRoots {
15050 roots,
15051 unknown_root_bindings,
15052 total_bindings,
15053 } = actual
15054 else {
15055 panic!("expected live roots")
15056 };
15057 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
15058 assert_eq!(unknown_root_bindings, 0);
15059 assert_eq!(
15060 roots.len(),
15061 2,
15062 "root-known arm must retain each canonical root"
15063 );
15064 assert_eq!(
15065 total_bindings,
15066 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15067 );
15068 let counts = roots
15069 .iter()
15070 .map(|root| (root.project_root.clone(), root.bound, root.pending))
15071 .collect::<Vec<_>>();
15072 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
15073 expected.sort_by(|a, b| a.0.cmp(&b.0));
15074 assert_eq!(
15075 counts, expected,
15076 "roots must sort by path and count pending separately"
15077 );
15078 handler
15079 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
15080 .await
15081 .unwrap();
15082 assert!(pending.await.unwrap().is_empty());
15083 }
15084
15085 #[tokio::test(start_paused = true)]
15086 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
15087 let forwarding = Arc::new(ForwardingTable::default());
15088 let handler =
15089 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
15090 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
15091 hello_via_sink(
15092 &handler,
15093 &target_ctx,
15094 &mut target_rx,
15095 hello_frame("target", PROTOCOL_VERSION, 1),
15096 )
15097 .await;
15098 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
15099 let pending = forwarding
15100 .begin_route_bind_relay_for_test(
15101 client_ctx.connection_id,
15102 client_ctx.egress.clone(),
15103 2,
15104 "target",
15105 )
15106 .unwrap();
15107 forwarding
15108 .complete_pending_relay(
15109 target_ctx.connection_id,
15110 pending.corr,
15111 RouteBindRelayOutcome::Accepted,
15112 )
15113 .unwrap();
15114 let actual = query_live_roots(&handler, &target_ctx).await;
15115 let ModuleControlResponseToModule::LiveRoots {
15116 roots,
15117 unknown_root_bindings,
15118 total_bindings,
15119 } = actual
15120 else {
15121 panic!("expected live roots")
15122 };
15123 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
15124 assert_eq!(
15125 unknown_root_bindings, 1,
15126 "unknown-root arm must not read as no bindings"
15127 );
15128 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
15129 assert_eq!(
15130 total_bindings,
15131 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15132 );
15133 }
15134
15135 #[tokio::test(start_paused = true)]
15143 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
15144 let forwarding = Arc::new(ForwardingTable::default());
15145 let handler =
15146 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
15147 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
15148 let replies = handler
15149 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
15150 .await
15151 .unwrap();
15152 let queued_by_hello = module_rx.len();
15153
15154 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
15155 let open_handler = handler.clone();
15156 let open = tokio::spawn(async move {
15157 open_handler
15158 .handle_control_frame(
15159 &client_ctx,
15160 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
15161 )
15162 .await
15163 .unwrap()
15164 });
15165 let mut spins = 0;
15167 while module_rx.len() == queued_by_hello {
15168 spins += 1;
15169 assert!(spins < 10_000, "route.open never queued a route.bind");
15170 tokio::task::yield_now().await;
15171 }
15172
15173 for reply in replies {
15175 module_ctx.egress.send(reply).await.unwrap();
15176 }
15177
15178 let first = module_rx.recv().await.unwrap().frame;
15179 assert_eq!(
15180 first.header.ty,
15181 FrameType::HelloAck,
15182 "the first frame a registering module reads must be its HELLO_ACK"
15183 );
15184 assert_eq!(first.header.corr, 7);
15185 let second = module_rx.recv().await.unwrap().frame;
15186 assert_eq!(second.header.ty, FrameType::Request);
15187 assert!(
15188 matches!(
15189 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
15190 ModuleControlRequest::RouteBind { .. }
15191 ),
15192 "the route.bind follows the ack"
15193 );
15194 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
15195
15196 handler
15197 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
15198 .await
15199 .unwrap();
15200 assert!(open.await.unwrap().is_empty());
15201 let _ = client_rx.recv().await.unwrap();
15202 }
15203
15204 #[tokio::test(start_paused = true)]
15205 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
15206 let handler = ControlHandler::with_forwarding(
15207 Arc::new(Registry::default()),
15208 Arc::new(ForwardingTable::default()),
15209 );
15210 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
15211 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
15212 hello_via_sink(
15213 &handler,
15214 &first_ctx,
15215 &mut first_rx,
15216 hello_frame("first", PROTOCOL_VERSION, 1),
15217 )
15218 .await;
15219 hello_via_sink(
15220 &handler,
15221 &second_ctx,
15222 &mut second_rx,
15223 hello_frame("second", PROTOCOL_VERSION, 2),
15224 )
15225 .await;
15226 let root = unique_project_root("second-only");
15227 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
15228 let cloned = handler.clone();
15229 let open = tokio::spawn(async move {
15230 cloned
15231 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
15232 .await
15233 .unwrap()
15234 });
15235 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
15236 .await
15237 .unwrap()
15238 .unwrap();
15239 let first = query_live_roots(&handler, &first_ctx).await;
15240 let second = query_live_roots(&handler, &second_ctx).await;
15241 assert!(
15242 matches!(
15243 first,
15244 ModuleControlResponseToModule::LiveRoots {
15245 total_bindings: 0,
15246 ..
15247 }
15248 ),
15249 "cross-module scope must not expose another module's roots"
15250 );
15251 assert!(
15252 matches!(
15253 second,
15254 ModuleControlResponseToModule::LiveRoots {
15255 total_bindings: 1,
15256 ..
15257 }
15258 ),
15259 "second module must see its pending route"
15260 );
15261 handler
15262 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
15263 .await
15264 .unwrap();
15265 assert!(open.await.unwrap().is_empty());
15266 }
15267
15268 #[tokio::test(start_paused = true)]
15269 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
15270 let handler = ControlHandler::with_forwarding(
15271 Arc::new(Registry::default()),
15272 Arc::new(ForwardingTable::default()),
15273 );
15274 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
15275 hello_via_sink(
15276 &handler,
15277 &target_ctx,
15278 &mut target_rx,
15279 hello_frame("target", PROTOCOL_VERSION, 1),
15280 )
15281 .await;
15282 let actual = query_live_roots(&handler, &target_ctx).await;
15283 let ModuleControlResponseToModule::LiveRoots {
15284 roots,
15285 unknown_root_bindings,
15286 total_bindings,
15287 } = actual
15288 else {
15289 panic!("expected live roots")
15290 };
15291 assert!(roots.is_empty());
15292 assert_eq!(unknown_root_bindings, 0);
15293 assert_eq!(total_bindings, 0);
15294 assert_eq!(
15295 total_bindings,
15296 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15297 );
15298 }
15299
15300 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
15307 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
15308 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
15309 let text = std::fs::read_to_string(&path)
15310 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
15311 let vectors: Vec<(String, Value)> = text
15312 .lines()
15313 .filter(|line| !line.trim().is_empty())
15314 .map(|line| {
15315 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
15316 let id = entry["corpus_id"]
15317 .as_str()
15318 .expect("every vector carries a corpus_id")
15319 .to_string();
15320 (id, entry["package"].clone())
15321 })
15322 .collect();
15323 assert_eq!(
15327 vectors.len(),
15328 3,
15329 "vendored fed corpus changed size; re-sync from subc-federation"
15330 );
15331
15332 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
15347 let richest = vectors
15348 .iter()
15349 .filter_map(|(_, package)| package.as_object())
15350 .map(|object| {
15351 object
15352 .keys()
15353 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
15354 .count()
15355 })
15356 .max()
15357 .unwrap_or(0);
15358 assert!(
15359 richest >= 2,
15360 "vendored corpus no longer carries a package with unmodelled fields, \
15361 so the relay test can no longer distinguish a verbatim relay from a lossy one"
15362 );
15363
15364 vectors
15365 }
15366
15367 #[tokio::test]
15375 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
15376 for (corpus_id, package) in fed_admission_facts_vectors() {
15377 let registry = Arc::new(Registry::default());
15378 let forwarding = Arc::new(ForwardingTable::default());
15379 let supervisor = SupervisorHandle::new();
15380 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15381 let handler =
15382 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15383 .with_supervisor(supervisor)
15384 .with_admission_facts_config(
15385 Some("fed".to_string()),
15386 Some(vec!["target".to_string()]),
15387 );
15388
15389 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
15390 hello_via_sink(
15391 &handler,
15392 &target_ctx,
15393 &mut target_rx,
15394 hello_frame("target", PROTOCOL_VERSION, 1),
15395 )
15396 .await;
15397
15398 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
15399 let route_handler = handler.clone();
15400 let expected = package.clone();
15401 let route_task = tokio::spawn(async move {
15402 route_handler
15403 .handle_control_frame(
15404 &client_ctx,
15405 route_open_frame_with_admission_facts(
15406 20,
15407 "target",
15408 unique_project_root("admission-facts"),
15409 Some(subc_control::ConsumerIdentity {
15410 module_id: "fed".to_string(),
15411 launch_nonce: "fed-nonce".to_string(),
15412 }),
15413 Some(package),
15414 ),
15415 )
15416 .await
15417 .unwrap()
15418 });
15419
15420 let bind_frame = target_rx.recv().await.unwrap();
15421 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15422 let ModuleControlRequest::RouteBind {
15423 admission_facts, ..
15424 } = bind
15425 else {
15426 panic!("{corpus_id}: expected route.bind")
15427 };
15428 assert_eq!(
15429 admission_facts,
15430 Some(expected),
15431 "{corpus_id}: relay must not add, drop or reshape any field"
15432 );
15433
15434 handler
15435 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15436 .await
15437 .unwrap();
15438 route_task.await.unwrap();
15439 }
15440 }
15441
15442 #[tokio::test]
15443 async fn route_bind_never_relays_the_openers_build_class() {
15444 let registry = Arc::new(Registry::default());
15445 let forwarding = Arc::new(ForwardingTable::default());
15446 let handler =
15447 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15448 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
15449 hello_via_sink(
15450 &handler,
15451 &target_ctx,
15452 &mut target_rx,
15453 hello_frame("target", PROTOCOL_VERSION, 1),
15454 )
15455 .await;
15456
15457 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
15458 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
15459 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15460 busy_refusals: None,
15461 target: RouteTarget::ToolProvider {
15462 module_id: "target".to_string(),
15463 },
15464 identity,
15465 consumer_identity: None,
15466 consumer_capabilities: None,
15467 role_versions: None,
15468 admission_facts: None,
15469 scope: None,
15470 })
15471 .unwrap();
15472 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
15473
15474 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
15475 let route_handler = handler.clone();
15476 let route_task = tokio::spawn(async move {
15477 route_handler
15478 .handle_control_frame(&client_ctx, open)
15479 .await
15480 .unwrap()
15481 });
15482 let bind_frame = target_rx.recv().await.unwrap();
15483 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15484 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
15485 panic!("expected route.bind")
15486 };
15487 assert_eq!(
15488 identity.build_class, None,
15489 "route.bind carried the build class"
15490 );
15491 assert!(
15492 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
15493 "the route.bind bytes still name build_class"
15494 );
15495 handler
15496 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15497 .await
15498 .unwrap();
15499 route_task.await.unwrap();
15500 }
15501
15502 #[tokio::test]
15503 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
15504 let registry = Arc::new(Registry::default());
15505 let forwarding = Arc::new(ForwardingTable::default());
15506 let supervisor = SupervisorHandle::new();
15507 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15508 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
15509 let handler =
15510 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15511 .with_supervisor(supervisor)
15512 .with_admission_facts_config(
15513 Some("fed".to_string()),
15514 Some(vec!["target".to_string()]),
15515 );
15516
15517 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
15518 hello_via_sink(
15519 &handler,
15520 &target_ctx,
15521 &mut target_rx,
15522 hello_frame("target", PROTOCOL_VERSION, 1),
15523 )
15524 .await;
15525 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
15526 hello_via_sink(
15527 &handler,
15528 &other_ctx,
15529 &mut other_rx,
15530 hello_frame("other", PROTOCOL_VERSION, 2),
15531 )
15532 .await;
15533
15534 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
15535 let expected_facts = facts.clone();
15536 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
15537 let route_handler = handler.clone();
15538 let route_task = tokio::spawn(async move {
15539 route_handler
15540 .handle_control_frame(
15541 &client_ctx,
15542 route_open_frame_with_admission_facts(
15543 10,
15544 "target",
15545 unique_project_root("admission-facts"),
15546 Some(subc_control::ConsumerIdentity {
15547 module_id: "fed".to_string(),
15548 launch_nonce: "fed-nonce".to_string(),
15549 }),
15550 Some(facts.clone()),
15551 ),
15552 )
15553 .await
15554 .unwrap()
15555 });
15556 let bind_frame = target_rx.recv().await.unwrap();
15557 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15558 let ModuleControlRequest::RouteBind {
15559 admission_facts, ..
15560 } = bind
15561 else {
15562 panic!("expected route.bind")
15563 };
15564 assert_eq!(admission_facts, Some(expected_facts));
15565 handler
15566 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15567 .await
15568 .unwrap();
15569 assert!(route_task.await.unwrap().is_empty());
15570 assert!(matches!(
15571 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
15572 .unwrap(),
15573 ClientControlResponse::RouteOpen { .. }
15574 ));
15575
15576 let direct = handler
15577 .handle_control_frame(
15578 &route_ctx(ConnectionId::new(73)).0,
15579 route_open_frame_with_admission_facts(
15580 11,
15581 "target",
15582 unique_project_root("admission-facts"),
15583 None,
15584 Some(json!({"x": 1})),
15585 ),
15586 )
15587 .await
15588 .unwrap();
15589 assert_eq!(
15590 parse_error(&direct[0])["code"],
15591 "admission_facts_not_permitted"
15592 );
15593
15594 let different_reserved = handler
15595 .handle_control_frame(
15596 &route_ctx(ConnectionId::new(77)).0,
15597 route_open_frame_with_admission_facts(
15598 15,
15599 "target",
15600 unique_project_root("admission-facts"),
15601 Some(subc_control::ConsumerIdentity {
15602 module_id: "other".to_string(),
15603 launch_nonce: "other-nonce".to_string(),
15604 }),
15605 Some(json!({"x": 1})),
15606 ),
15607 )
15608 .await
15609 .unwrap();
15610 assert_eq!(
15611 parse_error(&different_reserved[0])["code"],
15612 "admission_facts_not_permitted"
15613 );
15614
15615 let other_target = handler
15616 .handle_control_frame(
15617 &route_ctx(ConnectionId::new(74)).0,
15618 route_open_frame_with_admission_facts(
15619 12,
15620 "other",
15621 unique_project_root("admission-facts"),
15622 Some(subc_control::ConsumerIdentity {
15623 module_id: "fed".to_string(),
15624 launch_nonce: "fed-nonce".to_string(),
15625 }),
15626 Some(json!({"x": 1})),
15627 ),
15628 )
15629 .await
15630 .unwrap();
15631 assert_eq!(
15632 parse_error(&other_target[0])["code"],
15633 "admission_facts_target_not_allowed"
15634 );
15635
15636 let nonexistent = handler
15637 .handle_control_frame(
15638 &route_ctx(ConnectionId::new(75)).0,
15639 route_open_frame_with_admission_facts(
15640 13,
15641 "missing",
15642 unique_project_root("admission-facts"),
15643 None,
15644 Some(json!({"x": 1})),
15645 ),
15646 )
15647 .await
15648 .unwrap();
15649 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
15650
15651 let described = handler
15652 .handle_control_frame(
15653 &route_ctx(ConnectionId::new(76)).0,
15654 Frame::build(
15655 FrameType::Request,
15656 control_flags(),
15657 0,
15658 0,
15659 14,
15660 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15661 )
15662 .unwrap(),
15663 )
15664 .await
15665 .unwrap();
15666 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15667 serde_json::from_slice(&described[0].body).unwrap()
15668 else {
15669 panic!("expected server.describe response")
15670 };
15671 assert!(capabilities
15672 .iter()
15673 .any(|cap| cap == "admission_facts_relay_v1"));
15674 }
15675
15676 #[tokio::test]
15677 async fn admission_facts_without_configured_carrier_are_rejected() {
15678 let registry = Arc::new(Registry::default());
15679 let forwarding = Arc::new(ForwardingTable::default());
15680 let handler = ControlHandler::with_forwarding(registry, forwarding);
15681 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
15682 hello_via_sink(
15683 &handler,
15684 &target_ctx,
15685 &mut target_rx,
15686 hello_frame("target", PROTOCOL_VERSION, 1),
15687 )
15688 .await;
15689
15690 let responses = handler
15691 .handle_control_frame(
15692 &route_ctx(ConnectionId::new(79)).0,
15693 route_open_frame_with_admission_facts(
15694 16,
15695 "target",
15696 unique_project_root("admission-facts"),
15697 None,
15698 Some(json!({"x": 1})),
15699 ),
15700 )
15701 .await
15702 .unwrap();
15703 assert_eq!(
15704 parse_error(&responses[0])["code"],
15705 "admission_facts_not_permitted"
15706 );
15707 }
15708
15709 #[tokio::test]
15710 async fn route_open_relays_consumer_capabilities_verbatim() {
15711 let registry = Arc::new(Registry::default());
15712 let forwarding = Arc::new(ForwardingTable::default());
15713 let handler =
15714 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15715 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
15716 hello_via_sink(
15717 &handler,
15718 &module_ctx,
15719 &mut module_rx,
15720 hello_frame("aft", PROTOCOL_VERSION, 7),
15721 )
15722 .await;
15723
15724 let expected = vec!["elicitation".to_string(), "roots".to_string()];
15725 let expected_for_request = expected.clone();
15726 let project_root = unique_project_root("consumer-capabilities-present");
15727 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
15728 let route_handler = handler.clone();
15729 let route_task = tokio::spawn(async move {
15730 route_handler
15731 .handle_control_frame(
15732 &client_ctx,
15733 route_open_frame_with_consumer_capabilities(
15734 401,
15735 "aft",
15736 project_root,
15737 Some(expected_for_request),
15738 ),
15739 )
15740 .await
15741 .unwrap()
15742 });
15743 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15744 .await
15745 .unwrap()
15746 .unwrap();
15747 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15748 let ModuleControlRequest::RouteBind {
15749 consumer_capabilities,
15750 ..
15751 } = bind
15752 else {
15753 panic!("expected route.bind request, got {bind:?}");
15754 };
15755 assert_eq!(consumer_capabilities, Some(expected.clone()));
15756
15757 handler
15758 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15759 .await
15760 .unwrap();
15761 let route_response = route_task.await.unwrap();
15762 assert!(route_response.is_empty());
15763 let published = client_rx.recv().await.unwrap();
15764 assert!(matches!(
15765 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15766 ClientControlResponse::RouteOpen { .. }
15767 ));
15768 }
15769
15770 #[tokio::test]
15771 async fn route_open_without_consumer_capabilities_relays_none() {
15772 let registry = Arc::new(Registry::default());
15773 let forwarding = Arc::new(ForwardingTable::default());
15774 let handler =
15775 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15776 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
15777 hello_via_sink(
15778 &handler,
15779 &module_ctx,
15780 &mut module_rx,
15781 hello_frame("aft", PROTOCOL_VERSION, 7),
15782 )
15783 .await;
15784
15785 let project_root = unique_project_root("consumer-capabilities-absent");
15786 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
15787 let route_handler = handler.clone();
15788 let route_task = tokio::spawn(async move {
15789 route_handler
15790 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
15791 .await
15792 .unwrap()
15793 });
15794 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15795 .await
15796 .unwrap()
15797 .unwrap();
15798 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15799 let ModuleControlRequest::RouteBind {
15800 consumer_capabilities,
15801 ..
15802 } = bind
15803 else {
15804 panic!("expected route.bind request, got {bind:?}");
15805 };
15806 assert_eq!(consumer_capabilities, None);
15807
15808 handler
15809 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15810 .await
15811 .unwrap();
15812 let route_response = route_task.await.unwrap();
15813 assert!(route_response.is_empty());
15814 let published = client_rx.recv().await.unwrap();
15815 assert!(matches!(
15816 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15817 ClientControlResponse::RouteOpen { .. }
15818 ));
15819 }
15820
15821 async fn bind_role_versions_for(
15825 sent: Option<BTreeMap<String, String>>,
15826 connection: u64,
15827 ) -> Option<BTreeMap<String, String>> {
15828 let registry = Arc::new(Registry::default());
15829 let forwarding = Arc::new(ForwardingTable::default());
15830 let handler =
15831 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15832 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
15833 hello_via_sink(
15834 &handler,
15835 &module_ctx,
15836 &mut module_rx,
15837 hello_frame("aft", PROTOCOL_VERSION, 7),
15838 )
15839 .await;
15840 let project_root = unique_project_root("role-versions");
15841 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
15842 let route_handler = handler.clone();
15843 let route_task = tokio::spawn(async move {
15844 route_handler
15845 .handle_control_frame(
15846 &client_ctx,
15847 route_open_frame_with_role_versions(403, "aft", project_root, sent),
15848 )
15849 .await
15850 .unwrap()
15851 });
15852 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15853 .await
15854 .expect("a well-formed route.open reaches the module as a bind")
15855 .unwrap();
15856 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15857 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
15858 panic!("expected route.bind request, got {bind:?}");
15859 };
15860 handler
15861 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15862 .await
15863 .unwrap();
15864 assert!(route_task.await.unwrap().is_empty());
15865 let published = client_rx.recv().await.unwrap();
15866 assert!(matches!(
15867 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15868 ClientControlResponse::RouteOpen { .. }
15869 ));
15870 role_versions
15871 }
15872
15873 #[tokio::test]
15874 async fn route_open_relays_role_versions_verbatim() {
15875 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
15876 assert_eq!(
15877 bind_role_versions_for(Some(sent.clone()), 141).await,
15878 Some(sent)
15879 );
15880 }
15881
15882 #[tokio::test]
15885 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
15886 assert_eq!(bind_role_versions_for(None, 143).await, None);
15887 assert_eq!(
15888 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
15889 None
15890 );
15891 }
15892
15893 #[tokio::test]
15894 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
15895 let registry = Arc::new(Registry::default());
15896 let forwarding = Arc::new(ForwardingTable::default());
15897 let handler =
15898 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15899 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
15900 hello_via_sink(
15901 &handler,
15902 &module_ctx,
15903 &mut module_rx,
15904 hello_frame("aft", PROTOCOL_VERSION, 7),
15905 )
15906 .await;
15907
15908 let nine: BTreeMap<String, String> = (0..9)
15909 .map(|index| (format!("role-{index}"), "v1".to_string()))
15910 .collect();
15911 for (label, malformed) in [
15912 (
15913 "invalid role name",
15914 role_versions(&[("Tool_Provider", "v1")]),
15915 ),
15916 ("invalid version", role_versions(&[("tool-provider", "v0")])),
15917 ("nine entries", nine),
15918 ] {
15919 let responses = tokio::time::timeout(
15922 Duration::from_secs(1),
15923 handler.handle_control_frame(
15924 &route_ctx(ConnectionId::new(148)).0,
15925 route_open_frame_with_role_versions(
15926 404,
15927 "aft",
15928 unique_project_root("role-versions-malformed"),
15929 Some(malformed),
15930 ),
15931 ),
15932 )
15933 .await
15934 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
15935 .unwrap();
15936 assert_eq!(responses.len(), 1, "{label}");
15937 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
15938 let error = parse_error(&responses[0]);
15939 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
15940 assert_eq!(
15941 error["detail"]["field"], "role_versions",
15942 "{label}: {error}"
15943 );
15944 assert!(
15945 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
15946 "{label}: a malformed declaration is terminal"
15947 );
15948 assert!(
15949 module_rx.try_recv().is_err(),
15950 "{label}: the module must never see a bind"
15951 );
15952 }
15953 }
15954
15955 #[tokio::test]
15959 async fn route_role_versions_capability_is_advertised() {
15960 let handler = ControlHandler::new(Arc::new(Registry::default()));
15961 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
15962 let ack = hello_via_sink(
15963 &handler,
15964 &ctx,
15965 &mut rx,
15966 hello_frame("m", PROTOCOL_VERSION, 1),
15967 )
15968 .await;
15969 let ack = parse_ack(&ack);
15970 assert!(
15971 ack.subc_capabilities
15972 .iter()
15973 .any(|c| c == "route-role-versions/v1"),
15974 "{:?}",
15975 ack.subc_capabilities
15976 );
15977
15978 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15979 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15980 let reply = handler
15981 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
15982 .await
15983 .unwrap()
15984 .pop()
15985 .unwrap();
15986 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15987 serde_json::from_slice(&reply.body).unwrap()
15988 else {
15989 panic!("not a server.describe reply");
15990 };
15991 assert!(
15992 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
15993 "{capabilities:?}"
15994 );
15995 }
15996
15997 #[tokio::test]
16001 async fn catalog_events_capability_is_advertised() {
16002 let handler = ControlHandler::new(Arc::new(Registry::default()));
16003 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16004 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16005 let reply = handler
16006 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
16007 .await
16008 .unwrap()
16009 .pop()
16010 .unwrap();
16011 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16012 serde_json::from_slice(&reply.body).unwrap()
16013 else {
16014 panic!("not a server.describe reply");
16015 };
16016 assert!(
16017 capabilities
16018 .iter()
16019 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
16020 "{capabilities:?}"
16021 );
16022 }
16023
16024 #[tokio::test]
16025 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
16026 let handler = ControlHandler::new(Arc::new(Registry::default()));
16027 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
16028 let ack = hello_via_sink(
16029 &handler,
16030 &module,
16031 &mut module_rx,
16032 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
16033 )
16034 .await;
16035 let ack = parse_ack(&ack);
16036 assert!(
16037 ack.subc_capabilities
16038 .iter()
16039 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
16040 "{:?}",
16041 ack.subc_capabilities
16042 );
16043 assert!(ack
16044 .subc_capabilities
16045 .iter()
16046 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
16047
16048 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
16049 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16050 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16051 let reply = handler
16052 .handle_control_frame(&client, frame)
16053 .await
16054 .unwrap()
16055 .pop()
16056 .unwrap();
16057 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16058 serde_json::from_slice(&reply.body).unwrap()
16059 else {
16060 panic!("not a server.describe reply");
16061 };
16062 assert!(
16063 capabilities
16064 .iter()
16065 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
16066 "{capabilities:?}"
16067 );
16068 assert!(capabilities
16069 .iter()
16070 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
16071 }
16072
16073 #[tokio::test]
16074 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
16075 let handler = ControlHandler::new(Arc::new(Registry::default()));
16076 let (module, mut rx) = route_ctx(ConnectionId::new(156));
16077 let ack = hello_via_sink(
16078 &handler,
16079 &module,
16080 &mut rx,
16081 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
16082 )
16083 .await;
16084 assert!(parse_ack(&ack)
16085 .subc_capabilities
16086 .iter()
16087 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
16088 assert!(parse_ack(&ack)
16089 .subc_capabilities
16090 .iter()
16091 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
16092 let (client, _rx) = route_ctx(ConnectionId::new(157));
16093 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16094 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16095 let reply = handler
16096 .handle_control_frame(&client, frame)
16097 .await
16098 .unwrap()
16099 .pop()
16100 .unwrap();
16101 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16102 serde_json::from_slice(&reply.body).unwrap()
16103 else {
16104 panic!("server.describe reply")
16105 };
16106 assert!(capabilities
16107 .iter()
16108 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
16109 assert!(capabilities
16110 .iter()
16111 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
16112 }
16113
16114 #[tokio::test]
16115 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
16116 let handler = ControlHandler::new(Arc::new(Registry::default()));
16117 let (module, mut rx) = route_ctx(ConnectionId::new(158));
16118 let ack = hello_via_sink(
16119 &handler,
16120 &module,
16121 &mut rx,
16122 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
16123 )
16124 .await;
16125 assert!(parse_ack(&ack)
16126 .subc_capabilities
16127 .iter()
16128 .any(|cap| cap == "scope-workdir/v1"));
16129 let (client, _rx) = route_ctx(ConnectionId::new(159));
16130 let frame = Frame::build(
16131 FrameType::Request,
16132 control_flags(),
16133 0,
16134 0,
16135 5,
16136 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16137 )
16138 .unwrap();
16139 let reply = handler
16140 .handle_control_frame(&client, frame)
16141 .await
16142 .unwrap()
16143 .remove(0);
16144 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16145 serde_json::from_slice(&reply.body).unwrap()
16146 else {
16147 panic!("server.describe reply")
16148 };
16149 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
16150 }
16151
16152 #[tokio::test]
16153 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
16154 let handler = ControlHandler::new(Arc::new(Registry::default()));
16155 let (module, mut rx) = route_ctx(ConnectionId::new(160));
16156 let ack = hello_via_sink(
16157 &handler,
16158 &module,
16159 &mut rx,
16160 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
16161 )
16162 .await;
16163 assert!(parse_ack(&ack)
16164 .subc_capabilities
16165 .iter()
16166 .any(|cap| cap == "worker-runs/v1"));
16167 let (client, _rx) = route_ctx(ConnectionId::new(161));
16168 let frame = Frame::build(
16169 FrameType::Request,
16170 control_flags(),
16171 0,
16172 0,
16173 5,
16174 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16175 )
16176 .unwrap();
16177 let reply = handler
16178 .handle_control_frame(&client, frame)
16179 .await
16180 .unwrap()
16181 .remove(0);
16182 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16183 serde_json::from_slice(&reply.body).unwrap()
16184 else {
16185 panic!("server.describe reply")
16186 };
16187 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
16188 }
16189
16190 #[tokio::test]
16191 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
16192 let handler = ControlHandler::new(Arc::new(Registry::default()));
16193 let (module, mut rx) = route_ctx(ConnectionId::new(154));
16194 let ack = hello_via_sink(
16195 &handler,
16196 &module,
16197 &mut rx,
16198 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
16199 )
16200 .await;
16201 assert!(parse_ack(&ack)
16202 .subc_capabilities
16203 .iter()
16204 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16205 let (client, _rx) = route_ctx(ConnectionId::new(155));
16206 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16207 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16208 let reply = handler
16209 .handle_control_frame(&client, frame)
16210 .await
16211 .unwrap()
16212 .pop()
16213 .unwrap();
16214 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16215 serde_json::from_slice(&reply.body).unwrap()
16216 else {
16217 panic!("not a server.describe reply")
16218 };
16219 assert!(capabilities
16220 .iter()
16221 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16222 }
16223
16224 #[tokio::test]
16225 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
16226 let registry = Arc::new(Registry::default());
16227 let forwarding = Arc::new(ForwardingTable::default());
16228 let handler =
16229 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16230 .with_health_probe_timeout(Duration::from_secs(5));
16231 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
16232 hello_via_sink(
16233 &handler,
16234 &module_ctx,
16235 &mut module_rx,
16236 non_routable_hello_frame_with_control_ops(
16237 "mcp",
16238 300,
16239 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16240 ),
16241 )
16242 .await;
16243 assert!(registry
16244 .get_module("mcp")
16245 .unwrap()
16246 .unwrap()
16247 .manifest
16248 .provides
16249 .is_empty());
16250
16251 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
16252 let route_response = handler
16253 .handle_control_frame(
16254 &route_client_ctx,
16255 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
16256 )
16257 .await
16258 .unwrap();
16259 assert_eq!(route_response[0].header.ty, FrameType::Error);
16260 assert_eq!(
16261 parse_error(&route_response[0])["code"],
16262 "target_unavailable"
16263 );
16264 assert!(parse_error(&route_response[0])["message"]
16265 .as_str()
16266 .unwrap()
16267 .contains("does not provide the requested target"));
16268 assert!(module_rx.try_recv().is_err());
16269
16270 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
16271 let health_handler = handler.clone();
16272 let health_task = tokio::spawn(async move {
16273 health_handler
16274 .handle_control_frame(
16275 &health_client_ctx,
16276 supervisor_health_probe_frame(302, "mcp"),
16277 )
16278 .await
16279 .unwrap()
16280 });
16281 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16282 .await
16283 .unwrap()
16284 .unwrap();
16285 assert_eq!(
16286 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
16287 ModuleControlRequest::HealthCheck {}
16288 );
16289 handler
16290 .handle_control_frame(
16291 &module_ctx,
16292 health_response(health_frame.header.corr, HealthStatus::Ok),
16293 )
16294 .await
16295 .unwrap();
16296 let health_response = health_task.await.unwrap();
16297 assert_eq!(health_response[0].header.ty, FrameType::Response);
16298 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
16299 ClientControlResponse::SupervisorHealthProbe {
16300 module_id, status, ..
16301 } => {
16302 assert_eq!(module_id, "mcp");
16303 assert_eq!(status, HealthStatus::Ok);
16304 }
16305 other => panic!("unexpected health response: {other:?}"),
16306 }
16307
16308 forwarding
16313 .cleanup_connection(module_ctx.connection_id)
16314 .unwrap();
16315 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
16316 let cleanup_response = tokio::time::timeout(
16317 Duration::from_millis(200),
16318 handler.handle_control_frame(
16319 &cleanup_probe_ctx,
16320 supervisor_health_probe_frame(303, "mcp"),
16321 ),
16322 )
16323 .await
16324 .expect("probe should fail immediately when the control lane is gone")
16325 .unwrap();
16326 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
16327 assert_eq!(
16328 parse_error(&cleanup_response[0])["code"],
16329 "target_unavailable"
16330 );
16331 assert!(parse_error(&cleanup_response[0])["message"]
16332 .as_str()
16333 .unwrap()
16334 .contains("no module connection"));
16335
16336 handler
16337 .cleanup_connection(module_ctx.connection_id)
16338 .unwrap();
16339 }
16340
16341 #[tokio::test]
16342 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
16343 let registry = Arc::new(Registry::default());
16344 let supervisor_handle = SupervisorHandle::new();
16345 let supervisor =
16346 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
16347 .with_handle(supervisor_handle.clone())
16348 .with_connection_file_path(
16349 std::env::temp_dir()
16350 .join(format!("subc-route-open-warming-{}", std::process::id())),
16351 );
16352 let module = supervisor
16353 .supervise_configured(
16354 ModuleSpec {
16355 script: None,
16356 carry_containment: false,
16357 module_id: "warming".to_string(),
16358 program: fake_aft_stub_path(),
16359 args: Vec::new(),
16360 env: Vec::new(),
16361 reserved: false,
16362 reserved_prefixes: Vec::new(),
16363 protocol: ModuleProtocol::Subc,
16364 overlap: Default::default(),
16365 },
16366 true,
16367 )
16368 .unwrap();
16369 assert_eq!(module.state().unwrap(), ModuleState::Running);
16370
16371 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16372 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
16373 let response = handler
16374 .handle_control_frame(
16375 &ctx,
16376 route_open_frame(304, "warming", unique_project_root("warming")),
16377 )
16378 .await
16379 .unwrap();
16380 module.stop().await.unwrap();
16381
16382 assert_eq!(response[0].header.ty, FrameType::Error);
16383 let error = parse_error(&response[0]);
16384 assert_eq!(error["code"], "module_warming");
16385 assert!(error["message"]
16386 .as_str()
16387 .unwrap()
16388 .contains("state=running, enabled=true, live=false"));
16389 }
16390
16391 #[test]
16392 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
16393 let handler = ControlHandler::new(Arc::new(Registry::default()));
16394 let capture = EventCapture::default();
16395 let _subscriber =
16396 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16397 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
16398 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
16399 let pending = (0..limit).collect::<Vec<_>>();
16400 let response = handler
16401 .route_open_capacity_refusal(
16402 &ctx,
16403 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
16404 "busy",
16405 pending.len(),
16406 limit,
16407 )
16408 .unwrap();
16409 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16410 let event = capture
16411 .events()
16412 .into_iter()
16413 .find(|event| {
16414 event.target == "control"
16415 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
16416 })
16417 .expect("connection admission refusal event");
16418 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16419 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16420 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
16423 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
16424 }
16425
16426 #[test]
16427 fn route_open_target_cap_logs_admission_reason_and_capacity() {
16428 let handler = ControlHandler::new(Arc::new(Registry::default()));
16429 let capture = EventCapture::default();
16430 let _subscriber =
16431 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16432 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
16433 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
16434 let guards = (0..limit)
16435 .map(|_| {
16436 handler
16437 .route_bind_concurrency
16438 .try_admit("busy", limit)
16439 .unwrap()
16440 })
16441 .collect::<Vec<_>>();
16442 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
16443 Err(in_flight) => in_flight,
16444 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
16445 };
16446 let response = handler
16447 .route_open_target_capacity_refusal(
16448 &ctx,
16449 &RouteOpenCaller::unknown(),
16450 &route_open_frame(397, "busy", unique_project_root("target-cap")),
16451 "busy",
16452 in_flight,
16453 )
16454 .unwrap();
16455 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16456 let event = capture
16457 .events()
16458 .into_iter()
16459 .find(|event| {
16460 event.target == "control"
16461 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
16462 })
16463 .expect("target admission refusal event");
16464 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16465 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16466 drop(guards);
16467 }
16468
16469 #[cfg(unix)]
16470 #[tokio::test(start_paused = true)]
16471 async fn supervisor_list_shows_a_failed_modules_next_retry() {
16472 let registry = Arc::new(Registry::default());
16473 let handle = SupervisorHandle::new();
16474 let supervisor =
16475 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
16476 .with_handle(handle.clone());
16477 let module = supervisor
16478 .spawn(crate::ModuleSpec {
16479 script: None,
16480 carry_containment: false,
16481 module_id: "retry-list".into(),
16482 program: PathBuf::from("/bin/sh"),
16483 args: vec!["-c".into(), "exit 17".into()],
16484 env: vec![],
16485 reserved: false,
16486 reserved_prefixes: vec![],
16487 protocol: ModuleProtocol::None,
16488 overlap: Default::default(),
16489 })
16490 .unwrap();
16491 let deadline = std::time::Instant::now() + Duration::from_secs(10);
16492 while module.state().unwrap() != crate::ModuleState::Failed {
16493 assert!(
16494 std::time::Instant::now() < deadline,
16495 "waiting for budget failure"
16496 );
16497 tokio::task::yield_now().await;
16498 }
16499 let handler = ControlHandler::new(registry).with_supervisor(handle);
16500 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16501 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16502 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16503 assert_eq!(body["modules"][0]["state"], "failed");
16504 let retry = body["modules"][0]["next_retry_at_ms"]
16505 .as_u64()
16506 .expect("scheduled retry on the wire");
16507 let now = std::time::SystemTime::now()
16508 .duration_since(std::time::UNIX_EPOCH)
16509 .unwrap()
16510 .as_millis() as u64;
16511 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
16512 module.set_enabled(false).await.unwrap();
16513 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16514 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16515 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16516 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
16517 }
16518
16519 #[tokio::test]
16520 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
16521 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
16522 let supervisor =
16523 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
16524 let module = supervisor
16525 .spawn(crate::ModuleSpec {
16526 script: None,
16527 carry_containment: false,
16528 module_id: "pid-read-budget".into(),
16529 program: std::env::current_exe().unwrap(),
16530 args: vec![
16531 "--exact".into(),
16532 "control::tests::pid_resource_read_fixture".into(),
16533 ],
16534 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16535 .into_iter()
16536 .map(|key| (key.into(), dir.display().to_string()))
16537 .chain(std::iter::once((
16538 "SUBC_SUBTREE_PID_FIXTURE".into(),
16539 "1".into(),
16540 )))
16541 .collect(),
16542 reserved: false,
16543 reserved_prefixes: Vec::new(),
16544 protocol: ModuleProtocol::None,
16545 overlap: Default::default(),
16546 })
16547 .unwrap();
16548 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16549 while !matches!(
16550 module.child_resource_usage(),
16551 subc_control::ChildResourceUsage::Measured(_)
16552 ) {
16553 assert!(
16554 tokio::time::Instant::now() < ready_deadline,
16555 "fixture process was not measurable"
16556 );
16557 tokio::time::sleep(Duration::from_millis(10)).await;
16558 }
16559 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
16560 let reading = {
16561 let _guard = crate::subtree_resources::fault_read_guard().await;
16562 crate::subtree_resources::read_process(module.clone()).await
16563 };
16564 stop_resource_fixture(&module, &dir).await;
16565 assert!(
16566 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
16567 "{reading:?}"
16568 );
16569 }
16570
16571 #[test]
16572 fn pid_resource_read_fixture() {
16573 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
16574 let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
16575 let deadline = std::time::Instant::now() + Duration::from_secs(60);
16576 while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
16577 std::thread::sleep(Duration::from_millis(5));
16578 }
16579 std::fs::write(dir.join("module-stopped"), "").unwrap();
16580 }
16581 }
16582
16583 async fn stop_resource_fixture(
16584 module: &crate::supervise::SupervisedModule,
16585 dir: &std::path::Path,
16586 ) {
16587 let stopping = module.stop();
16588 tokio::pin!(stopping);
16589 tokio::select! {
16590 result = &mut stopping => result.unwrap(),
16591 () = async {
16592 while module.state().unwrap() != crate::supervise::ModuleState::Draining {
16593 tokio::time::sleep(Duration::from_millis(5)).await;
16594 }
16595 std::fs::write(dir.join("stop-module"), "").unwrap();
16596 } => stopping.await.unwrap(),
16597 }
16598 #[cfg(windows)]
16599 assert!(
16600 dir.join("module-stopped").exists(),
16601 "the resource fixture must exit cooperatively, not wait for the drain kill"
16602 );
16603 assert_eq!(
16604 module.state().unwrap(),
16605 crate::supervise::ModuleState::Stopped
16606 );
16607 }
16608
16609 #[tokio::test]
16610 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
16611 let registry = Arc::new(Registry::default());
16612 let handle = SupervisorHandle::new();
16613 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16614 .with_handle(handle.clone());
16615 supervisor
16616 .supervise_configured(
16617 crate::ModuleSpec {
16618 script: None,
16619 module_id: "disabled-resource-reading".into(),
16620 program: std::env::current_exe().unwrap(),
16621 args: Vec::new(),
16622 env: Vec::new(),
16623 reserved: false,
16624 reserved_prefixes: Vec::new(),
16625 protocol: ModuleProtocol::None,
16626 overlap: Default::default(),
16627 carry_containment: false,
16628 },
16629 false,
16630 )
16631 .unwrap();
16632 let handler = ControlHandler::new(registry).with_supervisor(handle);
16633 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16634 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16635 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16636 assert!(body["modules"][0].get("tree").is_none());
16637 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
16638 }
16639
16640 #[tokio::test]
16641 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
16642 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
16643 let registry = Arc::new(Registry::default());
16644 let handle = SupervisorHandle::new();
16645 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16646 .with_handle(handle.clone());
16647 let module = supervisor
16648 .spawn(crate::ModuleSpec {
16649 script: None,
16650 carry_containment: false,
16651 module_id: "list-process-priority".into(),
16652 program: std::env::current_exe().unwrap(),
16653 args: vec![
16654 "--exact".into(),
16655 "control::tests::pid_resource_read_fixture".into(),
16656 ],
16657 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16658 .into_iter()
16659 .map(|key| (key.into(), dir.display().to_string()))
16660 .chain(std::iter::once((
16661 "SUBC_SUBTREE_PID_FIXTURE".into(),
16662 "1".into(),
16663 )))
16664 .collect(),
16665 reserved: false,
16666 reserved_prefixes: Vec::new(),
16667 protocol: ModuleProtocol::None,
16668 overlap: Default::default(),
16669 })
16670 .unwrap();
16671 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16672 loop {
16673 let image = module.running_image_agreement().await;
16674 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
16675 break;
16676 }
16677 assert!(
16678 tokio::time::Instant::now() < ready_deadline,
16679 "the fixture process never became observable: {image:?}"
16680 );
16681 tokio::time::sleep(Duration::from_millis(10)).await;
16682 }
16683 let guard = crate::subtree_resources::fault_read_guard().await;
16684 let (target, release, _) = crate::subtree_resources::held();
16685 module.set_subtree_target_for_test(target);
16686 let handler = ControlHandler::new(registry).with_supervisor(handle);
16687 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16688 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16689 drop(release);
16690 drop(guard);
16691 stop_resource_fixture(&module, &dir).await;
16692 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16693 assert_eq!(
16694 body["modules"][0]["resources"]["status"], "measured",
16695 "the optional tree timeout must not discard the process reading: {body}"
16696 );
16697 assert!(body["modules"][0].get("tree").is_none());
16698 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
16699 assert_eq!(
16700 body["modules"][0]["pending_reload"]["image"]["status"], "match",
16701 "the tree timeout must not hide a completed image observation"
16702 );
16703 }
16704
16705 #[cfg(any(target_os = "linux", target_os = "macos"))]
16706 #[tokio::test]
16707 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
16708 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
16709 .with_daemon_provenance(
16710 std::process::id(),
16711 0,
16712 Some(std::env::current_exe().unwrap()),
16713 None,
16714 None,
16715 );
16716 let (probe, release, began) = ExecutableIdentityProbe::held();
16717 handler.daemon_provenance.probe = probe;
16718 let observe = |handler: ControlHandler, corr| {
16719 tokio::spawn(async move {
16720 let (ctx, _) = route_ctx(ConnectionId::new(corr));
16721 let frame =
16722 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
16723 .unwrap();
16724 handler
16725 .handle_supervisor_provenance(&ctx, frame, None)
16726 .await
16727 .unwrap()
16728 })
16729 };
16730 let explicit = observe(handler.clone(), 114);
16731 began.await.unwrap();
16732 let concurrent = observe(handler.clone(), 115);
16733 tokio::time::sleep(Duration::from_millis(300)).await;
16736 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16737 let list = tokio::time::timeout(
16738 Duration::from_secs(1),
16739 handler.handle_supervisor_list(frame),
16740 )
16741 .await;
16742 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
16743 drop(release);
16744 let replies = explicit.await.unwrap();
16745 let concurrent_replies = concurrent.await.unwrap();
16746 assert!(
16747 explicit_was_pending,
16748 "explicit provenance must await the full observation"
16749 );
16750 let list = list
16751 .expect("a concurrent explicit probe must not stall list")
16752 .unwrap();
16753 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
16754 assert_eq!(list_body["op"], "supervisor.list");
16755 for replies in [replies, concurrent_replies] {
16756 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16757 assert_eq!(
16758 body["daemon"]["daemon_observed"]["running_image"]["status"],
16759 "match"
16760 );
16761 }
16762 }
16763
16764 #[tokio::test]
16765 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
16766 let registry = Arc::new(Registry::default());
16767 let handle = SupervisorHandle::new();
16768 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16769 .with_handle(handle.clone());
16770 let module = supervisor
16771 .supervise_configured(
16772 crate::ModuleSpec {
16773 script: None,
16774 carry_containment: false,
16775 module_id: "memory-read".into(),
16776 program: PathBuf::from("disabled-module"),
16777 args: Vec::new(),
16778 env: Vec::new(),
16779 reserved: false,
16780 reserved_prefixes: Vec::new(),
16781 protocol: ModuleProtocol::Subc,
16782 overlap: Default::default(),
16783 },
16784 true,
16785 )
16786 .unwrap();
16787 let _guard = crate::subtree_resources::fault_read_guard().await;
16788 let (target, release, began) = crate::subtree_resources::held();
16789 let stalled_target = target.clone();
16792 let stalled = tokio::task::spawn_blocking(move || {
16793 stalled_target.tree(&subc_os::ProcessTable::default())
16794 });
16795 began.await.unwrap();
16796 module.set_subtree_target_for_test(target);
16797 let handler = ControlHandler::new(registry).with_supervisor(handle);
16798 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
16799 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16800 let replies = handler
16801 .handle_client_control_request(
16802 &ctx,
16803 frame,
16804 ClientControlRequest::SupervisorList {},
16805 &mut None,
16806 )
16807 .await
16808 .unwrap();
16809 assert!(
16810 replies.is_empty(),
16811 "list reply must be detached from the connection reader"
16812 );
16813 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16814 let replies = handler
16815 .handle_client_control_request(
16816 &ctx,
16817 frame,
16818 ClientControlRequest::SupervisorSpawnSnapshot {},
16819 &mut None,
16820 )
16821 .await
16822 .unwrap();
16823 assert_eq!(
16824 replies.len(),
16825 1,
16826 "another request must complete while the subtree read stalls"
16827 );
16828 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16829 .await
16830 .unwrap()
16831 .unwrap();
16832 release.send(()).unwrap();
16833 drop(release);
16834 stalled.await.unwrap();
16835 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
16836 assert!(
16837 body["modules"][0].get("tree").is_none(),
16838 "an unknown tree reading must be omitted after the bounded read expires"
16839 );
16840 handler
16844 .resource_tracker
16845 .read(handler.supervisor.list(), false, "released_tree_fixture")
16846 .await;
16847 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16848 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16849 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16850 assert_eq!(
16851 body["modules"][0]["tree"]["source"], "process_tree",
16852 "the next request must export the completed refresh"
16853 );
16854 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
16855 }
16856
16857 #[tokio::test]
16858 async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
16859 use subc_control::{
16860 AgentCommandResources, ChildMemoryKind, ChildResourceReading,
16861 ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
16862 TreeReading, TreeSource,
16863 };
16864 let registry = Arc::new(Registry::default());
16865 let handle = SupervisorHandle::new();
16866 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16867 .with_handle(handle.clone());
16868 supervisor
16869 .supervise_configured(
16870 crate::ModuleSpec {
16871 script: None,
16872 carry_containment: false,
16873 module_id: "budget-worker".into(),
16874 program: PathBuf::from("disabled-module"),
16875 args: Vec::new(),
16876 env: Vec::new(),
16877 reserved: false,
16878 reserved_prefixes: Vec::new(),
16879 protocol: ModuleProtocol::Subc,
16880 overlap: Default::default(),
16881 },
16882 true,
16883 )
16884 .unwrap();
16885 let handler = ControlHandler::new(registry).with_supervisor(handle);
16886 let now = u64::try_from(
16887 std::time::SystemTime::now()
16888 .duration_since(std::time::UNIX_EPOCH)
16889 .unwrap()
16890 .as_millis(),
16891 )
16892 .unwrap();
16893 let measured = now - 1_000;
16894 let hour = CpuLastHour::new(100, 60_000);
16895 let daemon = DaemonResources::new(7, 1, "test")
16896 .with_measured_at_ms(Some(measured))
16897 .with_resources(Some(ChildResourceUsage::Measured(
16898 ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
16899 )))
16900 .with_cpu_last_hour(Some(hour.clone()));
16901 let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
16902 .with_measured_at_ms(Some(measured))
16903 .with_cpu_last_hour(Some(hour.clone()));
16904 let tree = TreeReading::new(
16905 150,
16906 50,
16907 ChildMemoryKind::ResidentSet,
16908 1,
16909 TreeSource::ProcessTree,
16910 )
16911 .with_measured_at_ms(Some(measured));
16912 handler.resource_tracker.seed_committed_for_test(
16913 &handler.supervisor.list(),
16914 crate::resource_totals::Readings {
16915 modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
16916 daemon: Some(daemon),
16917 agent_commands: Some(vec![agents]),
16918 ..Default::default()
16919 },
16920 now,
16921 );
16922 list_diagnostics::scope(async {
16923 tokio::time::sleep(Duration::from_millis(110)).await;
16924 assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
16925 let request =
16926 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16927 let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
16928 let ClientControlResponse::SupervisorList {
16929 modules,
16930 daemon,
16931 agent_commands,
16932 ..
16933 } = serde_json::from_slice(&replies[0].body).unwrap()
16934 else {
16935 panic!("expected a list response")
16936 };
16937 assert!(matches!(
16938 modules[0].resources,
16939 Some(ChildResourceUsage::Unavailable {
16940 reason: ChildResourceUnavailableReason::Unreadable
16941 })
16942 ));
16943 assert_eq!(
16944 (
16945 modules[0].tree.as_ref().unwrap().cpu_ms,
16946 modules[0].tree.as_ref().unwrap().measured_at_ms
16947 ),
16948 (150, Some(measured))
16949 );
16950 let daemon = daemon.unwrap();
16951 let agents = agent_commands.unwrap();
16952 assert_eq!(daemon.measured_at_ms, Some(measured));
16953 assert_eq!(
16954 (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
16955 (1, 50, Some(measured))
16956 );
16957 for hour in [
16958 modules[0].cpu_last_hour.as_ref(),
16959 daemon.cpu_last_hour.as_ref(),
16960 agents[0].cpu_last_hour.as_ref(),
16961 ] {
16962 let hour = hour.unwrap();
16963 assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
16964 }
16965 })
16966 .await;
16967 }
16968
16969 #[tokio::test]
16970 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
16971 let registry = Arc::new(Registry::default());
16972 let handle = SupervisorHandle::new();
16973 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16974 .with_handle(handle.clone());
16975 supervisor
16976 .supervise_configured(
16977 crate::ModuleSpec {
16978 script: None,
16979 carry_containment: false,
16980 module_id: "drift".into(),
16981 program: PathBuf::from("disabled-module"),
16982 args: Vec::new(),
16983 env: Vec::new(),
16984 reserved: false,
16985 reserved_prefixes: Vec::new(),
16986 protocol: ModuleProtocol::Subc,
16987 overlap: Default::default(),
16988 },
16989 false,
16990 )
16991 .unwrap();
16992 let handler = ControlHandler::new(registry).with_supervisor(handle);
16993 for configured in [None, Some(false), Some(true)] {
16994 handler.capability_evaluator.configure(
16995 configured.map(|enabled| ("drift".to_string(), enabled)),
16996 BTreeMap::new(),
16997 );
16998 let frame =
16999 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
17000 let replies = handler
17001 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
17002 .await
17003 .unwrap();
17004 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
17005 assert!(body.get("configured_enabled").is_none());
17006 assert_eq!(body["op"], "supervisor.ack");
17007 let frame =
17008 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
17009 let replies = handler.handle_supervisor_list(frame).await.unwrap();
17010 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
17011 assert_eq!(
17012 body["modules"][0].get("configured_enabled"),
17013 configured.map(Value::Bool).as_ref()
17014 );
17015 assert!(body["modules"][0].get("tree").is_none());
17016 assert!(body["modules"][0].get("subtree").is_none());
17017 }
17018 }
17019
17020 #[tokio::test(flavor = "current_thread")]
17021 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
17022 let handler = ControlHandler::new(Arc::new(Registry::default()));
17023 let capture = EventCapture::default();
17024 let _subscriber =
17025 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17026 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
17027 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
17028
17029 let _ = handler
17030 .handle_client_control_request(
17031 &ctx,
17032 frame,
17033 ClientControlRequest::SupervisorSetEnabled {
17034 module_id: "broca".to_string(),
17035 enabled: false,
17036 },
17037 &mut None,
17038 )
17039 .await
17040 .unwrap();
17041
17042 let events = capture
17043 .events()
17044 .into_iter()
17045 .filter(|event| {
17046 event.target == "control"
17047 && event.fields.get("message").map(String::as_str)
17048 == Some("supervisor request received")
17049 })
17050 .collect::<Vec<_>>();
17051 assert_eq!(events.len(), 1, "one supervisor request log line");
17052 let event = &events[0];
17053 assert_eq!(event.level, tracing::Level::INFO);
17054 assert_eq!(
17055 event.fields.get("op"),
17056 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
17057 );
17058 assert_eq!(
17059 event.fields.get("module_id"),
17060 Some(&"\"broca\"".to_string())
17061 );
17062 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
17063 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
17064 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
17065 }
17066
17067 #[tokio::test(flavor = "current_thread")]
17068 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
17069 let registry = Arc::new(Registry::default());
17070 let module_connection = ConnectionId::new(303);
17071 registry
17072 .register_with_control_ops(
17073 manifest("aft", PROTOCOL_VERSION),
17074 PROTOCOL_VERSION,
17075 module_connection,
17076 module_baseline_control_ops(),
17077 )
17078 .unwrap();
17079 let handler = ControlHandler::new(registry);
17080 let capture = EventCapture::default();
17081 let _subscriber =
17082 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17083 let (ctx, _rx) = route_ctx(module_connection);
17084 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
17085
17086 let _ = handler
17087 .handle_client_control_request(
17088 &ctx,
17089 frame,
17090 ClientControlRequest::SupervisorSetEnabled {
17091 module_id: "broca".to_string(),
17092 enabled: false,
17093 },
17094 &mut None,
17095 )
17096 .await
17097 .unwrap();
17098
17099 let callers = capture
17100 .events()
17101 .into_iter()
17102 .filter(|event| {
17103 event.fields.get("message").map(String::as_str)
17104 == Some("supervisor request received")
17105 })
17106 .map(|event| event.fields.get("caller").cloned())
17107 .collect::<Vec<_>>();
17108 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
17109 }
17110
17111 #[tokio::test(flavor = "current_thread")]
17112 async fn supervisor_list_does_not_log_request_received() {
17113 let handler = ControlHandler::new(Arc::new(Registry::default()));
17114 let capture = EventCapture::default();
17115 let _subscriber =
17116 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17117 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
17118 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
17119
17120 handler
17121 .handle_client_control_request(
17122 &ctx,
17123 frame,
17124 ClientControlRequest::SupervisorList {},
17125 &mut None,
17126 )
17127 .await
17128 .unwrap();
17129
17130 assert!(capture.events().into_iter().all(|event| {
17131 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17132 }));
17133 }
17134
17135 #[tokio::test(flavor = "current_thread")]
17136 async fn supervisor_rescan_preview_does_not_log_request_received() {
17137 let handler = ControlHandler::new(Arc::new(Registry::default()));
17138 let capture = EventCapture::default();
17139 let _subscriber =
17140 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17141 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
17142 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
17143
17144 handler
17145 .handle_client_control_request(
17146 &ctx,
17147 frame,
17148 ClientControlRequest::SupervisorRescan { preview: true },
17149 &mut None,
17150 )
17151 .await
17152 .unwrap();
17153
17154 assert!(capture.events().into_iter().all(|event| {
17155 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17156 }));
17157 }
17158
17159 struct ApproveOperator;
17160 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
17161 fn prompt(
17162 &self,
17163 _: &str,
17164 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
17165 ) -> crate::operator_confirm::ProviderResult {
17166 crate::operator_confirm::ProviderResult::Approved
17167 }
17168 }
17169
17170 #[tokio::test]
17171 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
17172 assert_eq!(
17173 ControlHandler::default().privileged_operator_prompt,
17174 PrivilegedOperatorPrompt::Log
17175 );
17176 for (initial, changed, running) in [
17177 ("log", "enforce", PrivilegedOperatorPrompt::Log),
17178 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
17179 ] {
17180 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
17181 let path = dir.join("subc.jsonc");
17182 std::fs::write(
17183 &path,
17184 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
17185 )
17186 .unwrap();
17187 let config = crate::daemon_config::load(&path).unwrap().unwrap();
17188 let registry = Arc::new(Registry::default());
17189 let handle = SupervisorHandle::new();
17190 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
17191 .with_handle(handle.clone());
17192 let handler = ControlHandler::new(registry)
17193 .with_supervisor(handle)
17194 .with_privileged_operator_prompt(config.privileged_operator_prompt)
17195 .with_supervisor_rescan(supervisor, &path, config.port);
17196 handler.forwarding.operator_confirms().configure(
17197 Arc::new(ApproveOperator),
17198 crate::operator_confirm::OperatorLimits::default(),
17199 );
17200 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
17201
17202 for (disk_mode, needs_restart) in [
17203 (Some(initial), false),
17204 (Some(changed), true),
17205 (Some(changed), true),
17206 (None, running == PrivilegedOperatorPrompt::Enforce),
17207 (Some(initial), false),
17208 ] {
17209 let mut doc = json!({"version":1});
17210 if let Some(mode) = disk_mode {
17211 doc["privileged_operator_prompt"] = json!(mode);
17212 }
17213 std::fs::write(&path, doc.to_string()).unwrap();
17214 for preview in [true, false] {
17215 let request = ClientControlRequest::SupervisorRescan { preview };
17216 let frame = Frame::build(
17217 FrameType::Request,
17218 control_flags(),
17219 0,
17220 0,
17221 1,
17222 serde_json::to_vec(&request).unwrap(),
17223 )
17224 .unwrap();
17225 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17226 assert_eq!(frames.len(), 1);
17227 let ClientControlResponse::SupervisorRescan { result } =
17228 serde_json::from_slice(&frames[0].body).unwrap()
17229 else {
17230 panic!("expected a rescan response");
17231 };
17232 let expected = if needs_restart {
17233 vec!["privileged_operator_prompt"]
17234 } else {
17235 Vec::new()
17236 };
17237 assert_eq!(result.restart_required, expected);
17238 assert_eq!(handler.privileged_operator_prompt, running);
17239 assert_eq!(
17240 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
17241 running
17242 );
17243 }
17244 }
17245 }
17246 }
17247
17248 #[tokio::test]
17249 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
17250 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
17251 let path = dir.join("subc.jsonc");
17252 let registry = Arc::new(Registry::default());
17253 let handle = SupervisorHandle::new();
17254 let supervisor =
17255 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
17256 let handler = ControlHandler::new(registry)
17257 .with_supervisor(handle)
17258 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
17259 .with_supervisor_rescan(supervisor, &path, None);
17260 handler.forwarding.operator_confirms().configure(
17261 Arc::new(ApproveOperator),
17262 crate::operator_confirm::OperatorLimits::default(),
17263 );
17264 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
17265 for preview in [true, false] {
17266 std::fs::write(
17267 &path,
17268 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
17269 )
17270 .unwrap();
17271 let request = ClientControlRequest::SupervisorRescan { preview };
17272 let frame = Frame::build(
17273 FrameType::Request,
17274 control_flags(),
17275 0,
17276 0,
17277 1,
17278 serde_json::to_vec(&request).unwrap(),
17279 )
17280 .unwrap();
17281 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17282 assert_eq!(frames.len(), 1);
17283 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
17284 assert_eq!(
17285 handler.privileged_operator_prompt,
17286 PrivilegedOperatorPrompt::Enforce
17287 );
17288 }
17289 }
17290
17291 #[tokio::test]
17296 async fn route_open_refusal_names_the_check_that_refused() {
17297 let handler = ControlHandler::new(Arc::new(Registry::default()));
17298 let capture = EventCapture::default();
17299 let _subscriber =
17300 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17301 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17302 let response = handler
17303 .handle_control_frame(
17304 &ctx,
17305 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
17306 )
17307 .await
17308 .unwrap();
17309
17310 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17311 let event = capture
17312 .events()
17313 .into_iter()
17314 .find(|event| {
17315 event.target == "control"
17316 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17317 })
17318 .expect("route.open refusal event");
17319 assert_eq!(
17320 event.fields.get("reason"),
17321 Some(&"\"not_registered\"".to_string())
17322 );
17323 }
17324
17325 #[tokio::test]
17326 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
17327 let registry = Arc::new(Registry::default());
17328 let supervisor_handle = SupervisorHandle::new();
17329 let supervisor =
17330 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17331 .with_handle(supervisor_handle.clone())
17332 .with_connection_file_path(std::env::temp_dir().join(format!(
17333 "subc-route-open-refusal-info-{}",
17334 std::process::id()
17335 )));
17336 let module = supervisor
17337 .supervise_configured(
17338 ModuleSpec {
17339 script: None,
17340 carry_containment: false,
17341 module_id: "warming".to_string(),
17342 program: fake_aft_stub_path(),
17343 args: Vec::new(),
17344 env: Vec::new(),
17345 reserved: false,
17346 reserved_prefixes: Vec::new(),
17347 protocol: ModuleProtocol::Subc,
17348 overlap: Default::default(),
17349 },
17350 true,
17351 )
17352 .unwrap();
17353 assert_eq!(module.state().unwrap(), ModuleState::Running);
17354
17355 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17356 assert!(handler
17357 .counters()
17358 .snapshot()
17359 .get("route_open_refused_by_code")
17360 .is_none());
17361 let capture = EventCapture::default();
17362 let _subscriber =
17363 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17364 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
17365 let response = handler
17366 .handle_control_frame(
17367 &ctx,
17368 route_open_frame(394, "warming", unique_project_root("refusal-info")),
17369 )
17370 .await
17371 .unwrap();
17372 module.stop().await.unwrap();
17373
17374 assert_eq!(parse_error(&response[0])["code"], "module_warming");
17375 let event = capture
17376 .events()
17377 .into_iter()
17378 .find(|event| {
17379 event.target == "control"
17380 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
17381 })
17382 .expect("route.open refusal event");
17383 assert_eq!(
17384 event.fields.get("module_id"),
17385 Some(&"\"warming\"".to_string())
17386 );
17387 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
17388 assert_eq!(
17389 event.fields.get("reason"),
17390 Some(&"\"supervised_not_registered\"".to_string())
17391 );
17392 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
17393 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
17394 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
17395 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
17396 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
17397 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
17398 assert_eq!(
17399 handler.counters().snapshot()["route_open_refused_by_code"],
17400 json!({ "module_warming": 1 })
17401 );
17402 }
17403
17404 const OUTAGE_START: &str = "route.open refusing module: not serving";
17405 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
17406
17407 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
17408 capture
17409 .events()
17410 .into_iter()
17411 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
17412 .collect()
17413 }
17414
17415 fn supervise_stub(
17416 registry: &Arc<Registry>,
17417 module_id: &str,
17418 enabled: bool,
17419 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
17420 let supervisor_handle = SupervisorHandle::new();
17421 let supervisor =
17422 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
17423 .with_handle(supervisor_handle.clone())
17424 .with_connection_file_path(std::env::temp_dir().join(format!(
17425 "subc-route-outage-{module_id}-{}",
17426 std::process::id()
17427 )));
17428 let module = supervisor
17429 .supervise_configured(
17430 ModuleSpec {
17431 script: None,
17432 carry_containment: false,
17433 module_id: module_id.to_string(),
17434 program: fake_aft_stub_path(),
17435 args: Vec::new(),
17436 env: Vec::new(),
17437 reserved: false,
17438 reserved_prefixes: Vec::new(),
17439 protocol: ModuleProtocol::Subc,
17440 overlap: Default::default(),
17441 },
17442 enabled,
17443 )
17444 .unwrap();
17445 (supervisor_handle, module)
17446 }
17447
17448 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
17449 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
17450 module_id: module_id.to_string(),
17451 drain_timeout_ms: Some(50),
17452 })
17453 .unwrap();
17454 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
17455 }
17456
17457 #[test]
17461 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
17462 let registry = Arc::new(Registry::default());
17463 let forwarding = Arc::new(ForwardingTable::default());
17464 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17465 let second = ControlHandler::with_forwarding(registry, forwarding);
17466 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
17467 }
17468
17469 #[tokio::test(flavor = "current_thread")]
17473 async fn route_open_unknown_module_refusals_add_no_outage_state() {
17474 let handler = ControlHandler::new(Arc::new(Registry::default()));
17475 let capture = EventCapture::default();
17476 let _subscriber =
17477 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17478 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
17479 for corr in 0..8 {
17480 let response = handler
17481 .handle_control_frame(
17482 &ctx,
17483 route_open_frame(
17484 380 + corr,
17485 &format!("nobody-{corr}"),
17486 unique_project_root("outage-unknown"),
17487 ),
17488 )
17489 .await
17490 .unwrap();
17491 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17492 }
17493
17494 assert_eq!(handler.route_outages.tracked_module_count(), 0);
17495 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
17496 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
17497 }
17498
17499 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17504 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
17505 let registry = Arc::new(Registry::default());
17506 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
17507 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17508 let capture = EventCapture::default();
17509 let _subscriber =
17510 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17511 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
17512 handler.route_outages.record_accepted("outage-restart");
17515
17516 let response = handler
17517 .handle_control_frame(
17518 &ctx,
17519 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
17520 )
17521 .await
17522 .unwrap();
17523 assert_eq!(response[0].header.ty, FrameType::Error);
17524 let starts = outage_lines(&capture, OUTAGE_START);
17525 assert_eq!(starts.len(), 1, "{starts:?}");
17526 assert_eq!(starts[0].level, tracing::Level::WARN);
17527 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
17528 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
17529 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
17530 handler.route_outages.record_accepted("outage-restart");
17531 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
17532
17533 tokio::time::timeout(Duration::from_secs(10), async {
17537 while module.state().unwrap() != crate::ModuleState::Failed {
17538 tokio::task::yield_now().await;
17539 }
17540 })
17541 .await
17542 .expect("stub exit must be recorded before the operator restart");
17543
17544 let restart = handler
17545 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
17546 .await
17547 .unwrap();
17548 assert_eq!(
17549 restart[0].header.ty,
17550 FrameType::Response,
17551 "{:?}",
17552 parse_error(&restart[0])
17553 );
17554 handler
17555 .handle_control_frame(
17556 &ctx,
17557 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
17558 )
17559 .await
17560 .unwrap();
17561 module.stop().await.unwrap();
17562
17563 let starts = outage_lines(&capture, OUTAGE_START);
17564 assert_eq!(starts.len(), 2, "{starts:?}");
17565 assert_eq!(starts[1].level, tracing::Level::INFO);
17566 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
17567 }
17568
17569 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17573 async fn failed_operator_restart_leaves_no_operator_mark() {
17574 let registry = Arc::new(Registry::default());
17575 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
17576 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17577 let capture = EventCapture::default();
17578 let _subscriber =
17579 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17580 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
17581 handler.route_outages.record_accepted("outage-disabled");
17582
17583 let restart = handler
17584 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
17585 .await
17586 .unwrap();
17587 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
17588 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
17589
17590 handler
17591 .handle_control_frame(
17592 &ctx,
17593 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
17594 )
17595 .await
17596 .unwrap();
17597 let starts = outage_lines(&capture, OUTAGE_START);
17598 assert_eq!(starts.len(), 1, "{starts:?}");
17599 assert_eq!(starts[0].level, tracing::Level::WARN);
17600 }
17601
17602 #[tokio::test(flavor = "current_thread")]
17603 async fn route_open_unknown_module_escapes_target_module_id() {
17604 let handler = ControlHandler::new(Arc::new(Registry::default()));
17605 let capture = EventCapture::default();
17606 let _subscriber =
17607 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17608 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
17609 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17610 let response = handler
17611 .handle_control_frame(
17612 &ctx,
17613 route_open_frame(
17614 395,
17615 hostile_module_id,
17616 unique_project_root("hostile-target-module-id"),
17617 ),
17618 )
17619 .await
17620 .unwrap();
17621
17622 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17623 let event = capture
17624 .events()
17625 .into_iter()
17626 .find(|event| {
17627 event.target == "control"
17628 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17629 })
17630 .expect("route.open unknown-module refusal event");
17631 let logged = event.fields.get("module_id").expect("module_id field");
17632 assert!(!logged.bytes().any(|byte| byte < 0x20));
17633 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17634 }
17635
17636 #[tokio::test(flavor = "current_thread")]
17639 async fn route_open_refusal_carries_daemon_origin() {
17640 let handler = ControlHandler::new(Arc::new(Registry::default()));
17641 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17642 let response = handler
17643 .handle_control_frame(
17644 &ctx,
17645 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
17646 )
17647 .await
17648 .unwrap();
17649
17650 assert_eq!(response.len(), 1);
17651 assert_eq!(response[0].header.ty, FrameType::Error);
17652 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17653 assert!(
17654 response[0].header.flags.is_daemon_origin(),
17655 "route.open refusal must carry daemon origin: {:?}",
17656 response[0].header.flags
17657 );
17658 }
17659
17660 #[tokio::test(flavor = "current_thread")]
17664 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
17665 let registry = Arc::new(Registry::default());
17666 let forwarding = Arc::new(ForwardingTable::default());
17667 let handler =
17668 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17669 let module_connection = ConnectionId::new(95);
17670 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17671 hello_via_sink(
17672 &handler,
17673 &module_ctx,
17674 &mut module_rx,
17675 hello_frame("aft", PROTOCOL_VERSION, 395),
17676 )
17677 .await;
17678 forwarding
17679 .begin_module_drain("aft", RouteCloseReason::Restart)
17680 .unwrap()
17681 .expect("the module endpoint drains");
17682
17683 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
17684 let response = handler
17685 .handle_control_frame(
17686 &client_ctx,
17687 route_open_frame(396, "aft", unique_project_root("drain-origin")),
17688 )
17689 .await
17690 .unwrap();
17691
17692 assert_eq!(response.len(), 1);
17693 assert_eq!(response[0].header.ty, FrameType::Error);
17694 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17695 assert!(
17696 response[0].header.flags.is_daemon_origin(),
17697 "drain module_reloading must carry daemon origin: {:?}",
17698 response[0].header.flags
17699 );
17700 }
17701
17702 #[tokio::test(flavor = "current_thread")]
17706 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
17707 let registry = Arc::new(Registry::default());
17708 let forwarding = Arc::new(ForwardingTable::default());
17709 let handler =
17710 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17711 let module_connection = ConnectionId::new(95);
17712 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17713 hello_via_sink(
17714 &handler,
17715 &module_ctx,
17716 &mut module_rx,
17717 hello_frame("aft", PROTOCOL_VERSION, 395),
17718 )
17719 .await;
17720
17721 let client_connection = ConnectionId::new(96);
17722 let (client_ctx, _client_rx) = route_ctx(client_connection);
17723 let (route_task, bind) = relay_route_open(
17724 &handler,
17725 client_connection,
17726 &client_ctx.egress,
17727 &mut module_rx,
17728 396,
17729 "aft",
17730 "spoofed-reloading",
17731 )
17732 .await;
17733 let rejection = Frame::build(
17736 FrameType::Error,
17737 crate::router::daemon_error_flags(),
17738 0,
17739 0,
17740 bind.header.corr,
17741 serde_json::to_vec(&ErrorBody::new(
17742 "module_reloading",
17743 "module refused route.bind",
17744 ))
17745 .unwrap(),
17746 )
17747 .unwrap();
17748 handler
17749 .handle_control_frame(&module_ctx, rejection)
17750 .await
17751 .unwrap();
17752
17753 let response = route_task.await.unwrap();
17754 assert_eq!(response.len(), 1);
17755 assert_eq!(response[0].header.ty, FrameType::Error);
17756 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17757 assert!(
17758 !response[0].header.flags.is_daemon_origin(),
17759 "a module's own rejection must not carry daemon origin: {:?}",
17760 response[0].header.flags
17761 );
17762 }
17763
17764 #[tokio::test(flavor = "current_thread")]
17765 async fn route_open_module_rejection_uses_daemon_counter_key() {
17766 let registry = Arc::new(Registry::default());
17767 let forwarding = Arc::new(ForwardingTable::default());
17768 let handler =
17769 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17770 let module_connection = ConnectionId::new(95);
17771 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17772 hello_via_sink(
17773 &handler,
17774 &module_ctx,
17775 &mut module_rx,
17776 hello_frame("aft", PROTOCOL_VERSION, 395),
17777 )
17778 .await;
17779
17780 let client_connection = ConnectionId::new(96);
17781 let (client_ctx, _client_rx) = route_ctx(client_connection);
17782 let capture = EventCapture::default();
17783 static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
17790 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
17791 let _subscriber =
17792 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17793 let (route_task, bind) = relay_route_open(
17794 &handler,
17795 client_connection,
17796 &client_ctx.egress,
17797 &mut module_rx,
17798 396,
17799 "aft",
17800 "hostile-module-code",
17801 )
17802 .await;
17803 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
17804 let rejection = Frame::build(
17805 FrameType::Error,
17806 control_flags(),
17807 0,
17808 0,
17809 bind.header.corr,
17810 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
17811 )
17812 .unwrap();
17813 handler
17814 .handle_control_frame(&module_ctx, rejection)
17815 .await
17816 .unwrap();
17817
17818 let response = route_task.await.unwrap();
17819 assert_eq!(parse_error(&response[0])["code"], hostile_code);
17820 let counters = handler.counters().snapshot();
17821 assert_eq!(
17822 counters["route_open_refused_by_code"],
17823 json!({ "module_rejected": 1 })
17824 );
17825 assert!(counters["route_open_refused_by_code"]
17826 .get(hostile_code)
17827 .is_none());
17828
17829 let event = capture
17830 .events()
17831 .into_iter()
17832 .find(|event| {
17833 event.target == "control"
17834 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
17835 })
17836 .expect("route.open module-rejection refusal event");
17837 let logged = event.fields.get("module_code").expect("module_code field");
17838 assert!(!logged.bytes().any(|byte| byte < 0x20));
17839 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17840 }
17841
17842 #[tokio::test]
17843 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
17844 let registry = Arc::new(Registry::default());
17845 let supervisor_handle = SupervisorHandle::new();
17846 let missing_program = std::env::temp_dir().join(format!(
17847 "subc-route-open-missing-program-{}",
17848 std::process::id()
17849 ));
17850 let supervisor =
17851 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17852 .with_handle(supervisor_handle.clone());
17853 let module = supervisor
17854 .supervise_configured(
17855 ModuleSpec {
17856 script: None,
17857 carry_containment: false,
17858 module_id: "failed".to_string(),
17859 program: missing_program,
17860 args: Vec::new(),
17861 env: Vec::new(),
17862 reserved: false,
17863 reserved_prefixes: Vec::new(),
17864 protocol: ModuleProtocol::Subc,
17865 overlap: Default::default(),
17866 },
17867 true,
17868 )
17869 .unwrap();
17870 assert_eq!(module.state().unwrap(), ModuleState::Failed);
17871
17872 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17873 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
17874 let response = handler
17875 .handle_control_frame(
17876 &ctx,
17877 route_open_frame(305, "failed", unique_project_root("failed")),
17878 )
17879 .await
17880 .unwrap();
17881
17882 assert_eq!(response[0].header.ty, FrameType::Error);
17883 let error = parse_error(&response[0]);
17884 assert_eq!(error["code"], "target_unavailable");
17885 assert!(error["message"]
17886 .as_str()
17887 .unwrap()
17888 .contains("state=failed, enabled=true, live=false"));
17889 }
17890
17891 #[tokio::test]
17892 async fn route_open_role_mismatch_remains_target_unavailable() {
17893 let registry = Arc::new(Registry::default());
17894 let handler = ControlHandler::new(Arc::clone(®istry));
17895 handler
17896 .handle_control(
17897 ConnectionId::new(41),
17898 non_routable_hello_frame_with_control_ops("health-only", 306, None),
17899 )
17900 .unwrap();
17901
17902 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
17903 let response = handler
17904 .handle_control_frame(
17905 &ctx,
17906 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
17907 )
17908 .await
17909 .unwrap();
17910
17911 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17912 assert!(parse_error(&response[0])["message"]
17913 .as_str()
17914 .unwrap()
17915 .contains("does not provide the requested target"));
17916 }
17917
17918 #[tokio::test]
17919 async fn route_open_inactive_registration_remains_target_unavailable() {
17920 let registry = Arc::new(Registry::default());
17921 let handler = ControlHandler::new(Arc::clone(®istry));
17922 handler
17923 .handle_control(
17924 ConnectionId::new(43),
17925 hello_frame("inactive", PROTOCOL_VERSION, 308),
17926 )
17927 .unwrap();
17928 assert!(registry
17929 .set_module_state_for_test("inactive", ChannelState::Closed)
17930 .unwrap());
17931
17932 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
17933 let response = handler
17934 .handle_control_frame(
17935 &ctx,
17936 route_open_frame(309, "inactive", unique_project_root("inactive")),
17937 )
17938 .await
17939 .unwrap();
17940
17941 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17942 assert!(parse_error(&response[0])["message"]
17943 .as_str()
17944 .unwrap()
17945 .contains("is not active"));
17946 }
17947
17948 #[tokio::test]
17949 async fn late_health_reply_is_recorded_through_the_module_response_path() {
17950 let registry = Arc::new(Registry::default());
17951 let forwarding = Arc::new(ForwardingTable::default());
17952 let supervisor_handle = SupervisorHandle::new();
17953 let supervisor =
17954 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
17955 .with_forwarding(Arc::clone(&forwarding))
17956 .with_handle(supervisor_handle.clone());
17957 let module = supervisor
17958 .supervise_configured(
17959 crate::ModuleSpec {
17960 script: None,
17961 carry_containment: false,
17962 module_id: "late-health-response".to_string(),
17963 program: PathBuf::from("disabled-module"),
17964 args: Vec::new(),
17965 env: Vec::new(),
17966 reserved: false,
17967 reserved_prefixes: Vec::new(),
17968 protocol: ModuleProtocol::Subc,
17969 overlap: Default::default(),
17970 },
17971 false,
17972 )
17973 .unwrap();
17974 let handler =
17975 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17976 .with_supervisor(supervisor_handle);
17977 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
17978 handler
17979 .handle_control_frame(
17980 &module_ctx,
17981 hello_frame_with_control_ops(
17982 "late-health-response",
17983 PROTOCOL_VERSION,
17984 7,
17985 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17986 ),
17987 )
17988 .await
17989 .unwrap();
17990 let probe_started_at = Instant::now() - Duration::from_millis(80);
17991 let pending = forwarding
17992 .begin_health_probe_rpc_for(
17993 "late-health-response",
17994 MODULE_CONTROL_OP_HEALTH_CHECK,
17995 probe_started_at,
17996 Instant::now() - Duration::from_millis(1),
17997 )
17998 .unwrap();
17999 assert!(forwarding
18000 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
18001 .unwrap());
18002
18003 let responses = handler
18004 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
18005 .await
18006 .unwrap();
18007
18008 assert!(responses.is_empty());
18009 let health = module.status().unwrap().health;
18010 assert_eq!(health.late_answer_count, 1);
18011 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
18012 }
18013
18014 #[tokio::test]
18015 async fn health_probe_timeout_and_module_death_are_typed() {
18016 let registry = Arc::new(Registry::default());
18017 let forwarding = Arc::new(ForwardingTable::default());
18018 let handler =
18019 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18020 .with_health_probe_timeout(Duration::from_millis(50));
18021 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
18022 hello_via_sink(
18023 &handler,
18024 &module_ctx,
18025 &mut module_rx,
18026 hello_frame_with_control_ops(
18027 "aft",
18028 PROTOCOL_VERSION,
18029 7,
18030 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
18031 ),
18032 )
18033 .await;
18034
18035 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
18036 let responses = handler
18037 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
18038 .await
18039 .unwrap();
18040 assert_eq!(responses[0].header.ty, FrameType::Error);
18041 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
18042 let _ = module_rx.try_recv();
18043
18044 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
18045 let health_handler = handler.clone();
18046 let death_task = tokio::spawn(async move {
18047 health_handler
18048 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
18049 .await
18050 .unwrap()
18051 });
18052 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
18053 .await
18054 .unwrap()
18055 .unwrap();
18056 handler
18057 .cleanup_connection(module_ctx.connection_id)
18058 .unwrap();
18059 let responses = death_task.await.unwrap();
18060 assert_eq!(responses[0].header.ty, FrameType::Error);
18061 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
18062 }
18063
18064 #[test]
18065 fn hello_requires_exact_protocol_version() {
18066 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
18067 let registry = Arc::new(Registry::default());
18068 let handler = ControlHandler::new(Arc::clone(®istry));
18069 let responses = handler
18070 .handle_control(
18071 ConnectionId::new(connection),
18072 hello_frame("aft", offered, 9),
18073 )
18074 .unwrap();
18075
18076 assert_eq!(responses.len(), 1);
18077 assert_eq!(responses[0].header.ty, FrameType::Error);
18078 let error = parse_error(&responses[0]);
18079 assert_eq!(error["code"], "version_unsupported");
18080 assert!(registry.get_module("aft").unwrap().is_none());
18081 assert_eq!(registry.active_registration_count().unwrap(), 0);
18082 }
18083 }
18084
18085 #[test]
18086 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
18087 let registry = Arc::new(Registry::default());
18088 let forwarding = Arc::new(ForwardingTable::default());
18089 let handler =
18090 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18091 let module_connection = ConnectionId::new(301);
18092 let registration = registry
18093 .register_with_control_ops(
18094 manifest("aft-push", PROTOCOL_VERSION),
18095 PROTOCOL_VERSION,
18096 module_connection,
18097 module_baseline_control_ops(),
18098 )
18099 .unwrap();
18100 let (module_tx, _module_rx) = mpsc::channel(8);
18101 let endpoint = forwarding
18102 .register_module_connection(
18103 module_connection,
18104 "aft-push".to_string(),
18105 PROTOCOL_VERSION,
18106 manifest_concurrency(®istration.manifest),
18107 FrameSink::new(module_tx),
18108 )
18109 .unwrap();
18110
18111 let unknown = Frame::build(
18113 FrameType::Push,
18114 control_flags(),
18115 0,
18116 0,
18117 5,
18118 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
18119 )
18120 .unwrap();
18121 let out = handler.handle_status_update(endpoint, unknown).unwrap();
18122 assert!(
18123 out.is_empty(),
18124 "unknown push op must be ignored, got {out:?}"
18125 );
18126
18127 let malformed = Frame::build(
18129 FrameType::Push,
18130 control_flags(),
18131 0,
18132 0,
18133 6,
18134 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
18135 )
18136 .unwrap();
18137 let out = handler.handle_status_update(endpoint, malformed).unwrap();
18138 assert_eq!(out.len(), 1);
18139 assert_eq!(out[0].header.ty, FrameType::Error);
18140 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
18141 }
18142
18143 #[test]
18144 fn hello_rejected_when_connection_already_owns_client_routes() {
18145 let registry = Arc::new(Registry::default());
18146 let forwarding = Arc::new(ForwardingTable::default());
18147 let handler =
18148 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18149 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
18151 let client_connection = ConnectionId::new(202);
18152
18153 let responses = handler
18156 .handle_control(
18157 client_connection,
18158 hello_frame("aft-second", PROTOCOL_VERSION, 9),
18159 )
18160 .unwrap();
18161 assert_eq!(responses[0].header.ty, FrameType::Error);
18162 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
18163 assert!(registry.get_module("aft-second").unwrap().is_none());
18164 }
18165
18166 #[tokio::test]
18167 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
18168 let registry = Arc::new(Registry::default());
18169 let forwarding = Arc::new(ForwardingTable::default());
18170 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
18171 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
18172 hello_via_sink(
18173 &handler,
18174 &module_ctx,
18175 &mut module_rx,
18176 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
18177 )
18178 .await;
18179 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
18180 let pending = forwarding
18181 .begin_route_bind_relay_for_test(
18182 client_ctx.connection_id,
18183 client_ctx.egress.clone(),
18184 2,
18185 "alpha",
18186 )
18187 .unwrap();
18188 forwarding
18189 .complete_pending_relay(
18190 module_ctx.connection_id,
18191 pending.corr,
18192 RouteBindRelayOutcome::Accepted,
18193 )
18194 .unwrap();
18195 client_rx.try_recv().unwrap();
18196 for module_id in ["beta", "alpha"] {
18197 let replies = handler
18198 .handle_control_frame(
18199 &module_ctx,
18200 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
18201 )
18202 .await
18203 .unwrap();
18204 assert_eq!(replies.len(), 1, "second HELLO must be refused");
18205 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
18206 }
18207 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
18208 assert!(registry.get_module("beta").unwrap().is_none());
18209 assert!(matches!(
18210 forwarding
18211 .lookup_data_route(
18212 client_ctx.connection_id,
18213 pending.client_channel,
18214 pending.client_epoch,
18215 )
18216 .unwrap(),
18217 DataRoute::Client(DataRouteState::Bound(_))
18218 ));
18219 assert!(handler
18220 .hello_launch_nonces
18221 .lock()
18222 .unwrap()
18223 .presented(module_ctx.connection_id, Some("alpha-nonce")));
18224 assert!(module_rx.try_recv().is_err());
18225 }
18226
18227 #[test]
18228 fn reserved_module_hello_requires_matching_launch_nonce() {
18229 let registry = Arc::new(Registry::default());
18230 let supervisor = SupervisorHandle::new();
18231 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
18234 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18235
18236 let no_nonce = handler
18238 .handle_control(
18239 ConnectionId::new(1),
18240 hello_frame("vault", PROTOCOL_VERSION, 1),
18241 )
18242 .unwrap();
18243 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
18244 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
18245 assert!(registry.get_module("vault").unwrap().is_none());
18246
18247 let wrong = handler
18249 .handle_control(
18250 ConnectionId::new(2),
18251 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
18252 )
18253 .unwrap();
18254 assert_eq!(wrong[0].header.ty, FrameType::Error);
18255 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
18256 assert!(registry.get_module("vault").unwrap().is_none());
18257
18258 let ok = handler
18260 .handle_control(
18261 ConnectionId::new(3),
18262 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
18263 )
18264 .unwrap();
18265 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
18266 assert!(registry.get_module("vault").unwrap().is_some());
18267 }
18268
18269 #[test]
18270 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
18271 let registry = Arc::new(Registry::default());
18272 let supervisor = SupervisorHandle::new();
18273 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18274 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18275 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18276
18277 let squat = handler
18278 .handle_control(
18279 ConnectionId::new(1),
18280 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
18281 )
18282 .unwrap();
18283 assert_eq!(squat[0].header.ty, FrameType::Error);
18284 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
18285 assert!(parse_error(&squat[0])["message"]
18286 .as_str()
18287 .unwrap()
18288 .contains("fed:"));
18289
18290 let accepted_peer = handler
18291 .handle_control(
18292 ConnectionId::new(2),
18293 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
18294 )
18295 .unwrap();
18296 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
18297
18298 let accepted_short = handler
18299 .handle_control(
18300 ConnectionId::new(3),
18301 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
18302 )
18303 .unwrap();
18304 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
18305
18306 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
18307 let response = handler
18308 .handle_control(
18309 ConnectionId::new(conn),
18310 hello_frame(module_id, PROTOCOL_VERSION, conn),
18311 )
18312 .unwrap();
18313 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
18314 }
18315 }
18316
18317 #[test]
18318 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
18319 let registry = Arc::new(Registry::default());
18320 let supervisor = SupervisorHandle::new();
18321 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18322 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18323 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
18324 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18325
18326 let owner_nonce = handler
18327 .handle_control(
18328 ConnectionId::new(1),
18329 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
18330 )
18331 .unwrap();
18332 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
18333 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
18334 assert!(registry.get_module("fed:special").unwrap().is_none());
18335
18336 let exact_nonce = handler
18337 .handle_control(
18338 ConnectionId::new(2),
18339 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
18340 )
18341 .unwrap();
18342 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
18343 assert!(registry.get_module("fed:special").unwrap().is_some());
18344 }
18345
18346 #[test]
18347 fn non_reserved_module_ignores_launch_nonce() {
18348 let registry = Arc::new(Registry::default());
18349 let handler = ControlHandler::new(Arc::clone(®istry));
18352 let no_nonce = handler
18353 .handle_control(
18354 ConnectionId::new(1),
18355 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
18356 )
18357 .unwrap();
18358 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
18359 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
18360
18361 let echoed_nonce = handler
18362 .handle_control(
18363 ConnectionId::new(2),
18364 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
18365 )
18366 .unwrap();
18367 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
18368 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
18369 }
18370
18371 #[test]
18372 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
18373 let handler = ControlHandler::default();
18374 let conn = ConnectionId::new(1);
18375 let malformed = Frame::build(
18376 FrameType::Hello,
18377 control_flags(),
18378 0,
18379 0,
18380 3,
18381 b"{not json".to_vec(),
18382 )
18383 .unwrap();
18384
18385 let error = handler.handle_control(conn, malformed).unwrap();
18386 assert_eq!(error[0].header.ty, FrameType::Error);
18387 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
18388
18389 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18390 let pong = handler.handle_control(conn, ping).unwrap();
18391 assert_eq!(pong[0].header.ty, FrameType::Pong);
18392 assert_eq!(pong[0].header.corr, 4);
18393 }
18394
18395 #[test]
18396 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
18397 let registry = Arc::new(Registry::default());
18398 let handler = ControlHandler::new(Arc::clone(®istry));
18399
18400 handler
18401 .handle_control(
18402 ConnectionId::new(1),
18403 hello_frame("aft", PROTOCOL_VERSION, 1),
18404 )
18405 .unwrap();
18406 let duplicate = handler
18407 .handle_control(
18408 ConnectionId::new(2),
18409 hello_frame("aft", PROTOCOL_VERSION, 2),
18410 )
18411 .unwrap();
18412
18413 assert_eq!(duplicate[0].header.ty, FrameType::Error);
18414 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18415 let registration = registry.get_module("aft").unwrap().unwrap();
18416 assert_eq!(registration.connection_id, ConnectionId::new(1));
18417 }
18418
18419 #[test]
18420 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
18421 let registry = Arc::new(Registry::default());
18422 let forwarding = Arc::new(ForwardingTable::default());
18423 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
18424 let handler =
18425 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18426 .with_process_liveness(process_liveness);
18427 let (ctx, route_channel, route_epoch) =
18428 bind_liveness_route(®istry, &forwarding, "aft-dead");
18429 let responses = handler
18430 .handle_route_poll(
18431 &ctx,
18432 route_poll_frame(41, PollKind::Liveness, route_channel),
18433 route_channel,
18434 route_epoch,
18435 PollKind::Liveness,
18436 )
18437 .unwrap();
18438
18439 assert_eq!(responses.len(), 1);
18440 assert_eq!(responses[0].header.ty, FrameType::Response);
18441 assert_route_poll_liveness(&responses[0], false);
18442 }
18443
18444 #[test]
18445 fn liveness_poll_without_process_source_uses_bound_route() {
18446 let registry = Arc::new(Registry::default());
18447 let forwarding = Arc::new(ForwardingTable::default());
18448 let handler =
18449 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18450 let (ctx, route_channel, route_epoch) =
18451 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
18452 let responses = handler
18453 .handle_route_poll(
18454 &ctx,
18455 route_poll_frame(42, PollKind::Liveness, route_channel),
18456 route_channel,
18457 route_epoch,
18458 PollKind::Liveness,
18459 )
18460 .unwrap();
18461
18462 assert_route_poll_liveness(&responses[0], true);
18463 }
18464
18465 #[test]
18466 fn liveness_poll_untracked_process_source_uses_bound_route() {
18467 let registry = Arc::new(Registry::default());
18468 let forwarding = Arc::new(ForwardingTable::default());
18469 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
18470 let handler =
18471 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18472 .with_process_liveness(process_liveness);
18473 let (ctx, route_channel, route_epoch) =
18474 bind_liveness_route(®istry, &forwarding, "aft-untracked");
18475 let responses = handler
18476 .handle_route_poll(
18477 &ctx,
18478 route_poll_frame(43, PollKind::Liveness, route_channel),
18479 route_channel,
18480 route_epoch,
18481 PollKind::Liveness,
18482 )
18483 .unwrap();
18484
18485 assert_route_poll_liveness(&responses[0], true);
18486 }
18487
18488 #[tokio::test]
18489 async fn unknown_op_returns_unknown_control_op() {
18490 let handler = ControlHandler::default();
18491 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
18492 let request = Frame::build(
18493 FrameType::Request,
18494 control_flags(),
18495 0,
18496 0,
18497 55,
18498 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
18499 )
18500 .unwrap();
18501
18502 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18503
18504 assert_eq!(response.len(), 1);
18505 assert_eq!(response[0].header.ty, FrameType::Error);
18506 assert_eq!(response[0].header.corr, 55);
18507 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
18508 }
18509
18510 #[tokio::test]
18511 async fn supervisor_provenance_rejects_unknown_exact_module() {
18512 let handler = ControlHandler::default();
18513 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
18514 let request = Frame::build(
18515 FrameType::Request,
18516 control_flags(),
18517 0,
18518 0,
18519 57,
18520 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
18521 )
18522 .unwrap();
18523
18524 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18525
18526 assert_eq!(response.len(), 1);
18527 assert_eq!(response[0].header.ty, FrameType::Error);
18528 assert_eq!(response[0].header.corr, 57);
18529 let error = parse_error(&response[0]);
18530 assert_eq!(error["code"], "unknown_module");
18531 assert_eq!(error["message"], "module_id 'missing' is not supervised");
18532 }
18533
18534 #[test]
18535 fn provenance_probe_override_keeps_handler_tests_deterministic() {
18536 let expected = subc_control::RunningImageAgreement::Unavailable {
18537 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18538 };
18539 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
18540 assert_eq!(handler.provenance_probe_override, Some(expected));
18541 }
18542
18543 #[test]
18544 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
18545 let verdict = reload_verdict(
18546 std::path::Path::new("/bin/new"),
18547 Some(std::path::Path::new("/bin/old")),
18548 subc_control::RunningImageAgreement::Unavailable {
18549 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18550 },
18551 );
18552 assert!(matches!(
18553 verdict.path,
18554 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
18555 if configured == std::path::Path::new("/bin/new")
18556 && spawned_from == std::path::Path::new("/bin/old")
18557 ));
18558 }
18559
18560 #[test]
18561 fn launch_spec_without_running_digest_is_unavailable_not_match() {
18562 use subc_control::{
18563 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
18564 };
18565 let configured = crate::supervise::ModuleSpec {
18566 script: None,
18567 carry_containment: false,
18568 module_id: "legacy".into(),
18569 program: "/module".into(),
18570 args: Vec::new(),
18571 env: Vec::new(),
18572 reserved: false,
18573 reserved_prefixes: Vec::new(),
18574 protocol: subc_control::ModuleProtocol::Subc,
18575 overlap: Default::default(),
18576 };
18577 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
18578 assert_eq!(
18579 launch_spec_agreement(&configured, false, Some(&digest)),
18580 Agreement::Unavailable {
18581 reason: Reason::NotRunning
18582 }
18583 );
18584 assert_eq!(
18585 launch_spec_agreement(&configured, true, None),
18586 Agreement::Unavailable {
18587 reason: Reason::SpawnedSpecUnavailable
18588 }
18589 );
18590 assert_eq!(
18591 launch_spec_agreement(&configured, true, Some(&digest)),
18592 Agreement::Match
18593 );
18594 }
18595
18596 #[cfg(windows)]
18597 #[test]
18598 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
18599 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
18600 let configured = dir.join("module.exe");
18601 std::fs::write(&configured, b"executable path fixture").unwrap();
18602 let spawned = std::fs::canonicalize(&configured).unwrap();
18603 assert_ne!(
18604 configured, spawned,
18605 "fixture must exercise different path spellings"
18606 );
18607 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
18608 let lower_case = configured.to_string_lossy().to_lowercase();
18609 for spelling in [
18610 &configured,
18611 Path::new(&forward_slashes),
18612 Path::new(&lower_case),
18613 ] {
18614 let verdict = reload_verdict(
18615 spelling,
18616 Some(&spawned),
18617 subc_control::RunningImageAgreement::Unavailable {
18618 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18619 },
18620 );
18621 assert_eq!(
18622 verdict.path,
18623 ReloadPathAgreement::Match,
18624 "configured {spelling:?}, spawned {spawned:?}"
18625 );
18626 }
18627 }
18628
18629 #[cfg(windows)]
18630 #[test]
18631 fn windows_reload_path_keeps_different_existing_program_pending() {
18632 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
18633 let old = dir.join("old.exe");
18634 let new = dir.join("new.exe");
18635 std::fs::write(&old, b"same executable content").unwrap();
18636 std::fs::copy(&old, &new).unwrap();
18637 let spawned = std::fs::canonicalize(&old).unwrap();
18638 let image = subc_control::RunningImageAgreement::Unavailable {
18639 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18640 };
18641 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
18642 assert!(
18643 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
18644 if configured == new && spawned_from == spawned)
18645 );
18646 let missing = dir.join("missing.exe");
18647 assert!(matches!(
18648 reload_verdict(&missing, Some(&spawned), image).path,
18649 ReloadPathAgreement::Mismatch { .. }
18650 ));
18651 }
18652
18653 #[test]
18654 fn reload_verdict_detects_replaced_image_at_same_path() {
18655 let image = subc_control::RunningImageAgreement::Mismatch {
18656 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
18657 digest: "old".into(),
18658 },
18659 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
18660 digest: "new".into(),
18661 },
18662 };
18663 let verdict = reload_verdict(
18664 std::path::Path::new("/bin/same"),
18665 Some(std::path::Path::new("/bin/same")),
18666 image.clone(),
18667 );
18668 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18669 assert_eq!(verdict.image, image);
18670 }
18671
18672 #[test]
18673 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
18674 let image = subc_control::RunningImageAgreement::Unavailable {
18675 reason: subc_control::RunningImageUnavailableReason::NotRunning,
18676 };
18677 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
18678 assert_eq!(
18679 verdict.path,
18680 subc_control::ReloadPathAgreement::Unavailable {
18681 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
18682 }
18683 );
18684 assert_eq!(verdict.image, image);
18685
18686 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
18687 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18688 };
18689 let verdict = reload_verdict(
18690 std::path::Path::new("/bin/same"),
18691 Some(std::path::Path::new("/bin/same")),
18692 unconfirmed.clone(),
18693 );
18694 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18695 assert_eq!(verdict.image, unconfirmed);
18696 }
18697
18698 #[test]
18699 fn reload_verdict_preserves_each_image_unavailability_reason() {
18700 use subc_control::RunningImageUnavailableReason as Reason;
18701
18702 for reason in [
18703 Reason::NotRunning,
18704 Reason::UnsupportedPlatform,
18705 Reason::RunningExecutableUnreadable,
18706 Reason::SpawnedPathUnreadable,
18707 Reason::HashFailed,
18708 Reason::ProcessIdentityUnconfirmed,
18709 Reason::Unknown("future_probe_reason".to_string()),
18710 ] {
18711 let image = subc_control::RunningImageAgreement::Unavailable {
18712 reason: reason.clone(),
18713 };
18714 let verdict = reload_verdict(
18715 std::path::Path::new("/bin/same"),
18716 Some(std::path::Path::new("/bin/same")),
18717 image.clone(),
18718 );
18719 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18720 assert_eq!(verdict.image, image, "{reason:?}");
18721 }
18722 }
18723
18724 #[tokio::test]
18725 async fn malformed_control_bodies_return_invalid_control_body() {
18726 let handler = ControlHandler::default();
18727 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
18728
18729 for (corr, body) in [
18730 (56, br#"{"route_channel":1}"#.as_slice()),
18731 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
18732 (
18733 58,
18734 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
18735 ),
18736 ] {
18737 let request = Frame::build(
18738 FrameType::Request,
18739 control_flags(),
18740 0,
18741 0,
18742 corr,
18743 body.to_vec(),
18744 )
18745 .unwrap();
18746 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18747
18748 assert_eq!(response.len(), 1);
18749 assert_eq!(response[0].header.ty, FrameType::Error);
18750 assert_eq!(response[0].header.corr, corr);
18751 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
18752 }
18753 }
18754
18755 #[tokio::test]
18756 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
18757 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18758 let registry = Arc::new(Registry::default());
18759 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18760 let router = Router::with_control_handler(Arc::clone(&control));
18761 let connection = router.begin_connection();
18762 let (ctx, mut rx) = route_ctx(connection.id());
18763
18764 router
18765 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
18766 .await
18767 .unwrap();
18768 let response = rx.recv().await.unwrap();
18769 let ack = parse_ack(&response);
18770 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18771 let channel = 1;
18772
18773 let goodbye =
18774 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
18775 router.route_for_connection(&ctx, goodbye).await.unwrap();
18776 assert!(rx.try_recv().is_err());
18777 assert!(registry.get_module("aft").unwrap().is_none());
18778
18779 router
18780 .route_for_connection(&ctx, channel_request(channel, 13))
18781 .await
18782 .unwrap();
18783 let error_frame = rx.recv().await.unwrap();
18784 assert_eq!(error_frame.header.ty, FrameType::Error);
18785 assert_eq!(error_frame.header.channel, channel);
18786 let captured = crate::router::test_log::captured_logs(&logs);
18787 assert_eq!(
18788 captured
18789 .lines()
18790 .filter(|line| {
18791 line.contains("module_id=aft")
18792 && line.contains("reason=explicit_goodbye")
18793 && line.contains("module registration ended")
18794 })
18795 .count(),
18796 1,
18797 "unexpected GOODBYE registry log: {captured}"
18798 );
18799 }
18800
18801 #[tokio::test]
18802 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
18803 let registry = Arc::new(Registry::default());
18804 let handler = ControlHandler::new(registry).with_capability_config(
18805 [("prov".to_string(), true), ("cons".to_string(), true)],
18806 BTreeMap::new(),
18807 );
18808 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
18809 register_capability_manifest(
18810 &handler,
18811 &provider_ctx,
18812 &mut provider_rx,
18813 capability_manifest("prov", &["thing/v1"], &[]),
18814 1,
18815 )
18816 .await;
18817 let mut consumer = capability_manifest("cons", &[], &[]);
18818 consumer.capabilities.as_mut().unwrap().requires.push(
18819 subc_protocol::manifest::CapabilityRequirement {
18820 capability: "thing/v1".to_string(),
18821 need: subc_protocol::manifest::CapabilityNeed::Required,
18822 },
18823 );
18824 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
18825 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
18826 assert_eq!(
18827 handler.capability_evaluator.verdict("cons", "thing/v1"),
18828 Some(CapabilityVerdict::Provided)
18829 );
18830 let (mut client_rx, _) = open_route_for_capability_test(
18831 &handler,
18832 &provider_ctx,
18833 &mut provider_rx,
18834 703,
18835 3,
18836 "prov",
18837 None,
18838 )
18839 .await;
18840 let goodbye =
18841 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18842 handler
18843 .handle_control_frame(&provider_ctx, goodbye)
18844 .await
18845 .unwrap();
18846 assert_eq!(
18847 handler.capability_evaluator.verdict("cons", "thing/v1"),
18848 Some(CapabilityVerdict::NeverProvided)
18849 );
18850 let closed = client_rx
18851 .try_recv()
18852 .expect("GOODBYE pushes route.closed before route GOODBYE");
18853 assert!(
18854 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
18855 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
18856 );
18857 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
18858 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
18859 }
18860
18861 struct RouteClosedFixture {
18862 handler: ControlHandler,
18863 #[cfg(unix)]
18864 supervisor: Supervisor,
18865 module_ctx: RouteCtx,
18866 #[cfg(unix)]
18867 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18868 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18869 route: Arc<crate::forwarding::RouteBinding>,
18870 }
18871
18872 async fn route_closed_fixture() -> RouteClosedFixture {
18873 let registry = Arc::new(Registry::default());
18874 let handler = ControlHandler::new(Arc::clone(®istry));
18875 #[cfg(unix)]
18876 let supervisor = Supervisor::new(registry, RestartPolicy::default())
18877 .with_forwarding(Arc::clone(&handler.forwarding));
18878 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
18879 register_capability_manifest(
18880 &handler,
18881 &module_ctx,
18882 &mut module_rx,
18883 capability_manifest("prov", &[], &[]),
18884 1,
18885 )
18886 .await;
18887 let (client_rx, bind) = open_route_for_capability_test(
18888 &handler,
18889 &module_ctx,
18890 &mut module_rx,
18891 702,
18892 2,
18893 "prov",
18894 None,
18895 )
18896 .await;
18897 let ModuleControlRequest::RouteBind {
18898 route_channel: channel,
18899 epoch: route_epoch,
18900 ..
18901 } = bind
18902 else {
18903 panic!("expected route.bind");
18904 };
18905 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
18906 handler
18907 .forwarding
18908 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
18909 .unwrap()
18910 else {
18911 panic!("the acknowledged route must be bound");
18912 };
18913 RouteClosedFixture {
18914 handler,
18915 #[cfg(unix)]
18916 supervisor,
18917 module_ctx,
18918 #[cfg(unix)]
18919 module_rx,
18920 client_rx,
18921 route,
18922 }
18923 }
18924
18925 impl RouteClosedFixture {
18926 #[cfg(unix)]
18927 async fn cut(&mut self) {
18928 let notices = async {
18929 let mut draining = self.module_rx.recv().await.unwrap();
18930 assert!(matches!(
18931 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
18932 &draining.body
18933 )
18934 .unwrap(),
18935 subc_protocol::session::ModuleControlCommand::Draining {
18936 reason: RouteCloseReason::Disable,
18937 ..
18938 }
18939 ));
18940 draining.flushed.take().unwrap().send(()).unwrap();
18941 let mut closing = self.client_rx.recv().await.unwrap();
18942 assert!(matches!(
18943 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
18944 ClientControlPush::RouteClosing {
18945 reason: RouteCloseReason::Restart,
18946 ..
18947 }
18948 ));
18949 closing.flushed.take().unwrap().send(()).unwrap();
18950 };
18951 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
18952 result.unwrap();
18953 }
18954
18955 fn disconnect(&mut self) -> ClientControlPush {
18956 self.handler
18957 .cleanup_connection(self.module_ctx.connection_id)
18958 .unwrap();
18959 let closed = self.client_rx.try_recv().unwrap();
18960 assert_eq!(closed.header.ty, FrameType::Push);
18961 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
18962 assert!(matches!(
18963 &push,
18964 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
18965 if channels == &[self.route.client_channel]
18966 ));
18967 assert_eq!(
18968 self.client_rx.try_recv().unwrap().header.ty,
18969 FrameType::Goodbye
18970 );
18971 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
18972 push
18973 }
18974 }
18975
18976 #[cfg(unix)]
18977 #[tokio::test(start_paused = true)]
18978 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
18979 let mut fixture = route_closed_fixture().await;
18980 fixture.cut().await;
18981 assert!(matches!(
18982 fixture.disconnect(),
18983 ClientControlPush::RouteClosed {
18984 reason: RouteCloseReason::Restart,
18985 drained: true,
18986 ..
18987 }
18988 ));
18989 }
18990
18991 #[cfg(unix)]
18992 #[tokio::test(start_paused = true)]
18993 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
18994 let mut fixture = route_closed_fixture().await;
18995 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
18996 let started = tokio::time::Instant::now();
18997 fixture.cut().await;
18998 assert!(started.elapsed() >= Duration::from_secs(2));
18999 assert_eq!(fixture.route.flow.in_flight(), 1);
19000 assert!(matches!(
19001 fixture.disconnect(),
19002 ClientControlPush::RouteClosed {
19003 reason: RouteCloseReason::Restart,
19004 drained: false,
19005 ..
19006 }
19007 ));
19008 }
19009
19010 #[tokio::test]
19011 async fn module_connection_crash_reports_crash_and_not_drained() {
19012 let mut fixture = route_closed_fixture().await;
19013 assert!(matches!(
19017 fixture.disconnect(),
19018 ClientControlPush::RouteClosed {
19019 reason: RouteCloseReason::Crash,
19020 drained: false,
19021 ..
19022 }
19023 ));
19024 }
19025
19026 #[tokio::test]
19027 async fn dropping_router_connection_releases_registration() {
19028 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19029 let registry = Arc::new(Registry::default());
19030 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
19031 let router = Router::with_control_handler(Arc::clone(&control));
19032 let connection = router.begin_connection();
19033 let connection_id = connection.id();
19034 let (ctx, mut rx) = route_ctx(connection_id);
19035
19036 router
19037 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
19038 .await
19039 .unwrap();
19040 let response = rx.recv().await.unwrap();
19041 let ack = parse_ack(&response);
19042 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
19043 assert!(registry.get_module("aft").unwrap().is_some());
19044
19045 drop(connection);
19046
19047 assert!(registry.get_module("aft").unwrap().is_none());
19048 assert_eq!(registry.active_registration_count().unwrap(), 0);
19049
19050 control
19051 .cleanup_connection(ConnectionId::new(u64::MAX))
19052 .unwrap();
19053 let captured = crate::router::test_log::captured_logs(&logs);
19054 println!("captured registry lifecycle logs:\n{captured}");
19055 let events: Vec<_> = captured
19056 .lines()
19057 .filter(|line| {
19058 line.contains("module registered module_id=aft ")
19059 || line.contains("module registration ended")
19060 })
19061 .collect();
19062 assert_eq!(
19063 events.len(),
19064 2,
19065 "unexpected registry lifecycle logs: {captured}"
19066 );
19067 assert!(events[0].contains("module registered module_id=aft "));
19068 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
19069 assert!(events[1].contains(&format!(
19070 "module_id=aft connection_id={}",
19071 connection_id.get()
19072 )));
19073 assert!(events[1].contains("reason=connection_closed"));
19074 assert!(events[1].contains("module registration ended"));
19075 assert_eq!(
19076 captured
19077 .lines()
19078 .filter(|line| line.contains("module registered module_id=aft "))
19079 .count(),
19080 1,
19081 "legacy registration admission line must appear once: {captured}"
19082 );
19083 }
19084
19085 #[tokio::test]
19086 async fn hello_registration_keeps_legacy_module_registered_line_once() {
19087 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19088 let registry = Arc::new(Registry::default());
19089 let control = ControlHandler::new(Arc::clone(®istry));
19090 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
19091 hello_via_sink(
19092 &control,
19093 &ctx,
19094 &mut rx,
19095 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
19096 )
19097 .await;
19098
19099 let captured = crate::router::test_log::captured_logs(&logs);
19100 let admissions: Vec<_> = captured
19101 .lines()
19102 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
19103 .collect();
19104 assert_eq!(
19105 admissions.len(),
19106 1,
19107 "legacy admission line must remain exactly once: {captured}"
19108 );
19109 assert!(admissions[0].contains("routable_provider=true"));
19110 assert!(admissions[0].contains("connection_id=777"));
19111 }
19112
19113 fn capability_manifest(
19114 module_id: &str,
19115 provides: &[&str],
19116 must_never_reach: &[&str],
19117 ) -> ModuleManifest {
19118 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
19119 manifest.capabilities = Some(CapabilityDeclarations {
19120 provides: provides
19121 .iter()
19122 .map(|capability| (*capability).to_string())
19123 .collect(),
19124 requires: Vec::new(),
19125 must_never_reach: must_never_reach
19126 .iter()
19127 .map(|capability| (*capability).to_string())
19128 .collect(),
19129 });
19130 manifest
19131 }
19132
19133 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
19134 Frame::build(
19135 FrameType::Hello,
19136 control_flags(),
19137 0,
19138 0,
19139 corr,
19140 serde_json::to_vec(&ModuleHelloBody {
19141 protocol_ver: manifest.protocol_ver,
19142 manifest,
19143 control_ops: None,
19144 launch_nonce: None,
19145 })
19146 .expect("capability test HELLO serializes"),
19147 )
19148 .expect("capability test HELLO frame builds")
19149 }
19150
19151 fn catalog_update_with_capabilities_frame(
19152 corr: u64,
19153 capabilities: CapabilityDeclarations,
19154 ) -> Frame {
19155 Frame::build(
19156 FrameType::Request,
19157 control_flags(),
19158 0,
19159 0,
19160 corr,
19161 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
19162 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
19163 capabilities: Some(capabilities),
19164 ready: None,
19165 })
19166 .expect("capability catalog.update serializes"),
19167 )
19168 .expect("capability catalog.update frame builds")
19169 }
19170
19171 async fn register_capability_manifest(
19172 handler: &ControlHandler,
19173 ctx: &RouteCtx,
19174 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19175 manifest: ModuleManifest,
19176 corr: u64,
19177 ) {
19178 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
19179 }
19180
19181 async fn open_route_for_capability_test(
19182 handler: &ControlHandler,
19183 target_ctx: &RouteCtx,
19184 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19185 client_connection_id: u64,
19186 corr: u64,
19187 target_module_id: &str,
19188 consumer_identity: Option<ConsumerIdentity>,
19189 ) -> (
19190 mpsc::Receiver<crate::router::OutboundFrame>,
19191 ModuleControlRequest,
19192 ) {
19193 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
19194 let route_handler = handler.clone();
19195 let target_module_id = target_module_id.to_string();
19196 let route_task = tokio::spawn(async move {
19197 route_handler
19198 .handle_control_frame(
19199 &client_ctx,
19200 route_open_frame_with_admission_facts(
19201 corr,
19202 &target_module_id,
19203 unique_project_root("admission-facts"),
19204 consumer_identity,
19205 None,
19206 ),
19207 )
19208 .await
19209 .expect("capability test route.open succeeds")
19210 });
19211 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
19212 .await
19213 .expect("capability test route.open must reach route.bind")
19214 .expect("target control receiver stays open");
19215 let bind_request: ModuleControlRequest =
19216 serde_json::from_slice(&bind.body).expect("route.bind decodes");
19217 handler
19218 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
19219 .await
19220 .expect("capability test route.bind ACK succeeds");
19221 assert!(route_task.await.expect("route.open task joins").is_empty());
19222 let opened = client_rx
19223 .recv()
19224 .await
19225 .expect("successful route.open publishes a response");
19226 assert!(matches!(
19227 serde_json::from_slice::<ClientControlResponse>(&opened.body),
19228 Ok(ClientControlResponse::RouteOpen { .. })
19229 ));
19230 (client_rx, bind_request)
19231 }
19232
19233 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
19234 assert_eq!(frame.header.ty, FrameType::Push);
19235 assert_eq!(frame.header.channel, 0);
19236 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
19237 .expect("route.closed control push decodes");
19238 let ClientControlPush::RouteClosed { channels, .. } = &push else {
19239 panic!("expected route.closed");
19240 };
19241 assert_eq!(channels.len(), 1, "exactly one violating route closed");
19242 let closed_channel = channels[0];
19243 let channels = channels.clone();
19244 assert_eq!(
19245 push,
19246 ClientControlPush::RouteClosed {
19247 module_id: target_module_id.to_string(),
19248 channels,
19249 reason: RouteCloseReason::CapabilityDenied,
19250 drained: false,
19251 abandoned: 0,
19252 excluded_subscriptions: 0,
19253 terminal: Some(false),
19254 }
19255 );
19256 closed_channel
19257 }
19258
19259 #[tokio::test]
19260 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
19261 let registry = Arc::new(Registry::default());
19262 let forwarding = Arc::new(ForwardingTable::default());
19263 let supervisor = SupervisorHandle::new();
19264 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19265 let handler =
19266 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19267 .with_supervisor(supervisor);
19268 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
19269 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
19270 register_capability_manifest(
19271 &handler,
19272 &target_ctx,
19273 &mut target_rx,
19274 capability_manifest("target", &["credentials-provider/v1"], &[]),
19275 1,
19276 )
19277 .await;
19278 register_capability_manifest(
19279 &handler,
19280 &opener_ctx,
19281 &mut opener_rx,
19282 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19283 2,
19284 )
19285 .await;
19286
19287 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
19288 let replies = handler
19289 .handle_control_frame(
19290 &client_ctx,
19291 route_open_frame_with_admission_facts(
19292 3,
19293 "target",
19294 unique_project_root("admission-facts"),
19295 Some(ConsumerIdentity {
19296 module_id: "opener".to_string(),
19297 launch_nonce: "opener-nonce".to_string(),
19298 }),
19299 None,
19300 ),
19301 )
19302 .await
19303 .expect("denied route.open returns a typed frame");
19304 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19305 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19306 assert!(
19307 target_rx.try_recv().is_err(),
19308 "forbidden route.open must not relay route.bind"
19309 );
19310 }
19311
19312 #[tokio::test]
19313 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
19314 let registry = Arc::new(Registry::default());
19315 let forwarding = Arc::new(ForwardingTable::default());
19316 let supervisor = SupervisorHandle::new();
19317 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19318 let handler =
19319 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19320 .with_supervisor(supervisor);
19321 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
19322 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
19323 register_capability_manifest(
19324 &handler,
19325 &target_ctx,
19326 &mut target_rx,
19327 capability_manifest("target", &["credentials-provider/v1"], &[]),
19328 1,
19329 )
19330 .await;
19331 register_capability_manifest(
19332 &handler,
19333 &old_opener_ctx,
19334 &mut old_opener_rx,
19335 capability_manifest("opener", &[], &[]),
19336 2,
19337 )
19338 .await;
19339 let (mut client_rx, _) = open_route_for_capability_test(
19340 &handler,
19341 &target_ctx,
19342 &mut target_rx,
19343 712,
19344 3,
19345 "target",
19346 Some(ConsumerIdentity {
19347 module_id: "opener".to_string(),
19348 launch_nonce: "opener-nonce".to_string(),
19349 }),
19350 )
19351 .await;
19352 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19353
19354 handler
19355 .cleanup_connection(old_opener_ctx.connection_id)
19356 .expect("old opener registration cleans up");
19357 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
19358 register_capability_manifest(
19359 &handler,
19360 &new_opener_ctx,
19361 &mut new_opener_rx,
19362 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19363 4,
19364 )
19365 .await;
19366
19367 let closed_channel = assert_capability_denied_push(
19368 client_rx
19369 .try_recv()
19370 .expect("HELLO deny addition must emit route.closed")
19371 .frame,
19372 "target",
19373 );
19374 let goodbye = client_rx
19375 .try_recv()
19376 .expect("HELLO deny addition must send the client a route GOODBYE")
19377 .frame;
19378 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19379 assert_eq!(goodbye.header.channel, closed_channel);
19380 assert_ne!(goodbye.header.channel, 0);
19381 assert_ne!(goodbye.header.epoch, 0);
19382 assert!(client_rx.try_recv().is_err());
19383 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19384 assert!(matches!(
19385 target_rx.try_recv(),
19386 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19387 ));
19388 }
19389
19390 #[tokio::test]
19391 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
19392 let registry = Arc::new(Registry::default());
19393 let forwarding = Arc::new(ForwardingTable::default());
19394 let supervisor = SupervisorHandle::new();
19395 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19396 let handler =
19397 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19398 .with_supervisor(supervisor);
19399 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
19400 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
19401 register_capability_manifest(
19402 &handler,
19403 &target_ctx,
19404 &mut target_rx,
19405 capability_manifest("target", &[], &[]),
19406 1,
19407 )
19408 .await;
19409 register_capability_manifest(
19410 &handler,
19411 &opener_ctx,
19412 &mut opener_rx,
19413 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19414 2,
19415 )
19416 .await;
19417 let (mut client_rx, _) = open_route_for_capability_test(
19418 &handler,
19419 &target_ctx,
19420 &mut target_rx,
19421 722,
19422 3,
19423 "target",
19424 Some(ConsumerIdentity {
19425 module_id: "opener".to_string(),
19426 launch_nonce: "opener-nonce".to_string(),
19427 }),
19428 )
19429 .await;
19430 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19431
19432 let replies = handler
19433 .handle_control_frame(
19434 &target_ctx,
19435 catalog_update_with_capabilities_frame(
19436 4,
19437 CapabilityDeclarations {
19438 provides: vec!["credentials-provider/v1".to_string()],
19439 requires: Vec::new(),
19440 must_never_reach: Vec::new(),
19441 },
19442 ),
19443 )
19444 .await
19445 .expect("claim catalog.update succeeds");
19446 assert!(matches!(
19447 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
19448 Ok(ModuleControlResponseToModule::CatalogUpdate {})
19449 ));
19450 let closed_channel = assert_capability_denied_push(
19451 client_rx
19452 .try_recv()
19453 .expect("claim addition must emit route.closed")
19454 .frame,
19455 "target",
19456 );
19457 let goodbye = client_rx
19458 .try_recv()
19459 .expect("claim addition must send the client a route GOODBYE")
19460 .frame;
19461 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19462 assert_eq!(goodbye.header.channel, closed_channel);
19463 assert_ne!(goodbye.header.channel, 0);
19464 assert_ne!(goodbye.header.epoch, 0);
19465 assert!(client_rx.try_recv().is_err());
19466 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19467 assert!(matches!(
19468 target_rx.try_recv(),
19469 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19470 ));
19471 }
19472
19473 #[tokio::test]
19474 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
19475 let registry = Arc::new(Registry::default());
19476 let forwarding = Arc::new(ForwardingTable::default());
19477 let supervisor = SupervisorHandle::new();
19478 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19479 let handler =
19480 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19481 .with_supervisor(supervisor);
19482 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
19483 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
19484 register_capability_manifest(
19485 &handler,
19486 &target_ctx,
19487 &mut target_rx,
19488 capability_manifest("target", &["credentials-provider/v1"], &[]),
19489 1,
19490 )
19491 .await;
19492 register_capability_manifest(
19493 &handler,
19494 &opener_ctx,
19495 &mut opener_rx,
19496 capability_manifest("opener", &[], &[]),
19497 2,
19498 )
19499 .await;
19500 let (mut client_rx, _) = open_route_for_capability_test(
19501 &handler,
19502 &target_ctx,
19503 &mut target_rx,
19504 732,
19505 3,
19506 "target",
19507 Some(ConsumerIdentity {
19508 module_id: "opener".to_string(),
19509 launch_nonce: "opener-nonce".to_string(),
19510 }),
19511 )
19512 .await;
19513 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19514
19515 handler
19516 .handle_control_frame(
19517 &target_ctx,
19518 catalog_update_with_capabilities_frame(
19519 4,
19520 CapabilityDeclarations {
19521 provides: Vec::new(),
19522 requires: Vec::new(),
19523 must_never_reach: Vec::new(),
19524 },
19525 ),
19526 )
19527 .await
19528 .expect("claim removal catalog.update succeeds");
19529 assert_eq!(
19530 forwarding.active_binding_count().unwrap(),
19531 1,
19532 "removing an attested target claim must leave the route census unchanged"
19533 );
19534 assert!(
19535 client_rx.try_recv().is_err(),
19536 "claim removal must not emit route.closed or a client route GOODBYE"
19537 );
19538 assert!(
19539 target_rx.try_recv().is_err(),
19540 "claim removal must not send the target a route GOODBYE"
19541 );
19542 }
19543
19544 #[tokio::test]
19547 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
19548 let registry = Arc::new(Registry::default());
19549 let forwarding = Arc::new(ForwardingTable::default());
19550 let supervisor = SupervisorHandle::new();
19551 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19552 let handler =
19553 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19554 .with_supervisor(supervisor);
19555 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
19556 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
19557 register_capability_manifest(
19558 &handler,
19559 &target_ctx,
19560 &mut target_rx,
19561 capability_manifest("target", &["credentials-provider/v1"], &[]),
19562 1,
19563 )
19564 .await;
19565 register_capability_manifest(
19566 &handler,
19567 &opener_ctx,
19568 &mut opener_rx,
19569 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19570 2,
19571 )
19572 .await;
19573
19574 let (_client_rx, bind) = open_route_for_capability_test(
19575 &handler,
19576 &target_ctx,
19577 &mut target_rx,
19578 742,
19579 3,
19580 "target",
19581 None,
19582 )
19583 .await;
19584 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
19585 panic!("direct scope-honesty route must bind");
19586 };
19587 assert_eq!(principal, Some(Principal::Direct));
19588 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19589 }
19590
19591 #[tokio::test]
19594 async fn must_never_reach_self_route_is_capability_forbidden() {
19595 let registry = Arc::new(Registry::default());
19596 let forwarding = Arc::new(ForwardingTable::default());
19597 let supervisor = SupervisorHandle::new();
19598 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
19599 let handler =
19600 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19601 .with_supervisor(supervisor);
19602 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
19603 register_capability_manifest(
19604 &handler,
19605 &self_ctx,
19606 &mut self_rx,
19607 capability_manifest(
19608 "self-provider",
19609 &["credentials-provider/v1"],
19610 &["credentials-provider/v1"],
19611 ),
19612 1,
19613 )
19614 .await;
19615
19616 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
19617 let replies = handler
19618 .handle_control_frame(
19619 &client_ctx,
19620 route_open_frame_with_admission_facts(
19621 2,
19622 "self-provider",
19623 unique_project_root("admission-facts"),
19624 Some(ConsumerIdentity {
19625 module_id: "self-provider".to_string(),
19626 launch_nonce: "self-nonce".to_string(),
19627 }),
19628 None,
19629 ),
19630 )
19631 .await
19632 .expect("self-route refusal returns a typed frame");
19633 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19634 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19635 assert!(
19636 self_rx.try_recv().is_err(),
19637 "self denial must not relay route.bind"
19638 );
19639 }
19640
19641 #[test]
19642 fn unsupported_channel_zero_frame_returns_error() {
19643 let handler = ControlHandler::default();
19644 let request = Frame::build(
19645 FrameType::Request,
19646 control_flags(),
19647 0,
19648 0,
19649 21,
19650 b"opaque".to_vec(),
19651 )
19652 .unwrap();
19653
19654 let response = handler
19655 .handle_control(ConnectionId::new(1), request)
19656 .unwrap();
19657
19658 assert_eq!(response[0].header.ty, FrameType::Error);
19659 assert_eq!(
19660 parse_error(&response[0])["code"],
19661 "unsupported_control_frame"
19662 );
19663 }
19664
19665 mod swap {
19670 use super::*;
19671
19672 const INCUMBENT: ConnectionId = ConnectionId::new(30);
19673 const CANDIDATE: ConnectionId = ConnectionId::new(40);
19674
19675 struct Swap {
19676 registry: Arc<Registry>,
19677 forwarding: Arc<ForwardingTable>,
19678 handler: ControlHandler,
19679 incumbent_ctx: RouteCtx,
19680 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19681 candidate_ctx: RouteCtx,
19682 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19683 }
19684
19685 async fn swap_with_incumbent() -> Swap {
19686 let registry = Arc::new(Registry::default());
19687 let forwarding = Arc::new(ForwardingTable::default());
19688 let handler =
19689 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
19690 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
19691 hello_via_sink(
19692 &handler,
19693 &incumbent_ctx,
19694 &mut incumbent_rx,
19695 hello_frame("aft", PROTOCOL_VERSION, 7),
19696 )
19697 .await;
19698 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
19699 Swap {
19700 registry,
19701 forwarding,
19702 handler,
19703 incumbent_ctx,
19704 incumbent_rx,
19705 candidate_ctx,
19706 candidate_rx,
19707 }
19708 }
19709
19710 fn register_candidate(swap: &Swap, ready: Option<bool>) {
19711 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
19712 candidate_manifest.ready = ready;
19713 let registration = swap
19714 .registry
19715 .register_candidate_with_control_ops(
19716 candidate_manifest,
19717 PROTOCOL_VERSION,
19718 CANDIDATE,
19719 module_baseline_control_ops(),
19720 )
19721 .unwrap();
19722 swap.forwarding
19723 .register_candidate_module_connection(
19724 CANDIDATE,
19725 "aft".to_string(),
19726 PROTOCOL_VERSION,
19727 manifest_concurrency(®istration.manifest),
19728 swap.candidate_ctx.egress.clone(),
19729 )
19730 .unwrap();
19731 }
19732
19733 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
19734 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
19735 swap.registry.promote_candidate("aft").unwrap().unwrap();
19736 cutover.incumbent.unwrap()
19737 }
19738
19739 #[tokio::test]
19748 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
19749 let mut swap = swap_with_incumbent().await;
19750 let hold = Arc::new(std::sync::Barrier::new(2));
19751 let handler = swap
19752 .handler
19753 .clone()
19754 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
19755 handler
19756 .supervisor
19757 .open_swap("aft", "candidate-nonce".to_string());
19758 let hello = {
19759 let handler = handler.clone();
19760 let candidate_ctx = swap.candidate_ctx.clone();
19761 std::thread::spawn(move || {
19762 handler
19763 .handle_hello(
19764 candidate_ctx.connection_id,
19765 Some(candidate_ctx.egress.clone()),
19766 hello_frame_with_nonce(
19767 "aft",
19768 PROTOCOL_VERSION,
19769 8,
19770 Some("candidate-nonce"),
19771 ),
19772 )
19773 .unwrap()
19774 })
19775 };
19776
19777 hold.wait();
19778 assert!(swap
19781 .registry
19782 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
19783 .unwrap()
19784 .is_some());
19785 assert!(swap
19786 .registry
19787 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19788 .unwrap()
19789 .is_none());
19790 assert!(swap
19791 .forwarding
19792 .module_endpoint_for_connection(CANDIDATE)
19793 .unwrap()
19794 .is_none());
19795
19796 hold.wait();
19797 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
19798 assert_eq!(
19799 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19800 FrameType::HelloAck
19801 );
19802 let published = swap
19803 .registry
19804 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19805 .unwrap()
19806 .expect("candidate is published once its HELLO completes");
19807 assert_eq!(published.connection_id, CANDIDATE);
19808 assert!(swap
19809 .forwarding
19810 .module_endpoint_for_connection(CANDIDATE)
19811 .unwrap()
19812 .is_some());
19813 }
19814
19815 #[tokio::test]
19816 async fn candidate_lookup_after_hello_ack_waits_for_publication() {
19817 let mut swap = swap_with_incumbent().await;
19818 let hold = Arc::new(std::sync::Barrier::new(2));
19819 swap.handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
19820 swap.handler
19821 .supervisor
19822 .open_swap("aft", "candidate-nonce".to_string());
19823 let hello = {
19824 let handler = swap.handler.clone();
19825 let candidate_ctx = swap.candidate_ctx.clone();
19826 std::thread::spawn(move || {
19827 handler
19828 .handle_hello(
19829 CANDIDATE,
19830 Some(candidate_ctx.egress),
19831 hello_frame_with_nonce(
19832 "aft",
19833 PROTOCOL_VERSION,
19834 8,
19835 Some("candidate-nonce"),
19836 ),
19837 )
19838 .unwrap()
19839 })
19840 };
19841 hold.wait();
19842 assert_eq!(
19843 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19844 FrameType::HelloAck
19845 );
19846 let reader_registry = Arc::clone(&swap.registry);
19847 let published = lookup_during_hello_publication(&swap.registry, &hold, move || {
19848 reader_registry
19849 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19850 .unwrap()
19851 });
19852 assert!(hello.join().unwrap().is_empty());
19853 assert_eq!(
19854 published
19855 .expect("ACK makes candidate publication visible")
19856 .connection_id,
19857 CANDIDATE
19858 );
19859 assert_eq!(
19860 swap.registry.generation().unwrap(),
19861 1,
19862 "candidate is not a second catalog entry"
19863 );
19864 }
19865
19866 #[tokio::test]
19867 async fn audit4_ungated_removed_candidate_publication_rolls_back_admission() {
19868 let mut swap = swap_with_incumbent().await;
19869 let hold = Arc::new(std::sync::Barrier::new(2));
19870 let handler = swap
19871 .handler
19872 .clone()
19873 .with_hello_registry_to_forwarding_hold(hold.clone());
19874 handler
19875 .supervisor
19876 .open_swap("aft", "candidate-nonce".into());
19877 let hello = {
19878 let handler = handler.clone();
19879 let ctx = swap.candidate_ctx.clone();
19880 std::thread::spawn(move || {
19881 handler
19882 .handle_hello(
19883 CANDIDATE,
19884 Some(ctx.egress),
19885 hello_frame_with_nonce(
19886 "aft",
19887 PROTOCOL_VERSION,
19888 8,
19889 Some("candidate-nonce"),
19890 ),
19891 )
19892 .unwrap()
19893 })
19894 };
19895 hold.wait();
19896 assert_eq!(
19897 swap.registry
19898 .deregister_connection(CANDIDATE)
19899 .unwrap()
19900 .len(),
19901 1
19902 );
19903 hold.wait();
19904 assert!(hello.join().unwrap().is_empty());
19905 assert_eq!(
19906 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19907 FrameType::HelloAck
19908 );
19909 assert!(swap.registry.get_candidate("aft").unwrap().is_none());
19910 assert_eq!(
19911 handler
19912 .supervisor
19913 .swap_hello_admission("aft", Some("candidate-nonce")),
19914 SwapHelloAdmission::Candidate,
19915 "failed publication must roll back candidate_admitted for an ungated nonce swap"
19916 );
19917 handler.supervisor.close_swap("aft");
19918 }
19919
19920 fn keyed_total(counters: &Value, key: &str) -> u64 {
19921 counters[key]
19922 .as_object()
19923 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
19924 .unwrap_or(0)
19925 }
19926
19927 #[tokio::test]
19928 async fn replacement_logs_old_end_and_new_admission_once() {
19929 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19930 let mut swap = swap_with_incumbent().await;
19931 swap.handler
19932 .supervisor
19933 .open_swap("aft", "candidate-nonce".to_string());
19934 hello_via_sink(
19935 &swap.handler,
19936 &swap.candidate_ctx,
19937 &mut swap.candidate_rx,
19938 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
19939 )
19940 .await;
19941 cutover(&swap);
19942 swap.handler.cleanup_connection(INCUMBENT).unwrap();
19943
19944 let captured = crate::router::test_log::captured_logs(&logs);
19945 println!("captured replacement registry logs:\n{captured}");
19946 let new_admissions: Vec<_> = captured
19947 .lines()
19948 .filter(|line| {
19949 line.contains("connection_id=40")
19950 && line.contains("swap candidate registered; not routable until cutover")
19951 })
19952 .collect();
19953 assert_eq!(
19954 new_admissions.len(),
19955 1,
19956 "unexpected admission logs: {captured}"
19957 );
19958 assert!(new_admissions[0].contains("module_id=aft"));
19959 assert!(new_admissions[0].contains("connection_id=40"));
19960 assert!(
19961 new_admissions[0].contains("swap candidate registered; not routable until cutover")
19962 );
19963 assert!(new_admissions[0].contains("ready=true"));
19964
19965 let old_admissions: Vec<_> = captured
19966 .lines()
19967 .filter(|line| {
19968 line.contains("module registered module_id=aft ")
19969 && line.contains("connection_id=30")
19970 })
19971 .collect();
19972 assert_eq!(
19973 old_admissions.len(),
19974 1,
19975 "unexpected incumbent admission: {captured}"
19976 );
19977
19978 let old_ends: Vec<_> = captured
19979 .lines()
19980 .filter(|line| {
19981 line.contains("connection_id=30") && line.contains("module registration ended")
19982 })
19983 .collect();
19984 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
19985 assert!(old_ends[0].contains("module_id=aft"));
19986 assert!(old_ends[0].contains("connection_id=30"));
19987 assert!(old_ends[0].contains("reason=replaced"));
19988 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
19989 assert_eq!(
19990 captured
19991 .lines()
19992 .filter(|line| line.contains("module registration promoted"))
19993 .count(),
19994 1,
19995 "unexpected promotion logs: {captured}"
19996 );
19997 }
19998
19999 #[tokio::test]
20005 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
20006 let mut swap = swap_with_incumbent().await;
20007 let handler = swap.handler.clone();
20008
20009 let cotenant = ConnectionId::new(31);
20011 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
20012 let (cotenant_task, cotenant_bind) = relay_route_open(
20013 &handler,
20014 cotenant,
20015 &cotenant_ctx.egress,
20016 &mut swap.incumbent_rx,
20017 100,
20018 "aft",
20019 "swap-cotenant",
20020 )
20021 .await;
20022 handler
20023 .handle_control_frame(
20024 &swap.incumbent_ctx,
20025 route_bind_ack(cotenant_bind.header.corr),
20026 )
20027 .await
20028 .unwrap();
20029 assert!(cotenant_task.await.unwrap().is_empty());
20030 let (cotenant_channel, cotenant_epoch) =
20031 published_route(&cotenant_rx.recv().await.unwrap());
20032
20033 let caller = ConnectionId::new(32);
20035 let (caller_ctx, mut caller_rx) = route_ctx(caller);
20036 let (caller_task, caller_bind) = relay_route_open(
20037 &handler,
20038 caller,
20039 &caller_ctx.egress,
20040 &mut swap.incumbent_rx,
20041 101,
20042 "aft",
20043 "swap-caller",
20044 )
20045 .await;
20046 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
20047
20048 register_candidate(&swap, None);
20049 cutover(&swap);
20050
20051 let ack = handler
20053 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
20054 .await;
20055 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
20056 if module_loop_error.is_some() {
20057 handler.cleanup_connection(INCUMBENT).unwrap();
20060 }
20061
20062 assert!(
20064 cotenant_rx.try_recv().is_err(),
20065 "the co-tenant route on the incumbent was torn down by one late ack: \
20066 {module_loop_error:?}"
20067 );
20068 assert!(matches!(
20069 swap.forwarding
20070 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
20071 .unwrap(),
20072 DataRoute::Client(DataRouteState::Bound(_))
20073 ));
20074 assert_eq!(module_loop_error, None);
20075 assert!(swap
20076 .registry
20077 .get_module_by_connection(INCUMBENT)
20078 .unwrap()
20079 .is_some());
20080
20081 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
20083 .await
20084 .expect("the incumbent is told to drop the abandoned binding")
20085 .unwrap()
20086 .frame;
20087 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
20088 assert_eq!(goodbye.header.channel, abandoned_channel);
20089 assert_eq!(goodbye.header.epoch, abandoned_epoch);
20090 assert!(swap.incumbent_rx.try_recv().is_err());
20091
20092 let response = caller_task.await.unwrap();
20094 assert_eq!(response.len(), 1);
20095 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
20096 assert!(caller_rx.try_recv().is_err());
20097
20098 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
20102 let counters = handler.counters().snapshot();
20103 assert_eq!(
20104 keyed_total(&counters, "route_open_accepted_by_principal"),
20105 1
20106 );
20107 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
20108 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
20109 }
20110
20111 #[tokio::test]
20115 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
20116 let mut swap = swap_with_incumbent().await;
20117 register_candidate(&swap, None);
20118 let incumbent = cutover(&swap);
20119 swap.forwarding
20120 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
20121 .unwrap()
20122 .expect("the incumbent is still registered");
20123
20124 let client = ConnectionId::new(33);
20125 let (client_ctx, mut client_rx) = route_ctx(client);
20126 let route_handler = swap.handler.clone();
20127 let open_ctx = RouteCtx {
20128 connection_id: client,
20129 egress: client_ctx.egress.clone(),
20130 };
20131 let mut route_task = tokio::spawn(async move {
20132 route_handler
20133 .handle_control_frame(
20134 &open_ctx,
20135 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
20136 )
20137 .await
20138 .unwrap()
20139 });
20140 let bind = tokio::select! {
20141 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
20142 response = &mut route_task => {
20143 let response = response.unwrap();
20144 panic!(
20145 "post-cutover route.open was refused instead of relayed to the candidate: {}",
20146 parse_error(&response[0])["code"]
20147 );
20148 }
20149 };
20150 swap.handler
20151 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
20152 .await
20153 .unwrap();
20154 assert!(route_task.await.unwrap().is_empty());
20155 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
20156 match swap
20157 .forwarding
20158 .lookup_data_route(client, channel, epoch)
20159 .unwrap()
20160 {
20161 DataRoute::Client(DataRouteState::Bound(route)) => {
20162 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
20163 }
20164 other => panic!("expected a bound route on the candidate, got {other:?}"),
20165 }
20166 assert!(swap.incumbent_rx.try_recv().is_err());
20167 }
20168
20169 #[tokio::test]
20174 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
20175 let swap = swap_with_incumbent().await;
20176 register_candidate(&swap, Some(false));
20177 let update = Frame::build(
20178 FrameType::Request,
20179 control_flags(),
20180 0,
20181 0,
20182 55,
20183 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
20184 provides: manifest("aft", PROTOCOL_VERSION).provides,
20185 capabilities: None,
20186 ready: Some(true),
20187 })
20188 .unwrap(),
20189 )
20190 .unwrap();
20191
20192 let replies = swap
20193 .handler
20194 .handle_control_frame(&swap.candidate_ctx, update)
20195 .await
20196 .unwrap();
20197
20198 assert_eq!(replies.len(), 1);
20199 assert_eq!(
20200 replies[0].header.ty,
20201 FrameType::Response,
20202 "candidate catalog.update was refused: {:?}",
20203 serde_json::from_slice::<Value>(&replies[0].body).ok()
20204 );
20205 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
20206 assert_eq!(
20207 swap.registry
20208 .get_module("aft")
20209 .unwrap()
20210 .unwrap()
20211 .connection_id,
20212 INCUMBENT
20213 );
20214 }
20215 }
20216
20217 mod swap_admission {
20221 use super::*;
20222
20223 const INCUMBENT_NONCE: &str = "incumbent-nonce";
20224 const CANDIDATE_NONCE: &str = "candidate-nonce";
20225
20226 fn handler_with_incumbent(
20227 module_id: &str,
20228 reserved: bool,
20229 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
20230 let registry = Arc::new(Registry::default());
20231 let supervisor = SupervisorHandle::new();
20232 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
20233 if reserved {
20234 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
20235 }
20236 let handler =
20237 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
20238 let incumbent = handler
20239 .handle_control(
20240 ConnectionId::new(1),
20241 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
20242 )
20243 .unwrap();
20244 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
20245 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
20246 (registry, supervisor, handler)
20247 }
20248
20249 #[test]
20255 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
20256 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
20257
20258 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
20259 let replies = handler
20260 .handle_control(
20261 ConnectionId::new(connection),
20262 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
20263 )
20264 .unwrap();
20265 assert_eq!(replies[0].header.ty, FrameType::Error);
20266 assert_eq!(
20267 parse_error(&replies[0])["code"],
20268 "swap_token_invalid",
20269 "nonce {nonce:?}"
20270 );
20271 }
20272 assert!(registry.get_candidate("aft").unwrap().is_none());
20273 assert_eq!(
20274 registry.get_module("aft").unwrap().unwrap().connection_id,
20275 ConnectionId::new(1)
20276 );
20277
20278 let admitted = handler
20281 .handle_control(
20282 ConnectionId::new(4),
20283 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
20284 )
20285 .unwrap();
20286 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
20287 assert_eq!(
20288 registry
20289 .get_candidate("aft")
20290 .unwrap()
20291 .unwrap()
20292 .connection_id,
20293 ConnectionId::new(4)
20294 );
20295 assert_eq!(
20296 registry.get_module("aft").unwrap().unwrap().connection_id,
20297 ConnectionId::new(1),
20298 "the candidate must not take the active slot"
20299 );
20300 let replayed = handler
20301 .handle_control(
20302 ConnectionId::new(5),
20303 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
20304 )
20305 .unwrap();
20306 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
20307
20308 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
20312 let squatter = handler
20313 .handle_control(
20314 ConnectionId::new(6),
20315 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
20316 )
20317 .unwrap();
20318 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
20319 assert!(
20320 registry.get_module("aft").unwrap().is_none(),
20321 "a squatter took the active slot of an id being swapped"
20322 );
20323 }
20324
20325 #[test]
20330 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
20331 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
20332
20333 let replies = handler
20334 .handle_control(
20335 ConnectionId::new(2),
20336 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20337 )
20338 .unwrap();
20339
20340 assert_eq!(
20341 replies[0].header.ty,
20342 FrameType::HelloAck,
20343 "reserved candidate refused: {:?}",
20344 serde_json::from_slice::<Value>(&replies[0].body).ok()
20345 );
20346 assert_eq!(
20347 registry
20348 .get_candidate("vault")
20349 .unwrap()
20350 .unwrap()
20351 .connection_id,
20352 ConnectionId::new(2)
20353 );
20354 }
20355
20356 #[test]
20359 fn without_an_open_swap_the_ordinary_gates_decide() {
20360 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
20361 supervisor.close_swap("vault");
20362
20363 let candidate = handler
20364 .handle_control(
20365 ConnectionId::new(2),
20366 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20367 )
20368 .unwrap();
20369 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
20370 let duplicate = handler
20371 .handle_control(
20372 ConnectionId::new(3),
20373 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
20374 )
20375 .unwrap();
20376 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
20377 assert!(registry.get_candidate("vault").unwrap().is_none());
20378 }
20379 }
20380
20381 mod scopes {
20385 mod close_reasons {
20386 include!("route_close_reason_tests.rs");
20387 }
20388 mod idle_tests {
20389 use super::*;
20390 include!("scope_idle_tests.rs");
20391 }
20392 mod route_open_timings {
20393 use super::*;
20394 include!("route_open_timing_tests.rs");
20395 }
20396 mod route_open_refusal_log {
20397 use super::*;
20398 include!("route_open_refusal_log_tests.rs");
20399 }
20400 mod activity {
20401 include!("activity_scope_tests.rs");
20402 }
20403 mod delta {
20404 include!("scope_handler_delta_tests.rs");
20405 }
20406 mod watch {
20407 include!("scope_watch_tests.rs");
20408 }
20409 mod call_modes {
20410 include!("scope_call_mode_tests.rs");
20411 }
20412 mod federated {
20413 include!("scope_federated_tests.rs");
20414 }
20415 mod authorize {
20416 include!("route_authorize_tests.rs");
20417 }
20418 #[tokio::test]
20419 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
20420 let rig = rig().await;
20421 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
20422 "s".into(),
20423 1,
20424 "tool".into(),
20425 ))
20426 .unwrap();
20427 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
20428 let (direct, _rx) = wide_ctx(999);
20429 for (ctx, expected) in [
20430 (&rig.owner, FrameType::Response),
20431 (&direct, FrameType::Error),
20432 ] {
20433 let frame =
20434 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
20435 .unwrap();
20436 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
20437 assert_eq!(replies[0].header.ty, expected);
20438 if expected == FrameType::Response {
20439 let reply: subc_control::ScopeServedByReply =
20440 serde_json::from_slice(&replies[0].body).unwrap();
20441 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
20442 } else {
20443 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
20444 }
20445 }
20446 let reply = call(
20447 &rig.handler,
20448 &rig.owner,
20449 &ModuleControlRequestFromModule::ScopeDescribe {
20450 owner: Principal::Reserved {
20451 module_id: OWNER.into(),
20452 },
20453 scope_ref: "s".into(),
20454 },
20455 )
20456 .await;
20457 assert!(matches!(
20458 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
20459 ModuleControlResponseToModule::ScopeDescribe { .. }
20460 ));
20461 }
20462
20463 use subc_protocol::scope::{
20464 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
20465 ScopeStatus,
20466 };
20467
20468 use super::*;
20469
20470 const OWNER: &str = "prefrontal-core";
20471
20472 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
20473 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
20474 }
20475
20476 async fn call(
20477 handler: &ControlHandler,
20478 ctx: &RouteCtx,
20479 request: &ModuleControlRequestFromModule,
20480 ) -> Frame {
20481 let body = serde_json::to_vec(request).unwrap();
20482 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
20483 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
20484 assert_eq!(replies.len(), 1, "{replies:?}");
20485 replies.pop().unwrap()
20486 }
20487
20488 async fn sync(
20489 handler: &ControlHandler,
20490 ctx: &RouteCtx,
20491 generation: u64,
20492 scopes: Vec<ScopeRecord>,
20493 ) -> Result<ModuleControlResponseToModule, String> {
20494 let reply = call(
20495 handler,
20496 ctx,
20497 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
20498 )
20499 .await;
20500 match reply.header.ty {
20501 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
20502 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
20503 }
20504 }
20505
20506 async fn describe(
20507 handler: &ControlHandler,
20508 ctx: &RouteCtx,
20509 owner: &str,
20510 scope_ref: &str,
20511 ) -> ModuleControlResponseToModule {
20512 let reply = call(
20513 handler,
20514 ctx,
20515 &ModuleControlRequestFromModule::ScopeDescribe {
20516 owner: Principal::Reserved {
20517 module_id: owner.to_string(),
20518 },
20519 scope_ref: scope_ref.to_string(),
20520 },
20521 )
20522 .await;
20523 assert_eq!(
20524 reply.header.ty,
20525 FrameType::Response,
20526 "{:?}",
20527 parse_error(&reply)
20528 );
20529 serde_json::from_slice(&reply.body).unwrap()
20530 }
20531
20532 #[tokio::test]
20533 async fn scope_describe_never_echoes_bind_proof_or_hash() {
20534 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
20535 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
20536 let mut rig = rig().await;
20537 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
20538 .await;
20539 let _bound = rig
20540 .bound(
20541 Some(AFT),
20542 PLEXUS,
20543 Some(
20544 rig.selector("s", Some(1))
20545 .with_bind_proof(Some(PROOF.into())),
20546 ),
20547 )
20548 .await;
20549 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
20550 let output = serde_json::to_string(&reply).unwrap();
20551 assert!(output.contains("\"status\":\"live\""));
20552 assert!(!output.contains(PROOF));
20553 assert!(!output.contains(HASH));
20554 }
20555
20556 async fn module(
20558 handler: &ControlHandler,
20559 connection: u64,
20560 module_id: &str,
20561 nonce: Option<&str>,
20562 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20563 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
20564 hello_via_sink(
20565 handler,
20566 &ctx,
20567 &mut rx,
20568 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
20569 )
20570 .await;
20571 (ctx, rx)
20572 }
20573
20574 #[tokio::test]
20577 async fn a_client_connection_cannot_sync_or_describe() {
20578 let handler = ControlHandler::new(Arc::new(Registry::default()));
20579 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
20580 for request in [
20581 ModuleControlRequestFromModule::ScopeSync {
20582 generation: 1,
20583 scopes: vec![head("s", 1)],
20584 },
20585 ModuleControlRequestFromModule::ScopeDescribe {
20586 owner: Principal::Direct,
20587 scope_ref: "s".to_string(),
20588 },
20589 ] {
20590 let reply = call(&handler, &ctx, &request).await;
20591 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
20592 }
20593 assert!(
20594 !handler
20595 .scopes
20596 .read()
20597 .unwrap()
20598 .describe(
20599 &Principal::Reserved {
20600 module_id: OWNER.to_string()
20601 },
20602 "s"
20603 )
20604 .owner_synced
20605 );
20606 }
20607
20608 #[tokio::test]
20611 async fn a_module_without_a_supervised_launch_cannot_sync() {
20612 let handler = ControlHandler::new(Arc::new(Registry::default()));
20613 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
20614 assert_eq!(
20615 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
20616 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20617 );
20618 }
20619
20620 #[tokio::test]
20621 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
20622 let supervisor = SupervisorHandle::new();
20623 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20624 let handler = ControlHandler::new(Arc::new(Registry::default()))
20625 .with_supervisor(supervisor.clone());
20626 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20627 sync(&handler, &incumbent, 1, vec![head("s", 1)])
20628 .await
20629 .expect("the current launch syncs");
20630
20631 supervisor.open_swap(OWNER, "n2".to_string());
20634 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
20635 assert_eq!(
20636 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20637 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20638 );
20639 sync(&handler, &incumbent, 2, vec![head("s", 1)])
20640 .await
20641 .expect("the serving owner syncs during the swap");
20642
20643 supervisor.close_swap(OWNER);
20646 assert_eq!(
20647 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20648 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20649 );
20650 sync(&handler, &incumbent, 3, vec![head("s", 1)])
20651 .await
20652 .expect("the serving owner syncs after the rollback");
20653 handler.cleanup_connection(candidate.connection_id).unwrap();
20654
20655 supervisor.open_swap(OWNER, "n3".to_string());
20660 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
20661 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
20662 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
20663 .await
20664 .expect("the promoted launch takes authority");
20665 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20666 panic!("unexpected reply {reply:?}");
20667 };
20668 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
20669 assert_eq!(
20670 sync(&handler, &incumbent, 4, Vec::new()).await,
20671 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20672 );
20673 }
20674
20675 #[tokio::test]
20678 async fn closing_the_authority_connection_frees_sync_authority() {
20679 let supervisor = SupervisorHandle::new();
20680 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20681 let handler = ControlHandler::new(Arc::new(Registry::default()))
20682 .with_supervisor(supervisor.clone());
20683 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20684 sync(&handler, &first, 10, vec![head("s", 1)])
20685 .await
20686 .unwrap();
20687 handler.cleanup_connection(first.connection_id).unwrap();
20688
20689 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
20690 sync(&handler, &second, 1, vec![head("s", 1)])
20691 .await
20692 .expect("the next connection takes the released authority");
20693 }
20694
20695 #[tokio::test]
20696 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
20697 let supervisor = SupervisorHandle::new();
20698 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20699 let handler =
20700 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
20701 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
20702 sync(&handler, &first, 10, vec![head("s", 1)])
20703 .await
20704 .unwrap();
20705 handler
20706 .handle_control_frame(
20707 &first,
20708 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
20709 )
20710 .await
20711 .unwrap();
20712 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
20713 sync(&handler, &second, 1, vec![head("s", 1)])
20714 .await
20715 .expect("GOODBYE releases authority even if the old socket remains open");
20716 }
20717
20718 #[tokio::test]
20719 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
20720 let registry = Arc::new(Registry::default());
20721 let supervisor_handle = SupervisorHandle::new();
20722 let supervisor =
20723 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20724 .with_handle(supervisor_handle.clone())
20725 .with_daemon_incarnation("incarnation-7".to_string());
20726 supervisor
20729 .supervise_configured(
20730 ModuleSpec {
20731 script: None,
20732 carry_containment: false,
20733 module_id: OWNER.to_string(),
20734 program: PathBuf::from("/nonexistent/prefrontal-core"),
20735 args: Vec::new(),
20736 env: Vec::new(),
20737 reserved: false,
20738 reserved_prefixes: Vec::new(),
20739 protocol: ModuleProtocol::Subc,
20740 overlap: Default::default(),
20741 },
20742 false,
20743 )
20744 .unwrap();
20745 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
20746 let handler =
20747 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
20748 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
20749
20750 let ModuleControlResponseToModule::ScopeDescribe {
20752 status,
20753 daemon_incarnation,
20754 owner_synced,
20755 owner_configured,
20756 scope,
20757 ..
20758 } = describe(&handler, &reader, OWNER, "s").await
20759 else {
20760 panic!("not a describe reply");
20761 };
20762 assert_eq!(status, ScopeStatus::NotLive);
20763 assert_eq!(daemon_incarnation, "incarnation-7");
20764 assert!(!owner_synced);
20765 assert!(owner_configured);
20766 assert!(scope.is_none());
20767
20768 let ModuleControlResponseToModule::ScopeDescribe {
20771 status,
20772 owner_configured,
20773 ..
20774 } = describe(&handler, &reader, "ghost", "s").await
20775 else {
20776 panic!("not a describe reply");
20777 };
20778 assert_eq!(status, ScopeStatus::NotLive);
20779 assert!(!owner_configured);
20780
20781 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
20783 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
20784 let ModuleControlResponseToModule::ScopeDescribe {
20785 status,
20786 scope_epoch,
20787 owner_synced,
20788 scope,
20789 ..
20790 } = describe(&handler, &reader, OWNER, "s").await
20791 else {
20792 panic!("not a describe reply");
20793 };
20794 assert_eq!(status, ScopeStatus::Live);
20795 assert_eq!(scope_epoch, Some(4));
20796 assert!(owner_synced);
20797 let stamp = scope.expect("a live scope carries its stamp");
20798 assert!(
20799 stamp.owner_authorized,
20800 "prefrontal-core is the default authority"
20801 );
20802 assert_eq!(stamp.kind, ScopeKind::Head);
20803 }
20804
20805 #[tokio::test]
20806 async fn scope_authority_owners_decides_owner_authorized() {
20807 let supervisor = SupervisorHandle::new();
20808 supervisor.set_spawn_nonce("broca", "b1".to_string());
20809 let handler = ControlHandler::new(Arc::new(Registry::default()))
20810 .with_supervisor(supervisor)
20811 .with_scope_authority_owners(vec!["broca".to_string()]);
20812 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
20813 let mut gated = head("s", 1);
20814 gated.attributes.agent_id = Some("agent".to_string());
20815 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
20816 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
20817 describe(&handler, &broca, "broca", "s").await
20818 else {
20819 panic!("not a describe reply");
20820 };
20821 assert!(scope.unwrap().owner_authorized);
20822 }
20823
20824 #[tokio::test]
20828 async fn scope_ops_and_the_scopes_capability_are_advertised() {
20829 let handler = ControlHandler::new(Arc::new(Registry::default()));
20830 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
20831 let ack = hello_via_sink(
20832 &handler,
20833 &ctx,
20834 &mut rx,
20835 hello_frame("m", PROTOCOL_VERSION, 1),
20836 )
20837 .await;
20838 let ack = parse_ack(&ack);
20839 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
20840 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
20841 }
20842 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
20843
20844 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
20845 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
20846 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
20847 let reply = handler
20848 .handle_control_frame(&client, frame)
20849 .await
20850 .unwrap()
20851 .pop()
20852 .unwrap();
20853 let ClientControlResponse::ServerDescribe { capabilities, .. } =
20854 serde_json::from_slice(&reply.body).unwrap()
20855 else {
20856 panic!("not a server.describe reply");
20857 };
20858 assert!(
20859 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
20860 "{capabilities:?}"
20861 );
20862 }
20863
20864 const PLEXUS: &str = "plexus";
20867 const OTHER: &str = "other";
20868 const AFT: &str = "aft";
20869 const BROCA: &str = "broca";
20870 const MAGIC: &str = "magic-context";
20871
20872 fn nonce(module_id: &str) -> String {
20873 format!("nonce-{module_id}")
20874 }
20875
20876 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20877 let (tx, rx) = mpsc::channel(64);
20878 (
20879 RouteCtx {
20880 connection_id: ConnectionId::new(connection),
20881 egress: FrameSink::new(tx),
20882 },
20883 rx,
20884 )
20885 }
20886
20887 struct Rig {
20891 handler: ControlHandler,
20892 forwarding: Arc<ForwardingTable>,
20893 owner: RouteCtx,
20894 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20895 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
20896 generation: u64,
20897 next_connection: u64,
20898 _supervisor: Supervisor,
20899 }
20900
20901 async fn rig() -> Rig {
20902 rig_with_flow_support(true).await
20903 }
20904
20905 async fn rig_with_flow_support(flow_support: bool) -> Rig {
20906 rig_with_scope_capabilities(flow_support, false).await
20907 }
20908
20909 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
20910 let mut provides = Vec::new();
20911 if flow_support {
20912 provides.push("flow-scopes/v1");
20913 }
20914 if run_support {
20915 provides.push("agent-run-scopes/v1");
20916 }
20917 rig_with_target_capabilities(&provides).await
20918 }
20919
20920 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
20921 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
20922 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20923 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20926 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
20927 Frame::build(
20928 FrameType::Hello,
20929 control_flags(),
20930 0,
20931 0,
20932 connection,
20933 serde_json::to_vec(&body).unwrap(),
20934 )
20935 .unwrap()
20936 }
20937
20938 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
20939 let registry = Arc::new(Registry::default());
20940 let forwarding = Arc::new(ForwardingTable::default());
20941 let supervisor_handle = SupervisorHandle::new();
20942 let supervisor =
20943 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20944 .with_handle(supervisor_handle.clone());
20945 supervisor
20946 .supervise_configured(
20947 ModuleSpec {
20948 script: None,
20949 carry_containment: false,
20950 module_id: OWNER.to_string(),
20951 program: PathBuf::from("/nonexistent/prefrontal-core"),
20952 args: Vec::new(),
20953 env: Vec::new(),
20954 reserved: false,
20955 reserved_prefixes: Vec::new(),
20956 protocol: ModuleProtocol::Subc,
20957 overlap: Default::default(),
20958 },
20959 false,
20960 )
20961 .unwrap();
20962 for module_id in [OWNER, AFT, BROCA, MAGIC] {
20963 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
20964 }
20965 let handler =
20966 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
20967 .with_supervisor(supervisor_handle);
20968 let (owner, mut owner_rx) = wide_ctx(1);
20969 hello_via_sink(
20970 &handler,
20971 &owner,
20972 &mut owner_rx,
20973 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
20974 )
20975 .await;
20976 let mut modules = BTreeMap::new();
20977 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
20978 let (ctx, mut rx) = wide_ctx(connection);
20979 let hello = hello_with_capabilities(module_id, connection, provides);
20980 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
20981 modules.insert(module_id.to_string(), (ctx, rx));
20982 }
20983 Rig {
20984 handler,
20985 forwarding,
20986 owner,
20987 _owner_rx: owner_rx,
20988 modules,
20989 generation: 0,
20990 next_connection: 100,
20991 _supervisor: supervisor,
20992 }
20993 }
20994
20995 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
20996 ScopeCarrier::new(Principal::Reserved {
20997 module_id: module_id.to_string(),
20998 })
20999 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
21000 }
21001
21002 fn session(scope_epoch: u64) -> ScopeRecord {
21005 let mut record = head("s", scope_epoch);
21006 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
21007 record.attributes.agent_id = Some("agent-1".to_string());
21008 record.attributes.delegates = true;
21009 record
21010 }
21011
21012 impl Rig {
21013 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
21014 self.generation += 1;
21015 sync(&self.handler, &self.owner, self.generation, scopes)
21016 .await
21017 .expect("the owner's sync is accepted");
21018 }
21019
21020 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
21021 ScopeSelector::new(
21022 Principal::Reserved {
21023 module_id: OWNER.to_string(),
21024 },
21025 scope_ref.to_string(),
21026 scope_epoch,
21027 )
21028 }
21029
21030 fn open_frame(
21031 &mut self,
21032 opener: Option<&str>,
21033 target: &str,
21034 scope: Option<ScopeSelector>,
21035 ) -> (
21036 RouteCtx,
21037 mpsc::Receiver<crate::router::OutboundFrame>,
21038 Frame,
21039 ) {
21040 self.next_connection += 1;
21041 let (ctx, rx) = wide_ctx(self.next_connection);
21042 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
21043 busy_refusals: None,
21044 target: RouteTarget::ToolProvider {
21045 module_id: target.to_string(),
21046 },
21047 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
21048 consumer_identity: opener.map(|module_id| ConsumerIdentity {
21049 module_id: module_id.to_string(),
21050 launch_nonce: nonce(module_id),
21051 }),
21052 consumer_capabilities: None,
21053 role_versions: None,
21054 admission_facts: None,
21055 scope,
21056 })
21057 .unwrap();
21058 let frame = Frame::build(
21059 FrameType::Request,
21060 control_flags(),
21061 0,
21062 0,
21063 self.next_connection,
21064 body,
21065 )
21066 .unwrap();
21067 (ctx, rx, frame)
21068 }
21069
21070 async fn refused(
21072 &mut self,
21073 opener: Option<&str>,
21074 target: &str,
21075 scope: Option<ScopeSelector>,
21076 ) -> String {
21077 self.refusal_body(opener, target, scope).await["code"]
21078 .as_str()
21079 .unwrap()
21080 .to_string()
21081 }
21082
21083 async fn refusal_body(
21084 &mut self,
21085 opener: Option<&str>,
21086 target: &str,
21087 scope: Option<ScopeSelector>,
21088 ) -> Value {
21089 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
21090 let replies = tokio::time::timeout(
21091 Duration::from_secs(2),
21092 self.handler.handle_control_frame(&ctx, frame),
21093 )
21094 .await
21095 .expect("the open must be refused before waiting for a bind ack")
21096 .unwrap();
21097 assert_eq!(replies.len(), 1, "{replies:?}");
21098 assert_eq!(replies[0].header.ty, FrameType::Error);
21099 let (_, module_rx) = self.modules.get_mut(target).unwrap();
21100 assert!(
21101 module_rx.try_recv().is_err(),
21102 "a refused open relays nothing"
21103 );
21104 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
21105 parse_error(&replies[0])
21106 }
21107
21108 async fn relayed(
21110 &mut self,
21111 opener: Option<&str>,
21112 target: &str,
21113 scope: Option<ScopeSelector>,
21114 ) -> Relayed {
21115 self.relayed_with_legacy_root(opener, target, scope, None)
21116 .await
21117 }
21118
21119 async fn relayed_with_legacy_root(
21120 &mut self,
21121 opener: Option<&str>,
21122 target: &str,
21123 scope: Option<ScopeSelector>,
21124 legacy_root: Option<&std::path::Path>,
21125 ) -> Relayed {
21126 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
21127 if let Some(root) = legacy_root {
21128 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
21129 body["identity"]["project_root"] = serde_json::json!(root);
21130 frame.body = serde_json::to_vec(&body).unwrap();
21131 }
21132 let handler = self.handler.clone();
21133 let task_ctx = ctx.clone();
21134 let task = tokio::spawn(async move {
21135 handler
21136 .handle_control_frame(&task_ctx, frame)
21137 .await
21138 .unwrap()
21139 });
21140 let (_, module_rx) = self.modules.get_mut(target).unwrap();
21141 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
21142 .await
21143 .expect("the target receives the relayed route.bind")
21144 .unwrap()
21145 .frame;
21146 Relayed {
21147 target: target.to_string(),
21148 client: ctx,
21149 client_rx: rx,
21150 task,
21151 bind,
21152 }
21153 }
21154
21155 async fn ack(&self, relayed: &Relayed) {
21156 let (module, _) = &self.modules[&relayed.target];
21157 self.handler
21158 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
21159 .await
21160 .unwrap();
21161 }
21162
21163 async fn bound(
21165 &mut self,
21166 opener: Option<&str>,
21167 target: &str,
21168 scope: Option<ScopeSelector>,
21169 ) -> Bound {
21170 let relayed = self.relayed(opener, target, scope).await;
21171 self.commit_relayed(relayed).await
21172 }
21173
21174 async fn bound_with_legacy_root(
21175 &mut self,
21176 opener: Option<&str>,
21177 target: &str,
21178 scope: Option<ScopeSelector>,
21179 root: &std::path::Path,
21180 ) -> Bound {
21181 let relayed = self
21182 .relayed_with_legacy_root(opener, target, scope, Some(root))
21183 .await;
21184 self.commit_relayed(relayed).await
21185 }
21186
21187 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
21188 self.ack(&relayed).await;
21189 let Relayed {
21190 target,
21191 client,
21192 mut client_rx,
21193 task,
21194 bind,
21195 } = relayed;
21196 assert!(
21197 task.await.unwrap().is_empty(),
21198 "the open is answered by commit"
21199 );
21200 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
21201 Bound {
21202 target,
21203 client,
21204 client_rx,
21205 channel,
21206 epoch,
21207 bind,
21208 }
21209 }
21210
21211 fn live(&self, route: &Bound) -> bool {
21212 matches!(
21213 self.forwarding
21214 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21215 .unwrap(),
21216 DataRoute::Client(DataRouteState::Bound(_))
21217 )
21218 }
21219 }
21220
21221 struct Relayed {
21222 target: String,
21223 client: RouteCtx,
21224 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21225 task: tokio::task::JoinHandle<Vec<Frame>>,
21226 bind: Frame,
21227 }
21228
21229 struct Bound {
21230 target: String,
21231 client: RouteCtx,
21232 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21233 channel: u16,
21234 epoch: u32,
21235 bind: Frame,
21236 }
21237
21238 impl Bound {
21239 fn closed_reason(&mut self) -> RouteCloseReason {
21242 let mut reason = None;
21243 let mut goodbye = false;
21244 while let Ok(outbound) = self.client_rx.try_recv() {
21245 let frame = outbound.frame;
21246 match frame.header.ty {
21247 FrameType::Goodbye => {
21248 assert_eq!(
21249 (frame.header.channel, frame.header.epoch),
21250 (self.channel, self.epoch)
21251 );
21252 goodbye = true;
21253 }
21254 FrameType::Push => {
21255 let ClientControlPush::RouteClosed {
21256 reason: r,
21257 module_id,
21258 ..
21259 } = serde_json::from_slice(&frame.body).unwrap()
21260 else {
21261 panic!("unexpected push");
21262 };
21263 assert_eq!(module_id, self.target);
21264 reason = Some(r);
21265 }
21266 other => panic!("unexpected frame {other:?}"),
21267 }
21268 }
21269 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
21270 reason.expect("the client is told why the route closed")
21271 }
21272
21273 fn untouched(&mut self) -> bool {
21274 self.client_rx.try_recv().is_err()
21275 }
21276
21277 fn stamp(&self) -> Option<ScopeStamp> {
21278 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
21279 ModuleControlRequest::RouteBind { scope, .. } => scope,
21280 other => panic!("expected a route.bind, got {other:?}"),
21281 }
21282 }
21283 }
21284
21285 #[tokio::test]
21286 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
21287 ) {
21288 let mut rig = rig().await;
21289 let mut record = session(1);
21290 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
21291 record.child_owners = vec![Principal::Reserved {
21292 module_id: MAGIC.to_string(),
21293 }];
21294 rig.sync(vec![record]).await;
21295 let scope = || Some(rig_selector("s", Some(1)));
21296
21297 rig.bound(Some(OWNER), PLEXUS, scope()).await;
21300 rig.bound(Some(AFT), OTHER, scope()).await;
21301 rig.bound(Some(BROCA), PLEXUS, scope()).await;
21302
21303 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
21307 assert_eq!(
21308 rig.refused(opener, target, scope()).await,
21309 error_codes::SCOPE_NOT_CARRIER,
21310 "{opener:?} -> {target}"
21311 );
21312 }
21313 }
21314
21315 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
21316 ScopeSelector::new(
21317 Principal::Reserved {
21318 module_id: OWNER.to_string(),
21319 },
21320 scope_ref.to_string(),
21321 scope_epoch,
21322 )
21323 }
21324
21325 #[tokio::test]
21326 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
21327 let mut rig = rig().await;
21328 rig.sync(vec![session(1)]).await;
21329 for opener in [OWNER, AFT] {
21330 assert_eq!(
21331 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
21332 .await,
21333 error_codes::SCOPE_EPOCH_REQUIRED,
21334 "{opener}"
21335 );
21336 }
21337 }
21338
21339 #[tokio::test]
21340 async fn admission_separates_not_synced_not_live_and_ended() {
21341 let mut rig = rig().await;
21342 let code = rig
21344 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21345 .await;
21346 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
21347 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
21348
21349 let ghost = ScopeSelector::new(
21351 Principal::Reserved {
21352 module_id: "ghost".to_string(),
21353 },
21354 "s".to_string(),
21355 Some(1),
21356 );
21357 assert_eq!(
21358 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
21359 error_codes::SCOPE_NOT_LIVE
21360 );
21361
21362 rig.sync(vec![session(2)]).await;
21363 assert_eq!(
21364 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
21365 .await,
21366 error_codes::SCOPE_NOT_LIVE
21367 );
21368 for epoch in [1, 3] {
21369 assert_eq!(
21370 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
21371 .await,
21372 error_codes::SCOPE_ENDED,
21373 "epoch {epoch}"
21374 );
21375 }
21376 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
21378 .await;
21379 }
21380
21381 #[tokio::test]
21382 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
21383 let mut rig = rig().await;
21384 rig.sync(vec![session(1)]).await;
21385 let route = rig
21386 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21387 .await;
21388 let stamp = route.stamp().expect("a scoped bind carries the stamp");
21389 assert_eq!(stamp.scope_ref, "s");
21390 assert_eq!(stamp.scope_epoch, 1);
21391 assert_eq!(stamp.kind, ScopeKind::Head);
21392 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
21393 assert!(stamp.attributes.delegates);
21394 assert!(stamp.owner_authorized);
21395 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21396 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
21397
21398 let (broca, mut broca_rx) = wide_ctx(50);
21401 hello_via_sink(
21402 &rig.handler,
21403 &broca,
21404 &mut broca_rx,
21405 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
21406 )
21407 .await;
21408 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
21409 .await
21410 .unwrap();
21411 let own = ScopeSelector::new(
21412 Principal::Reserved {
21413 module_id: BROCA.to_string(),
21414 },
21415 "b".to_string(),
21416 Some(1),
21417 );
21418 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
21419 assert!(!route.stamp().unwrap().owner_authorized);
21420 }
21421
21422 #[tokio::test]
21423 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
21424 let mut rig = rig().await;
21425 let mut record = head("s", 1);
21426 record.carriers = vec![carrier(AFT, None)];
21427 let flow_id = "Flow:run-7/step_2!~";
21428 record.attributes.flow_id = Some(flow_id.to_string());
21429 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
21430 .await
21431 .unwrap();
21432 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
21433 panic!("not a sync reply");
21434 };
21435 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
21436 let route = rig
21437 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21438 .await;
21439 assert!(rig.live(&route), "the stamped bind committed");
21440 let stamp = route.stamp().expect("a flow scope carries a stamp");
21441 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
21442 assert_eq!(stamp.attributes.agent_id, None);
21443 assert!(!stamp.attributes.delegates);
21444 assert!(stamp.owner_authorized);
21445 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21446 assert_eq!(unscoped.stamp(), None);
21447 }
21448
21449 #[tokio::test]
21450 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
21451 let mut rig = rig_with_flow_support(false).await;
21452 let mut record = session(1);
21453 record.attributes.flow_id = Some("flow:7".to_string());
21454 rig.sync(vec![record]).await;
21455 for opener in [OWNER, AFT] {
21456 let body = rig
21457 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21458 .await;
21459 assert_eq!(body["code"], "target_flow_unsupported");
21460 let message = body["message"].as_str().unwrap();
21461 for required in [PLEXUS, "flow-scopes/v1"] {
21462 assert!(message.contains(required), "{message}");
21463 }
21464 }
21465 }
21466
21467 #[tokio::test]
21468 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
21469 let mut rig = rig_with_flow_support(true).await;
21470 let mut record = session(1);
21471 record.attributes.flow_id = Some("flow:7".to_string());
21472 rig.sync(vec![record]).await;
21473 for opener in [OWNER, AFT] {
21474 let route = rig
21475 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21476 .await;
21477 assert!(rig.live(&route));
21478 assert_eq!(
21479 route.stamp().unwrap().attributes.flow_id.as_deref(),
21480 Some("flow:7")
21481 );
21482 }
21483 }
21484
21485 #[tokio::test]
21486 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
21487 use std::future::Future;
21488
21489 let mut rig = rig_with_flow_support(true).await;
21490 let mut record = session(1);
21491 record.attributes.flow_id = Some("flow:7".to_string());
21492 rig.sync(vec![record]).await;
21493 let (client, mut client_rx, frame) =
21494 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
21495 for _ in 0..64 {
21498 client.egress.try_send(route_bind_ack(1)).unwrap();
21499 }
21500 let handler = rig.handler.clone();
21501 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21502 std::future::poll_fn(|cx| {
21503 assert!(open.as_mut().poll(cx).is_pending());
21504 std::task::Poll::Ready(())
21505 })
21506 .await;
21507 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21508
21509 let old_connection = rig.modules[PLEXUS].0.connection_id;
21510 rig.handler.cleanup_connection(old_connection).unwrap();
21511 let (replacement, mut replacement_rx) = wide_ctx(200);
21512 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
21513 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
21514 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
21515 let hello = Frame::build(
21516 FrameType::Hello,
21517 control_flags(),
21518 0,
21519 0,
21520 200,
21521 serde_json::to_vec(&body).unwrap(),
21522 )
21523 .unwrap();
21524 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
21525
21526 client_rx.try_recv().unwrap();
21527 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21528 .await
21529 .expect("the replacement is refused without waiting for a bind ack")
21530 .unwrap();
21531 assert_eq!(replies.len(), 1);
21532 let body = parse_error(&replies[0]);
21533 assert_eq!(body["code"], "target_flow_unsupported");
21534 for required in [PLEXUS, "flow-scopes/v1"] {
21535 assert!(body["message"].as_str().unwrap().contains(required));
21536 }
21537 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
21538 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21539 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21540 assert!(rig
21541 .handler
21542 .registry
21543 .get_module_by_connection(replacement.connection_id)
21544 .unwrap()
21545 .is_some());
21546 }
21547
21548 #[tokio::test]
21549 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
21550 use std::future::Future;
21551 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
21552 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21553 const NEXT_HASH: &str =
21554 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21555 let mut rig = rig().await;
21556 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
21557 record.carriers.push(ScopeCarrier::new(Principal::Direct));
21558 rig.sync(vec![record.clone()]).await;
21559 let (client, mut rx, frame) = rig.open_frame(
21560 None,
21561 PLEXUS,
21562 Some(
21563 rig.selector("s", Some(1))
21564 .with_bind_proof(Some(PROOF.into())),
21565 ),
21566 );
21567 for _ in 0..64 {
21570 client.egress.try_send(route_bind_ack(1)).unwrap();
21571 }
21572 let handler = rig.handler.clone();
21573 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21574 std::future::poll_fn(|cx| {
21575 assert!(open.as_mut().poll(cx).is_pending());
21576 std::task::Poll::Ready(())
21577 })
21578 .await;
21579 record.bind_proof_sha256 = Some(NEXT_HASH.into());
21580 rig.sync(vec![record]).await;
21581 rx.try_recv().unwrap();
21582 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21583 .await
21584 .unwrap()
21585 .unwrap();
21586 assert_eq!(
21587 parse_error(&replies[0])["code"],
21588 error_codes::SCOPE_BIND_PROOF_MISMATCH
21589 );
21590 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
21591 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21592 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21593 }
21594
21595 #[tokio::test]
21596 async fn scope_bind_proof_reserved_openers_ignore_proof() {
21597 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21598 let mut rig = rig().await;
21599 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21600 .await;
21601 for opener in [OWNER, AFT] {
21602 for proof in [None, Some("malformed+proof=")] {
21603 let route = rig
21604 .bound(
21605 Some(opener),
21606 PLEXUS,
21607 Some(
21608 rig.selector("s", Some(1))
21609 .with_bind_proof(proof.map(str::to_string)),
21610 ),
21611 )
21612 .await;
21613 assert!(rig.live(&route));
21614 }
21615 }
21616 }
21617
21618 #[tokio::test]
21619 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
21620 use std::future::Future;
21621 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21622 const NEXT_HASH: &str =
21623 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21624 let mut rig = rig().await;
21625 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21626 .await;
21627 let (client, mut rx, frame) =
21628 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
21629 for _ in 0..64 {
21630 client.egress.try_send(route_bind_ack(1)).unwrap();
21631 }
21632 let handler = rig.handler.clone();
21633 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21634 std::future::poll_fn(|cx| {
21635 assert!(open.as_mut().poll(cx).is_pending());
21636 std::task::Poll::Ready(())
21637 })
21638 .await;
21639 rig.sync(vec![
21640 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
21641 ])
21642 .await;
21643 rx.try_recv().unwrap();
21644 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21645 let bind = std::future::poll_fn(|cx| {
21646 assert!(open.as_mut().poll(cx).is_pending());
21647 module_rx.poll_recv(cx)
21648 })
21649 .await
21650 .unwrap()
21651 .frame;
21652 handler
21653 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
21654 .await
21655 .unwrap();
21656 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21657 .await
21658 .unwrap()
21659 .unwrap();
21660 assert_eq!(replies.len(), 1);
21661 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
21664 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21665 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
21666 let route = rig
21667 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
21668 .await;
21669 assert!(rig.live(&route));
21670 }
21671
21672 #[tokio::test]
21673 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
21674 {
21675 let mut rig = rig_with_flow_support(false).await;
21676 rig.sync(vec![session(1)]).await;
21677 let route = rig
21678 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21679 .await;
21680 assert!(rig.live(&route));
21681 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
21682 let mut record = session(1);
21683 record.attributes.flow_id = Some("flow:7".to_string());
21684 rig.sync(vec![record]).await;
21685 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
21686 assert!(rig.live(&unscoped));
21687 assert_eq!(unscoped.stamp(), None);
21688 }
21689
21690 #[tokio::test]
21691 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
21692 let mut rig = rig().await;
21693 let mut record = session(1);
21694 record.attributes.flow_id = Some("flow:7".to_string());
21695 rig.sync(vec![record.clone()]).await;
21696 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21697 let mut owner_route = rig
21698 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21699 .await;
21700 let mut carrier_route = rig
21701 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
21702 .await;
21703 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21704 rig.sync(vec![record.clone()]).await;
21705 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21706 assert!(rig.live(&owner_route) && owner_route.untouched());
21707 assert!(rig.live(&carrier_route) && carrier_route.untouched());
21708
21709 record.attributes.flow_id = Some("flow:8".to_string());
21710 rig.sync(vec![record]).await;
21711 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
21712 let before = before.unwrap();
21713 assert_eq!(after.scope_epoch, before.scope_epoch);
21714 assert!(after.version > before.version);
21715 for route in [&mut owner_route, &mut carrier_route] {
21716 assert!(!rig.live(route));
21717 assert_eq!(
21718 route.closed_reason(),
21719 RouteCloseReason::ScopeDelegationChanged
21720 );
21721 }
21722 assert!(rig.live(&unscoped) && unscoped.untouched());
21723 for target in [PLEXUS, OTHER] {
21726 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
21727 let goodbye = module_rx
21728 .try_recv()
21729 .expect("the provider sees the drain")
21730 .frame;
21731 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21732 assert!(module_rx.try_recv().is_err());
21733 }
21734 let rebound = rig
21735 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21736 .await;
21737 assert_eq!(
21738 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
21739 Some("flow:8")
21740 );
21741 }
21742
21743 #[tokio::test]
21748 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
21749 let mut rig = rig().await;
21750 rig.sync(vec![session(1)]).await;
21751 let mut cotenant = rig
21752 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21753 .await;
21754
21755 let mut changed = session(1);
21756 changed.child_owners.push(Principal::Reserved {
21757 module_id: MAGIC.to_string(),
21758 });
21759 let mut ended = None;
21760 for (code, next) in [
21761 (error_codes::SCOPE_CHANGED, vec![changed]),
21762 (error_codes::SCOPE_ENDED, Vec::new()),
21763 ] {
21764 let relayed = rig
21765 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21766 .await;
21767 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
21768 ended = Some(next.is_empty());
21769 rig.sync(next).await;
21770 rig.ack(&relayed).await;
21771 let replies = relayed.task.await.unwrap();
21772 assert_eq!(replies.len(), 1, "{replies:?}");
21773 assert_eq!(parse_error(&replies[0])["code"], code);
21774 assert_eq!(
21775 subc_protocol::error_codes::is_retryable_route_open(code),
21776 code == error_codes::SCOPE_CHANGED
21777 );
21778 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21779 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21781 let mut goodbyes = Vec::new();
21784 while let Ok(outbound) = plexus_rx.try_recv() {
21785 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
21786 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
21787 }
21788 assert!(
21789 goodbyes.contains(&(bind_channel, bind_epoch)),
21790 "{goodbyes:?}"
21791 );
21792 assert!(rig
21793 .handler
21794 .registry
21795 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
21796 .unwrap()
21797 .is_some());
21798 }
21799 assert_eq!(ended, Some(true));
21800 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
21803 }
21804
21805 #[tokio::test]
21808 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
21809 struct Case {
21810 name: &'static str,
21811 change: fn(&mut ScopeRecord),
21812 closed: [Option<RouteCloseReason>; 4],
21815 }
21816 use RouteCloseReason::*;
21817 let cases = [
21818 Case {
21819 name: "a carrier entry removed",
21820 change: |r| {
21821 r.carriers.retain(|c| {
21822 c.principal
21823 != Principal::Reserved {
21824 module_id: AFT.to_string(),
21825 }
21826 })
21827 },
21828 closed: [None, Some(ScopeCarrierRemoved), None, None],
21829 },
21830 Case {
21831 name: "a target removed from a carrier",
21832 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
21833 closed: [None, None, None, Some(ScopeCarrierRemoved)],
21834 },
21835 Case {
21836 name: "a bare carrier narrowed to targets",
21837 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
21838 closed: [None, Some(ScopeCarrierRemoved), None, None],
21839 },
21840 Case {
21841 name: "delegates turned off",
21842 change: |r| r.attributes.delegates = false,
21843 closed: [Some(ScopeDelegationChanged); 4],
21844 },
21845 Case {
21846 name: "agent_id changed",
21847 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
21848 closed: [Some(ScopeDelegationChanged); 4],
21849 },
21850 Case {
21851 name: "a carrier added, child owners changed, the record re-sent",
21852 change: |r| {
21853 r.carriers.push(carrier(MAGIC, None));
21854 r.child_owners.push(Principal::Reserved {
21855 module_id: MAGIC.to_string(),
21856 });
21857 },
21858 closed: [None; 4],
21859 },
21860 Case {
21861 name: "a target added",
21862 change: |r| {
21863 r.carriers[1]
21864 .targets
21865 .as_mut()
21866 .unwrap()
21867 .push("third".to_string())
21868 },
21869 closed: [None; 4],
21870 },
21871 Case {
21872 name: "delegates turned on",
21873 change: |r| r.attributes.delegates = true,
21874 closed: [None; 4],
21875 },
21876 ];
21877 for case in cases {
21878 let mut rig = rig().await;
21879 rig.sync(vec![session(1)]).await;
21880 let scope = || Some(rig_selector("s", Some(1)));
21881 let mut routes = [
21882 rig.bound(Some(OWNER), PLEXUS, scope()).await,
21883 rig.bound(Some(AFT), PLEXUS, scope()).await,
21884 rig.bound(Some(BROCA), PLEXUS, scope()).await,
21885 rig.bound(Some(BROCA), OTHER, scope()).await,
21886 ];
21887 let mut record = session(1);
21888 (case.change)(&mut record);
21889 rig.sync(vec![record]).await;
21890 for (index, expected) in case.closed.iter().enumerate() {
21891 let route = &mut routes[index];
21892 match expected {
21893 Some(reason) => {
21894 assert!(!rig.live(route), "{}: route {index} still live", case.name);
21895 assert_eq!(
21896 route.closed_reason(),
21897 *reason,
21898 "{}: route {index}",
21899 case.name
21900 );
21901 }
21902 None => {
21903 assert!(rig.live(route), "{}: route {index} closed", case.name);
21904 assert!(
21905 route.untouched(),
21906 "{}: route {index} was told something",
21907 case.name
21908 );
21909 }
21910 }
21911 }
21912 }
21913 }
21914
21915 #[tokio::test]
21916 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
21917 for next in [Vec::new(), vec![session(2)]] {
21919 let mut rig = rig().await;
21920 rig.sync(vec![session(1)]).await;
21921 let mut route = rig
21922 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21923 .await;
21924 rig.sync(next).await;
21925 assert!(!rig.live(&route));
21926 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
21927 }
21928
21929 let mut rig = rig().await;
21932 let mut child = session(1);
21933 child.scope_ref = "child".to_string();
21934 child.kind = ScopeKind::Worker;
21935 child.parent = Some(ScopeParent::new(
21936 Principal::Reserved {
21937 module_id: OWNER.to_string(),
21938 },
21939 "s".to_string(),
21940 1,
21941 ));
21942 rig.sync(vec![session(1), child.clone()]).await;
21943 let mut child_route = rig
21944 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
21945 .await;
21946 assert_eq!(
21947 child_route.stamp().unwrap().parent_state,
21948 Some(ParentState::Linked)
21949 );
21950 rig.sync(vec![child]).await;
21951 assert!(!rig.live(&child_route));
21952 assert_eq!(
21953 child_route.closed_reason(),
21954 RouteCloseReason::ScopeParentEnded
21955 );
21956 }
21957
21958 #[tokio::test]
21959 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
21960 ) {
21961 let mut rig = rig().await;
21962 rig.sync(vec![session(1)]).await;
21963 let mut route = rig
21964 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21965 .await;
21966 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21967 rig.sync(vec![session(1)]).await;
21968 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21969 assert!(rig.live(&route) && route.untouched());
21970
21971 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
21975 .forwarding
21976 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21977 .unwrap()
21978 else {
21979 panic!("the route is bound");
21980 };
21981 binding.flow.acquire_tagged(9, false).await.unwrap();
21982 let mut widened = session(1);
21983 widened.carriers.push(carrier(MAGIC, None));
21984 rig.sync(vec![widened]).await;
21985 assert!(rig.live(&route) && route.untouched());
21986 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21987 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
21988 assert_eq!(binding.flow.in_flight(), 1);
21991 binding
21992 .flow
21993 .acquire_tagged(10, false)
21994 .await
21995 .expect("the flow is still open");
21996 }
21997
21998 #[tokio::test]
22001 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
22002 let mut rig = rig().await;
22003 rig.sync(vec![session(1)]).await;
22004 let mut on_incumbent = rig
22005 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
22006 .await;
22007
22008 let (candidate, _candidate_rx) = wide_ctx(9);
22011 let registration = rig
22012 .handler
22013 .registry
22014 .register_candidate_with_control_ops(
22015 manifest(PLEXUS, PROTOCOL_VERSION),
22016 PROTOCOL_VERSION,
22017 candidate.connection_id,
22018 module_baseline_control_ops(),
22019 )
22020 .unwrap();
22021 rig.forwarding
22022 .register_candidate_module_connection(
22023 candidate.connection_id,
22024 PLEXUS.to_string(),
22025 PROTOCOL_VERSION,
22026 manifest_concurrency(®istration.manifest),
22027 candidate.egress.clone(),
22028 )
22029 .unwrap();
22030 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
22031 rig.handler
22032 .registry
22033 .promote_candidate(PLEXUS)
22034 .unwrap()
22035 .unwrap();
22036 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
22037
22038 rig.sync(Vec::new()).await;
22039 assert!(!rig.live(&on_incumbent));
22040 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
22041 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
22042 let goodbye = incumbent_rx
22043 .try_recv()
22044 .expect("the superseded endpoint is told")
22045 .frame;
22046 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
22047 }
22048 }
22049}
22050
22051#[cfg(test)]
22052mod concurrency_default_exposure_tests {
22053 use super::*;
22054
22055 fn hello_body(role_json: &str) -> Vec<u8> {
22056 format!(
22057 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
22058 )
22059 .into_bytes()
22060 }
22061
22062 fn manifest_from(body: &[u8]) -> ModuleManifest {
22063 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
22064 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
22065 .expect("manifest parses")
22066 }
22067
22068 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
22069
22070 #[test]
22071 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
22072 let body = hello_body(&format!(
22073 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
22074 ));
22075 let manifest = manifest_from(&body);
22076 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
22079 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
22080 }
22081
22082 #[test]
22083 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
22084 let body = hello_body(&format!(
22085 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
22086 ));
22087 let manifest = manifest_from(&body);
22088 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
22089 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
22090 }
22091
22092 #[test]
22093 fn non_management_roles_are_never_reported() {
22094 let body = hello_body(
22095 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
22096 );
22097 let manifest = manifest_from(&body);
22098 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
22099 }
22100}