1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26 error_codes,
27 manifest::{
28 validate_hello_capability_grammar, validate_hello_confirm_operations,
29 validate_hello_display_name, validate_hello_event_declarations,
30 validate_hello_flow_operations, validate_hello_operation_deferral,
31 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32 validate_hello_self_signal_declarations, validate_role_declarations_value,
33 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34 ProviderRole,
35 },
36 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
37 provider_declarations::ProviderDeclarations,
38 scope::{
39 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
40 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
41 SCOPE_SYNC_OP,
42 },
43 session::{
44 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
45 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
46 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
47 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
48 },
49 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
50 Priority, RouteTarget, PROTOCOL_VERSION,
51};
52use tokio::time::{timeout_at, Instant};
53use tracing::{debug, info, instrument::WithSubscriber, warn};
54
55use crate::{
56 caller_identity::CallerIdentity,
57 capability_requirements::{
58 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
59 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
60 RequirementStatus, RuntimeModule,
61 },
62 child_roster::DaemonShutdownFlag,
63 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
64 forwarding::{
65 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
66 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
67 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
68 },
69 observability::{
70 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
71 },
72 privileged_prompt::{prompt_text, PrivilegedOperation},
73 provenance::{
74 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
75 },
76 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
77 router::{daemon_error_flags, RouteCtx, RouterError},
78 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
79 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
80 stderr_tail::{CaptureState, TailEntry},
81 supervise::{
82 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
83 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
84 },
85 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
86};
87
88pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
94
95const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
96const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
97const CAP_PING_PONG: &str = "ping_pong_v1";
98const CAP_SESSION_ATTACH: &str = "session_attach_v1";
99const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
100
101const SUBC_CONTROL_OPS: &[&str] = &[
102 "claim.mint",
103 "claim.present",
104 ops::DAEMON_SHUTDOWN_INTENT,
105 ops::ACTIVITY_RECORD,
106 ops::ACTIVITY_QUERY,
107 ops::ACTIVITY_SUBSCRIBE,
108 ops::SERVER_DESCRIBE,
109 ops::CATALOG_LIST,
110 ops::CATALOG_WATCH,
111 ops::ROUTE_OPEN,
112 ops::ROUTE_POLL,
113 ops::ROUTE_CLOSING,
114 ops::ROUTE_CLOSED,
115 ops::SUPERVISOR_LIST,
116 ops::SUPERVISOR_RESTART,
117 ops::SUPERVISOR_SWAP,
118 ops::SUPERVISOR_RELOAD,
119 ops::SUPERVISOR_RESCAN,
120 ops::SUPERVISOR_RELEASE_RESERVED,
121 ops::SUPERVISOR_SET_ENABLED,
122 ops::SUPERVISOR_HEALTH_PROBE,
123 ops::SUPERVISOR_HEALTH,
124 ops::SUPERVISOR_STDERR_TAIL,
125 ops::SUPERVISOR_TERMINALS,
126 ops::SUPERVISOR_ROUTES,
127 ops::SUPERVISOR_PROVENANCE,
128 ops::SUPERVISOR_SPAWN_SNAPSHOT,
129 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
130];
131
132const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[
139 ops::SUPERVISOR_RESCAN_CHECKED,
140 "config.status",
141 "config.plan",
142 "config.apply",
143 "config.stamp",
144 "config.unstamp",
145 "config.history",
146 "config.rollback",
147];
148
149mod caller_attestation;
150mod channel_operations;
151#[cfg(all(test, unix, feature = "test-support"))]
152#[path = "control/channel_operations_tests.rs"]
153mod channel_operations_tests;
154pub(crate) mod claims;
155#[cfg(test)]
156#[path = "control/config_advertisement_tests.rs"]
157mod config_advertisement_tests;
158mod config_gate;
159use caller_attestation::{caller_for_target, scope_acts_for_another};
160#[cfg(all(test, unix, feature = "test-support"))]
161#[path = "control/caller_attestation_tests.rs"]
162mod caller_attestation_tests;
163mod scope_expiry;
164
165mod agent_commands;
166pub(crate) mod catalog;
167mod rescan_checked;
168use rescan_checked::{RescanMode, RescanPlan};
169
170const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
171 "claim.mint",
172 "claim.present",
173 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
174 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
175 "supervisor.live_roots",
176 "supervisor.agent_commands",
177 SCOPE_SYNC_OP,
178 SCOPE_APPLY_OP,
179 SCOPE_DESCRIBE_OP,
180 subc_protocol::scope::ROUTE_AUTHORIZE_OP,
181 "operator.confirm",
182];
183
184const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
188
189const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
190
191pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
199
200pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
215
216pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
234
235const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
236const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
237
238fn reload_verdict(
239 configured: &Path,
240 spawned_from: Option<&Path>,
241 image: subc_control::RunningImageAgreement,
242) -> PendingReloadVerdict {
243 let path = match spawned_from {
244 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
245 ReloadPathAgreement::Match
246 }
247 Some(spawned_from) => ReloadPathAgreement::Mismatch {
248 configured: configured.to_path_buf(),
249 spawned_from: spawned_from.to_path_buf(),
250 },
251 None => ReloadPathAgreement::Unavailable {
252 reason: if matches!(
253 image,
254 subc_control::RunningImageAgreement::Unavailable {
255 reason: subc_control::RunningImageUnavailableReason::NotRunning
256 }
257 ) {
258 ReloadPathUnavailableReason::NotRunning
259 } else {
260 ReloadPathUnavailableReason::SpawnedPathUnavailable
261 },
262 },
263 };
264 PendingReloadVerdict::new(path, image)
265}
266
267fn launch_spec_agreement(
268 configured: &crate::supervise::ModuleSpec,
269 running: bool,
270 spawned: Option<&crate::supervise::LaunchSpecDigest>,
271) -> subc_control::LaunchSpecAgreement {
272 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
273 if !running {
274 return Agreement::Unavailable {
275 reason: Reason::NotRunning,
276 };
277 }
278 match spawned {
279 None => Agreement::Unavailable {
280 reason: Reason::SpawnedSpecUnavailable,
281 },
282 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
283 Agreement::Match
284 }
285 Some(_) => Agreement::Mismatch,
286 }
287}
288
289fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
290 if configured == spawned_from {
291 return true;
292 }
293 #[cfg(windows)]
294 {
295 match (
301 std::fs::canonicalize(configured),
302 std::fs::canonicalize(spawned_from),
303 ) {
304 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
305 _ => false,
306 }
307 }
308 #[cfg(not(windows))]
309 {
310 false
311 }
312}
313
314#[derive(Clone)]
315struct DaemonProvenanceFacts {
316 build: DaemonBuildProvenance,
317 pid: Option<u32>,
318 started_at_ms: Option<u64>,
319 start_clock: Option<crate::clock::StartClock>,
320 executable_path: Option<PathBuf>,
321 executable_identity: Option<SpawnedFileIdentity>,
322 process_start_time: Option<u64>,
323 probe: ExecutableIdentityProbe,
324}
325
326impl Default for DaemonProvenanceFacts {
327 fn default() -> Self {
328 Self {
329 build: DaemonBuildProvenance {
330 build_git_sha: None,
331 build_lock_digest: None,
332 },
333 pid: None,
334 started_at_ms: None,
335 start_clock: None,
336 executable_path: None,
337 executable_identity: None,
338 process_start_time: None,
339 probe: ExecutableIdentityProbe::default(),
340 }
341 }
342}
343
344#[derive(Debug, Clone)]
345struct SupervisorRescanContext {
346 supervisor: Supervisor,
347 config_path: PathBuf,
348 configured_port: Option<u16>,
349 storage_config: Option<crate::daemon_config::StorageConfig>,
350 admission_facts_carrier_module_id: Option<String>,
351 admission_facts_targets: Option<Vec<String>>,
352 scope_authority_owners: Vec<String>,
353 federated_scope_owner: String,
354 privileged_operator_prompt: PrivilegedOperatorPrompt,
355 generation: Arc<std::sync::atomic::AtomicU64>,
361}
362
363const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
384 "reloading",
385 "supervisor_not_live",
386 "registration_not_active",
387 "no_forwarding_connection",
388 "relay_send_failed",
389];
390
391#[cfg(test)]
393struct ActivityRequestPause {
394 connection: ConnectionId,
395 started: tokio::sync::Notify,
396 cancelled: tokio::sync::Notify,
397 cleanup_started: std::sync::atomic::AtomicBool,
398 cancelled_after_cleanup: std::sync::atomic::AtomicBool,
399}
400
401#[cfg(test)]
402struct ActivityRequestCancellation(Arc<ActivityRequestPause>);
403
404#[cfg(test)]
405impl Drop for ActivityRequestCancellation {
406 fn drop(&mut self) {
407 self.0.cancelled_after_cleanup.store(
408 self.0
409 .cleanup_started
410 .load(std::sync::atomic::Ordering::Acquire),
411 std::sync::atomic::Ordering::Release,
412 );
413 self.0.cancelled.notify_one();
414 }
415}
416
417#[cfg(test)]
419type TestPause = (
420 tokio::sync::oneshot::Sender<()>,
421 std::sync::mpsc::Receiver<()>,
422);
423
424#[cfg(test)]
426type TestHook = Box<dyn FnOnce() + Send>;
427
428#[cfg(test)]
430type AsyncTestPause = (
431 tokio::sync::oneshot::Sender<()>,
432 tokio::sync::oneshot::Receiver<()>,
433);
434
435#[derive(Clone)]
437pub struct ControlHandler {
438 #[cfg(test)]
439 activity_request_pause: Arc<Mutex<Option<Arc<ActivityRequestPause>>>>,
440 #[cfg(test)]
444 cleanup_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
445 #[cfg(test)]
448 after_nonce_attestation: Arc<Mutex<Option<TestHook>>>,
449 #[cfg(test)]
452 hello_ingress_pause: Arc<Mutex<Option<AsyncTestPause>>>,
453 #[cfg(test)]
455 agent_commands_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
456 claims: Arc<claims::Claims>,
457 registry: Arc<Registry>,
458 config_runtime: Option<Arc<crate::config_gate::runtime::RuntimeOwner>>,
459 config_review_admission: Arc<dyn crate::config_gate::ReviewAdmission>,
460 config_review_admission_is_default: bool,
461 activity: Option<crate::activity::ActivityLedger>,
462 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
463 catalog_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
464 forwarding: Arc<ForwardingTable>,
465 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
466 supervisor: SupervisorHandle,
467 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
468 agent_declarations: agent_commands::Queues,
469 list_admission: Arc<tokio::sync::Semaphore>,
470 subc_capabilities: Arc<[String]>,
471 route_bind_relay_timeout: Duration,
475 route_bind_relay_timeouts: BTreeMap<String, Duration>,
479 route_bind_breakers: RouteBindBreakers,
482 route_bind_concurrency: RouteBindConcurrency,
485 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
490 route_bind_breaker_threshold: u32,
492 route_bind_breaker_cooldown: Duration,
494 health_probe_timeout: Duration,
495 storage_config: Option<crate::daemon_config::StorageConfig>,
498 machine_id: Option<crate::machine_id::MachineId>,
502 admission_facts_carrier_module_id: Option<String>,
503 admission_facts_targets: Option<Vec<String>>,
504 scopes: Arc<RwLock<ScopeTable>>,
508 scope_deadline_added: Arc<tokio::sync::Notify>,
512 scope_authority_owners: Vec<String>,
515 federated_scope_owner: String,
516 privileged_operator_prompt: PrivilegedOperatorPrompt,
518 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
521 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
522 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
523 captured_peers: Arc<Mutex<HashMap<ConnectionId, subc_os::local_peer::CapturedPeer>>>,
524 config_desktop_stamps:
525 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
526 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
527 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
528 pub(crate) operator_apps:
529 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
530 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
535 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
541 #[cfg(test)]
543 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
544 #[cfg(test)]
545 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
546 #[cfg(test)]
547 recovery_exit_test_after_step: Option<fn(usize) -> Result<(), crate::config_gate::GateError>>,
548 #[cfg(test)]
549 recovery_exit_test_completed: Option<fn(u64)>,
550 #[cfg(test)]
551 apply_reconcile_pause: Arc<Mutex<Option<AsyncTestPause>>>,
552 #[cfg(test)]
553 recovery_exit_test_quiet_retry: bool,
554 #[cfg(test)]
555 recovery_exit_test_retry_pause: Option<fn(u64) -> Option<PathBuf>>,
556 #[cfg(test)]
557 config_delivery_test_wakes:
558 Option<tokio::sync::mpsc::UnboundedSender<(&'static str, tokio::time::Instant)>>,
559 #[cfg(all(test, unix))]
560 config_delivery_test_idle:
561 Option<tokio::sync::mpsc::UnboundedSender<tokio::sync::oneshot::Sender<()>>>,
562 #[allow(dead_code)]
566 #[cfg(windows)]
567 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
568 #[allow(dead_code)]
569 #[cfg(windows)]
570 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
571 #[allow(dead_code)]
575 #[cfg(windows)]
576 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
577 #[cfg(all(test, windows))]
580 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
581 #[cfg(test)]
582 caller_lookup: Option<TestCallerLookup>,
583 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
586 rescan: Option<SupervisorRescanContext>,
587 connected_clients: ConnectedClients,
588 counters: DaemonCounters,
589 local_endpoint_unavailable: Option<String>,
592 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
593 daemon_shutdown: DaemonShutdownFlag,
598 daemon_provenance: DaemonProvenanceFacts,
599 #[cfg(test)]
600 control_dispatch_delay: Option<Duration>,
601 #[cfg(test)]
602 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
603 #[cfg(test)]
607 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
608 #[cfg(test)]
614 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
615 #[cfg(test)]
616 hello_ack_to_publication_hold: Option<Arc<std::sync::Barrier>>,
617 #[cfg(test)]
618 hello_before_publication_hold: Option<Arc<std::sync::Barrier>>,
619 #[cfg(test)]
620 hello_after_launch_check_hold: Option<Arc<std::sync::Barrier>>,
621 #[cfg(test)]
622 hello_inside_publication_hold: Option<Arc<std::sync::Barrier>>,
623}
624
625impl fmt::Debug for ControlHandler {
626 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
627 f.debug_struct("ControlHandler")
628 .field("registry", &self.registry)
629 .field("forwarding", &self.forwarding)
630 .field("process_liveness", &self.process_liveness.is_some())
631 .field("supervisor", &self.supervisor)
632 .field("subc_capabilities", &self.subc_capabilities)
633 .finish()
634 }
635}
636
637struct RouteOpenRequest {
638 target: RouteTarget,
639 identity: BindIdentity,
640 consumer_identity: Option<ConsumerIdentity>,
641 consumer_capabilities: Option<Vec<String>>,
642 role_versions: Option<BTreeMap<String, String>>,
643 busy_refusals: Option<bool>,
644 admission_facts: Option<serde_json::Value>,
645 scope: Option<ScopeSelector>,
646 accepts_close_reasons: Option<Vec<String>>,
647}
648
649#[derive(Clone, Copy)]
650enum RouteOpenPhase {
651 Admission,
652 ProjectRoot,
653 EgressReservation,
654 BindSend,
655 BindAckWait,
656}
657
658struct RouteOpenTimings {
662 module_id: String,
663 principal: String,
664 admission: Duration,
665 project_root: Duration,
666 egress_reservation: Duration,
667 bind_send: Duration,
668 bind_ack_wait: Duration,
669 phase: Option<RouteOpenPhase>,
670 phase_started_at: Instant,
671}
672
673impl RouteOpenTimings {
674 fn new() -> Self {
675 Self {
676 module_id: String::new(),
677 principal: "unresolved".to_string(),
679 admission: Duration::ZERO,
680 project_root: Duration::ZERO,
681 egress_reservation: Duration::ZERO,
682 bind_send: Duration::ZERO,
683 bind_ack_wait: Duration::ZERO,
684 phase: Some(RouteOpenPhase::Admission),
685 phase_started_at: Instant::now(),
686 }
687 }
688
689 fn finish(&mut self) {
690 let elapsed = self.phase_started_at.elapsed();
691 match self.phase.take() {
692 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
693 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
694 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
695 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
696 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
697 None => {}
698 }
699 }
700
701 fn enter(&mut self, phase: RouteOpenPhase) {
702 self.finish();
703 self.phase = Some(phase);
704 self.phase_started_at = Instant::now();
705 }
706}
707
708struct RouteBindReservationGuard {
709 forwarding: Arc<ForwardingTable>,
710 endpoint: ModuleEndpointId,
711 relay_corr: u64,
712 armed: bool,
713}
714
715struct ModuleControlRpcGuard {
716 forwarding: Arc<ForwardingTable>,
717 endpoint: ModuleEndpointId,
718 corr: u64,
719 armed: bool,
720}
721
722impl ModuleControlRpcGuard {
723 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
724 Self {
725 forwarding,
726 endpoint,
727 corr,
728 armed: true,
729 }
730 }
731
732 fn disarm(&mut self) {
733 self.armed = false;
734 }
735}
736
737impl Drop for ModuleControlRpcGuard {
738 fn drop(&mut self) {
739 if self.armed {
740 let _ = self
741 .forwarding
742 .cancel_module_control_rpc(self.endpoint, self.corr);
743 }
744 }
745}
746
747impl RouteBindReservationGuard {
748 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
749 Self {
750 forwarding,
751 endpoint,
752 relay_corr,
753 armed: true,
754 }
755 }
756
757 fn release_and_disarm(&mut self) {
758 if !self.armed {
759 return;
760 }
761 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
762 self.endpoint,
763 self.relay_corr,
764 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
765 ) {
766 send_goodbye_target_best_effort(
767 &self.forwarding.counters(),
768 &target,
769 "canceled route.bind",
770 );
771 }
772 self.armed = false;
773 }
774
775 fn disarm(&mut self) {
776 self.armed = false;
777 }
778}
779
780impl Drop for RouteBindReservationGuard {
781 fn drop(&mut self) {
782 self.release_and_disarm();
783 }
784}
785
786#[derive(Debug, Clone, Default)]
816pub(crate) struct RouteBindBreakers {
817 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
818}
819
820#[derive(Debug, Clone, Default)]
821pub(crate) struct RouteBindConcurrency {
822 modules: Arc<Mutex<HashMap<String, usize>>>,
823}
824
825struct RouteBindConcurrencyGuard {
826 concurrency: RouteBindConcurrency,
827 module_id: String,
828}
829
830impl RouteBindConcurrency {
831 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
834 let mut modules = self
835 .modules
836 .lock()
837 .expect("route.bind concurrency mutex poisoned");
838 let in_flight = modules.entry(module_id.to_string()).or_default();
839 if *in_flight >= limit {
840 return Err(*in_flight);
841 }
842 *in_flight += 1;
843 Ok(RouteBindConcurrencyGuard {
844 concurrency: self.clone(),
845 module_id: module_id.to_string(),
846 })
847 }
848}
849
850impl Drop for RouteBindConcurrencyGuard {
851 fn drop(&mut self) {
852 let mut modules = self
853 .concurrency
854 .modules
855 .lock()
856 .expect("route.bind concurrency mutex poisoned");
857 let remove = {
858 let in_flight = modules
859 .get_mut(&self.module_id)
860 .expect("admitted route.bind has a concurrency entry");
861 *in_flight -= 1;
862 *in_flight == 0
863 };
864 if remove {
865 modules.remove(&self.module_id);
866 }
867 }
868}
869
870#[derive(Debug, Default)]
871struct ModuleBreakerState {
872 consecutive_timeouts: u32,
874 cooldown_until: Option<Instant>,
877 probe_in_flight: Option<Arc<()>>,
882}
883
884enum RouteBindAdmission<'a> {
886 Admitted {
887 guard: RouteBindBreakerGuard<'a>,
888 probe: bool,
891 },
892 Refused {
893 consecutive_timeouts: u32,
894 retry_in: Duration,
898 probe_in_flight: bool,
899 },
900}
901
902struct RouteBindBreakerGuard<'a> {
909 breakers: RouteBindBreakers,
910 module_id: &'a str,
911 probe_token: Option<Arc<()>>,
912 settled: bool,
913}
914
915impl RouteBindBreakerGuard<'_> {
916 fn record_accepted(&mut self) -> bool {
920 self.settled = true;
921 self.breakers.record_accepted(self.module_id)
922 }
923
924 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
927 self.settled = true;
928 self.breakers.record_timeout(
929 self.module_id,
930 self.probe_token.as_ref(),
931 threshold,
932 cooldown,
933 )
934 }
935
936 fn record_inconclusive(&mut self) {
945 self.settled = true;
946 self.breakers
947 .record_inconclusive(self.module_id, self.probe_token.as_ref());
948 }
949}
950
951impl Drop for RouteBindBreakerGuard<'_> {
952 fn drop(&mut self) {
953 if !self.settled {
954 self.breakers
955 .record_inconclusive(self.module_id, self.probe_token.as_ref());
956 }
957 }
958}
959
960struct BreakerOpened {
964 consecutive_timeouts: u32,
965 reopened_after_probe: bool,
968}
969
970impl RouteBindBreakers {
971 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
972 self.modules
973 .lock()
974 .expect("route.bind breaker mutex poisoned")
975 }
976
977 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
980 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
981 probe: probe_token.is_some(),
982 guard: RouteBindBreakerGuard {
983 breakers: self.clone(),
984 module_id,
985 probe_token,
986 settled: false,
987 },
988 };
989
990 let mut modules = self.lock();
991 let Some(state) = modules.get_mut(module_id) else {
992 return admitted(None);
993 };
994 let Some(cooldown_until) = state.cooldown_until else {
995 return admitted(None);
996 };
997 if state.probe_in_flight.is_some() {
998 return RouteBindAdmission::Refused {
999 consecutive_timeouts: state.consecutive_timeouts,
1000 retry_in: Duration::ZERO,
1001 probe_in_flight: true,
1002 };
1003 }
1004 let now = Instant::now();
1005 if now < cooldown_until {
1006 return RouteBindAdmission::Refused {
1007 consecutive_timeouts: state.consecutive_timeouts,
1008 retry_in: cooldown_until - now,
1009 probe_in_flight: false,
1010 };
1011 }
1012 let token = Arc::new(());
1013 state.probe_in_flight = Some(Arc::clone(&token));
1014 admitted(Some(token))
1015 }
1016
1017 fn record_accepted(&self, module_id: &str) -> bool {
1018 self.lock()
1019 .remove(module_id)
1020 .is_some_and(|state| state.cooldown_until.is_some())
1021 }
1022
1023 fn record_timeout(
1024 &self,
1025 module_id: &str,
1026 probe_token: Option<&Arc<()>>,
1027 threshold: u32,
1028 cooldown: Duration,
1029 ) -> Option<BreakerOpened> {
1030 let mut modules = self.lock();
1031 let state = modules.entry(module_id.to_string()).or_default();
1032 let was_open = state.cooldown_until.is_some();
1033 let was_probe = Self::owns_probe(state, probe_token);
1034 if was_probe {
1035 state.probe_in_flight = None;
1036 }
1037 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
1038 if state.consecutive_timeouts < threshold {
1039 return None;
1040 }
1041 state.cooldown_until = Some(Instant::now() + cooldown);
1042 Some(BreakerOpened {
1043 consecutive_timeouts: state.consecutive_timeouts,
1044 reopened_after_probe: was_open && was_probe,
1045 })
1046 }
1047
1048 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
1049 state
1053 .probe_in_flight
1054 .as_ref()
1055 .zip(token)
1056 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
1057 }
1058
1059 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
1060 if let Some(state) = self.lock().get_mut(module_id) {
1061 if Self::owns_probe(state, probe_token) {
1062 state.probe_in_flight = None;
1063 }
1064 }
1065 }
1066
1067 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
1082 self.lock()
1083 .remove(module_id)
1084 .map(|state| state.consecutive_timeouts)
1085 .filter(|discarded| *discarded > 0)
1086 }
1087
1088 fn open_snapshot(&self) -> Option<serde_json::Value> {
1095 let now = Instant::now();
1096 let modules = self.lock();
1097 let open = modules
1098 .iter()
1099 .filter_map(|(module_id, state)| {
1100 let cooldown_until = state.cooldown_until?;
1101 Some((
1102 module_id.clone(),
1103 serde_json::json!({
1104 "consecutive_timeouts": state.consecutive_timeouts,
1105 "cooldown_remaining_ms":
1106 cooldown_until.saturating_duration_since(now).as_millis() as u64,
1107 "probe_in_flight": state.probe_in_flight.is_some(),
1108 }),
1109 ))
1110 })
1111 .collect::<serde_json::Map<String, serde_json::Value>>();
1112 (!open.is_empty()).then_some(serde_json::Value::Object(open))
1113 }
1114}
1115
1116impl ControlHandler {
1117 pub(crate) fn install_registration_service(
1118 &self,
1119 router: std::sync::Weak<crate::Router>,
1120 auth: crate::server::ServerAuth,
1121 ) {
1122 self.supervisor.install_registration_service(router, auth);
1123 }
1124 pub fn new(registry: Arc<Registry>) -> Self {
1125 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
1126 }
1127
1128 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
1129 let counters = forwarding.counters();
1130 let route_bind_breakers = forwarding.route_bind_breakers();
1134 let route_bind_concurrency = forwarding.route_bind_concurrency();
1135 let route_outages = forwarding.route_outages();
1136 Self {
1137 registry,
1138 #[cfg(test)]
1139 activity_request_pause: Arc::new(Mutex::new(None)),
1140 #[cfg(test)]
1141 cleanup_pause: Arc::new(Mutex::new(None)),
1142 #[cfg(test)]
1143 after_nonce_attestation: Arc::new(Mutex::new(None)),
1144 #[cfg(test)]
1145 hello_ingress_pause: Arc::new(Mutex::new(None)),
1146 #[cfg(test)]
1147 agent_commands_pause: Arc::new(Mutex::new(None)),
1148 claims: Arc::new(claims::Claims::default()),
1149 config_runtime: None,
1150 config_review_admission: Arc::new(crate::config_gate::MissingReviewAdmission),
1151 config_review_admission_is_default: true,
1152 activity: None,
1153 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1154 catalog_streams: Arc::new(Mutex::new(HashMap::new())),
1155 forwarding,
1156 process_liveness: None,
1157 supervisor: SupervisorHandle::new(),
1158 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1159 env!("CARGO_PKG_VERSION").to_owned(),
1160 )),
1161 agent_declarations: agent_commands::Queues::default(),
1162 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1163 subc_capabilities: Arc::from([
1164 CAP_MANIFEST_REGISTRATION.to_string(),
1165 CAP_CHANNEL_LIFECYCLE.to_string(),
1166 CAP_PING_PONG.to_string(),
1167 CAP_SESSION_ATTACH.to_string(),
1168 CAP_ADMISSION_FACTS_RELAY.to_string(),
1169 CAP_SCOPES_V1.to_string(),
1170 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1171 subc_protocol::peer::CALLER_ATTESTATION_CAPABILITY.to_string(),
1172 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1173 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1174 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1175 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1176 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1177 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1178 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1179 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1180 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1181 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1182 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1183 subc_protocol::ROUTE_CLOSE_REASONS_CAPABILITY.to_string(),
1184 ]),
1185 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1186 route_bind_relay_timeouts: BTreeMap::new(),
1187 route_bind_breakers,
1188 route_bind_concurrency,
1189 route_outages,
1190 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1191 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1192 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1193 storage_config: None,
1194 machine_id: None,
1195 admission_facts_carrier_module_id: None,
1196 admission_facts_targets: None,
1197 scopes: Arc::new(RwLock::new(ScopeTable::new(
1198 crate::daemon_config::default_scope_authority_owners(),
1199 ))),
1200 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1201 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1202 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1203 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1204 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1205 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1206 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1207 captured_peers: Arc::new(Mutex::new(HashMap::new())),
1208 config_desktop_stamps: Arc::new(Mutex::new(HashMap::new())),
1209 local_connections: Arc::new(Mutex::new(HashSet::new())),
1210 operator_app_policy: Default::default(),
1211 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1212 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1213 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1214 #[cfg(test)]
1215 app_open_test_platform: None,
1216 #[cfg(test)]
1217 app_open_test_launch: None,
1218 #[cfg(test)]
1219 recovery_exit_test_after_step: None,
1220 #[cfg(test)]
1221 recovery_exit_test_completed: None,
1222 #[cfg(test)]
1223 apply_reconcile_pause: Arc::new(Mutex::new(None)),
1224 #[cfg(test)]
1225 recovery_exit_test_quiet_retry: false,
1226 #[cfg(test)]
1227 recovery_exit_test_retry_pause: None,
1228 #[cfg(test)]
1229 config_delivery_test_wakes: None,
1230 #[cfg(all(test, unix))]
1231 config_delivery_test_idle: None,
1232 #[cfg(windows)]
1233 windows_verify: None,
1234 #[cfg(windows)]
1235 windows_flash: None,
1236 #[cfg(windows)]
1237 windows_track: None,
1238 #[cfg(all(test, windows))]
1239 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1240 #[cfg(test)]
1241 caller_lookup: None,
1242 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1243 rescan: None,
1244 connected_clients: ConnectedClients::new(),
1245 local_endpoint_unavailable: None,
1246 counters,
1247 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1248 daemon_shutdown: DaemonShutdownFlag::default(),
1249 daemon_provenance: DaemonProvenanceFacts::default(),
1250 #[cfg(test)]
1251 control_dispatch_delay: None,
1252 #[cfg(test)]
1253 provenance_probe_override: None,
1254 #[cfg(test)]
1255 route_bind_relay_sent_hold: None,
1256 #[cfg(test)]
1257 hello_registry_to_forwarding_hold: None,
1258 #[cfg(test)]
1259 hello_ack_to_publication_hold: None,
1260 #[cfg(test)]
1261 hello_before_publication_hold: None,
1262 #[cfg(test)]
1263 hello_after_launch_check_hold: None,
1264 #[cfg(test)]
1265 hello_inside_publication_hold: None,
1266 }
1267 }
1268
1269 pub fn with_storage_config(
1272 mut self,
1273 storage_config: Option<crate::daemon_config::StorageConfig>,
1274 ) -> Self {
1275 self.storage_config = storage_config;
1276 self
1277 }
1278
1279 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1282 self.activity = Some(crate::activity::ActivityLedger::open(
1283 run_dir.join("activity.jsonl"),
1284 incarnation,
1285 ));
1286 self.supervisor.set_activity(self.activity.clone());
1287 self
1288 }
1289
1290 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1293 self.machine_id = machine_id;
1294 self
1295 }
1296
1297 pub fn with_admission_facts_config(
1301 mut self,
1302 carrier_module_id: Option<String>,
1303 targets: Option<Vec<String>>,
1304 ) -> Self {
1305 self.admission_facts_carrier_module_id = carrier_module_id;
1306 self.admission_facts_targets = targets;
1307 self
1308 }
1309
1310 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1314 self.scopes = Arc::new(RwLock::new(
1315 ScopeTable::new(owners.iter().cloned())
1316 .with_federated_owner(self.federated_scope_owner.clone()),
1317 ));
1318 self.scope_authority_owners = owners;
1319 self
1320 }
1321
1322 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1324 self.scopes = Arc::new(RwLock::new(
1325 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1326 .with_federated_owner(owner.clone()),
1327 ));
1328 self.federated_scope_owner = owner;
1329 self
1330 }
1331
1332 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1334 self.privileged_operator_prompt = mode;
1335 self
1336 }
1337
1338 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1341 self.route_bind_relay_timeout = timeout;
1342 self
1343 }
1344
1345 pub fn with_route_bind_relay_timeouts(
1351 mut self,
1352 timeouts: impl IntoIterator<Item = (String, Duration)>,
1353 ) -> Self {
1354 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1355 self
1356 }
1357
1358 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1364 self.route_bind_relay_timeouts
1365 .get(module_id)
1366 .copied()
1367 .unwrap_or(self.route_bind_relay_timeout)
1368 }
1369
1370 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1378 self.route_bind_breaker_threshold = threshold.max(1);
1379 self.route_bind_breaker_cooldown = cooldown;
1380 self
1381 }
1382
1383 #[cfg(test)]
1384 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1385 self.health_probe_timeout = timeout;
1386 self
1387 }
1388
1389 #[cfg(test)]
1390 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1391 self.control_dispatch_delay = Some(delay);
1392 self
1393 }
1394
1395 #[cfg(test)]
1399 pub(crate) fn with_route_bind_relay_sent_hold(
1400 mut self,
1401 hold: Arc<tokio::sync::Barrier>,
1402 ) -> Self {
1403 self.route_bind_relay_sent_hold = Some(hold);
1404 self
1405 }
1406
1407 #[cfg(test)]
1412 pub(crate) fn with_hello_registry_to_forwarding_hold(
1413 mut self,
1414 hold: Arc<std::sync::Barrier>,
1415 ) -> Self {
1416 self.hello_registry_to_forwarding_hold = Some(hold);
1417 self
1418 }
1419
1420 #[cfg(test)]
1421 fn hold_between_registry_and_forwarding(&self) {
1422 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1423 hold.wait();
1424 hold.wait();
1425 }
1426 }
1427
1428 #[cfg(test)]
1431 fn hold_between_ack_and_publication(&self) {
1432 if let Some(hold) = &self.hello_ack_to_publication_hold {
1433 hold.wait();
1434 hold.wait();
1435 }
1436 }
1437
1438 #[cfg(all(test, unix))]
1439 pub(crate) fn with_hello_before_publication_hold(
1440 mut self,
1441 hold: Arc<std::sync::Barrier>,
1442 ) -> Self {
1443 self.hello_before_publication_hold = Some(hold);
1444 self
1445 }
1446
1447 #[cfg(all(test, unix))]
1448 pub(crate) fn with_hello_after_launch_check_hold(
1449 mut self,
1450 hold: Arc<std::sync::Barrier>,
1451 ) -> Self {
1452 self.hello_after_launch_check_hold = Some(hold);
1453 self
1454 }
1455
1456 #[cfg(all(test, unix))]
1457 pub(crate) fn with_hello_inside_publication_hold(
1458 mut self,
1459 hold: Arc<std::sync::Barrier>,
1460 ) -> Self {
1461 self.hello_inside_publication_hold = Some(hold);
1462 self
1463 }
1464
1465 pub fn with_process_liveness(
1466 mut self,
1467 process_liveness: Arc<dyn ModuleProcessLiveness>,
1468 ) -> Self {
1469 self.process_liveness = Some(process_liveness);
1470 self
1471 }
1472
1473 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1474 self.supervisor = supervisor;
1475 if let Some(owner) = &self.config_runtime {
1476 self.supervisor.install_binding_retirement(owner);
1477 }
1478 self.supervisor.set_activity(self.activity.clone());
1479 self
1480 }
1481
1482 pub fn with_daemon_provenance(
1483 mut self,
1484 pid: u32,
1485 started_at_ms: u64,
1486 executable_path: Option<PathBuf>,
1487 build_git_sha: Option<String>,
1488 build_lock_digest: Option<String>,
1489 ) -> Self {
1490 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1491 let process_start_time = process_start_time(pid);
1492 self.daemon_provenance = DaemonProvenanceFacts {
1493 build: DaemonBuildProvenance {
1494 build_git_sha,
1495 build_lock_digest,
1496 },
1497 pid: Some(pid),
1498 started_at_ms: Some(started_at_ms),
1499 start_clock: None,
1500 executable_path,
1501 executable_identity,
1502 process_start_time,
1503 probe: ExecutableIdentityProbe::default(),
1504 };
1505 self
1506 }
1507
1508 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1509 self.daemon_provenance.start_clock = Some(clock);
1510 self
1511 }
1512
1513 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1515 self.scopes
1516 .write()
1517 .unwrap_or_else(|poisoned| poisoned.into_inner())
1518 .set_wall_clock(Arc::new(clock));
1519 self
1520 }
1521
1522 #[cfg(test)]
1523 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1524 self.provenance_probe_override = Some(result);
1525 self
1526 }
1527
1528 pub fn with_capability_config(
1532 self,
1533 modules: impl IntoIterator<Item = (String, bool)>,
1534 reserved_capabilities: BTreeMap<String, String>,
1535 ) -> Self {
1536 self.capability_evaluator
1537 .configure(modules, reserved_capabilities);
1538 self
1539 }
1540
1541 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1545 self.daemon_shutdown = flag;
1546 self
1547 }
1548
1549 pub fn with_supervisor_rescan(
1550 mut self,
1551 supervisor: Supervisor,
1552 config_path: impl Into<PathBuf>,
1553 configured_port: Option<u16>,
1554 ) -> Self {
1555 self.rescan = Some(SupervisorRescanContext {
1556 supervisor,
1557 config_path: config_path.into(),
1558 configured_port,
1559 storage_config: self.storage_config.clone(),
1560 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1561 admission_facts_targets: self.admission_facts_targets.clone(),
1562 scope_authority_owners: self.scope_authority_owners.clone(),
1563 federated_scope_owner: self.federated_scope_owner.clone(),
1564 privileged_operator_prompt: self.privileged_operator_prompt,
1565 generation: Arc::default(),
1566 });
1567 self
1568 }
1569
1570 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1571 self.connected_clients = connected_clients;
1572 self
1573 }
1574
1575 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1578 self.local_endpoint_unavailable = reason;
1579 self
1580 }
1581
1582 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1583 Arc::clone(&self.forwarding)
1584 }
1585
1586 pub(crate) fn counters(&self) -> DaemonCounters {
1587 self.counters.clone()
1588 }
1589
1590 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1593 let handler = Arc::downgrade(&self);
1594 let evaluator = self.capability_evaluator.clone();
1595 tokio::spawn(async move {
1596 loop {
1597 evaluator.wait_for_change_or_deadline().await;
1598 let Some(handler) = handler.upgrade() else {
1599 break;
1600 };
1601 handler.refresh_capability_requirements();
1602 }
1603 });
1604 }
1605
1606 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1612 let context = scope_expiry::ScopeExpiryContext::new(&self);
1613 let deadline_added = self.scope_deadline_added.clone();
1614 tokio::spawn(async move {
1615 loop {
1616 if !context.has_deadlines() {
1617 deadline_added.notified().await;
1618 continue;
1619 }
1620 tokio::time::sleep(Duration::from_secs(1)).await;
1621 if let Err(error) = context.sweep() {
1622 warn!(%error, "scope expiry sweep failed");
1623 }
1624 }
1625 });
1626 }
1627
1628 #[cfg(test)]
1629 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1630 scope_expiry::ScopeExpiryContext::new(self).sweep()
1631 }
1632
1633 fn publish_scope_changes(
1634 &self,
1635 changes: &[crate::scopes::ScopeTagChange],
1636 expired: &[crate::scopes::ScopeExpired],
1637 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1638 scope_expiry::ScopeExpiryContext::new(self).publish_scope_changes(changes, expired)
1639 }
1640
1641 fn runtime_capability_snapshot(
1642 &self,
1643 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1644 let runtime = self
1645 .supervisor
1646 .list()
1647 .into_iter()
1648 .map(|module| {
1649 let status = module.status().map_err(|err| {
1650 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1651 })?;
1652 Ok(RuntimeModule {
1653 module_id: status.module_id,
1654 state: status.state,
1655 enabled: status.enabled,
1656 })
1657 })
1658 .collect::<Result<Vec<_>, RouterError>>()?;
1659 let registrations = self.registry.list_registered_modules().map_err(|err| {
1665 RouterError::backend(
1666 0,
1667 0,
1668 format!("failed to list capability registrations: {err}"),
1669 )
1670 })?;
1671 let registrations = registrations
1672 .into_iter()
1673 .map(|registration| RegisteredModule {
1674 module_id: registration.manifest.module_id,
1675 module_version: registration.manifest.module_version,
1676 capabilities: registration.manifest.capabilities,
1677 })
1678 .collect();
1679 Ok((runtime, registrations))
1680 }
1681
1682 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1689 let cached_registration = RegisteredModule {
1690 module_id: registration.manifest.module_id.clone(),
1691 module_version: registration.manifest.module_version.clone(),
1692 capabilities: registration.manifest.capabilities.clone(),
1693 };
1694 let change = self.capability_evaluator.record_hello(&cached_registration);
1695 log_manifest_change(&cached_registration.module_id, &change);
1696 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1697 self.enforce_capability_denies();
1698 }
1699 self.refresh_capability_requirements();
1700 }
1701
1702 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1706 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1707 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1708 self.supervisor.set_swap_promotion_observer(observer);
1709 }
1710
1711 pub fn refresh_capability_requirements(&self) {
1712 match self.runtime_capability_snapshot() {
1713 Ok((runtime, registrations)) => {
1714 log_requirement_events(
1715 self.capability_evaluator
1716 .evaluate_now(&runtime, ®istrations),
1717 &self.daemon_shutdown,
1718 );
1719 }
1720 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1721 }
1722 }
1723
1724 fn enforce_capability_denies(&self) {
1728 let registrations = match self.registry.list_registered_modules() {
1729 Ok(snapshot) => snapshot,
1730 Err(err) => {
1731 warn!(error = %err, "failed to read registrations for capability deny census");
1732 return;
1733 }
1734 };
1735 let manifests = registrations
1736 .into_iter()
1737 .map(|registration| {
1738 (
1739 registration.manifest.module_id.clone(),
1740 registration.manifest,
1741 )
1742 })
1743 .collect::<BTreeMap<_, _>>();
1744 let census = match self.forwarding.route_census(None) {
1745 Ok(census) => census,
1746 Err(err) => {
1747 warn!(error = %err, "failed to read route census for capability deny enforcement");
1748 return;
1749 }
1750 };
1751
1752 for (target_module_id, routes) in census {
1753 let Some(target_manifest) = manifests.get(&target_module_id) else {
1754 continue;
1755 };
1756 let mut closed_routes = Vec::new();
1757 let mut goodbyes = Vec::new();
1758 for route in routes {
1759 let Principal::Reserved {
1760 module_id: opening_module_id,
1761 } = &route.principal
1762 else {
1763 continue;
1764 };
1765 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1766 continue;
1767 };
1768 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1769 continue;
1770 };
1771
1772 match self.forwarding.release_client_route(
1773 route.goodbye_target.connection_id,
1774 route.goodbye_target.channel,
1775 route.goodbye_target.epoch,
1776 ) {
1777 Ok(RouteRelease::Removed(module_goodbye)) => {
1778 warn!(
1779 opening_module_id,
1780 target_module_id,
1781 capability,
1782 "force-closing route because an attested capability deny edge now matches"
1783 );
1784 goodbyes.push(module_goodbye);
1785 goodbyes.push(route.goodbye_target.clone());
1786 closed_routes.push(route);
1787 }
1788 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1789 Err(err) => warn!(
1790 opening_module_id,
1791 target_module_id,
1792 capability,
1793 error = %err,
1794 "failed to force-close capability-denied route"
1795 ),
1796 }
1797 }
1798
1799 if closed_routes.is_empty() {
1800 continue;
1801 }
1802 send_route_control_pushes(
1803 &self.forwarding,
1804 closed_routes,
1805 ClientControlPush::RouteClosed {
1806 module_id: target_module_id,
1807 channels: Vec::new(),
1808 reason: RouteCloseReason::CapabilityDenied,
1809 drained: false,
1810 abandoned: 0,
1811 excluded_subscriptions: 0,
1812 terminal: Some(false),
1813 },
1814 );
1815 self.emit_route_goodbyes(goodbyes);
1818 }
1819 }
1820
1821 fn not_ready_reason(
1827 &self,
1828 registration: &crate::registry::ModuleRegistration,
1829 ) -> Option<NotReadyReason> {
1830 if !registration.ready {
1831 return Some(NotReadyReason {
1832 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1833 capability: None,
1834 });
1835 }
1836 self.first_unprovided_required_capability(registration)
1837 .map(|capability| NotReadyReason {
1838 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1839 capability: Some(capability),
1840 })
1841 }
1842
1843 fn first_unprovided_required_capability(
1861 &self,
1862 registration: &crate::registry::ModuleRegistration,
1863 ) -> Option<String> {
1864 let required = registration
1865 .manifest
1866 .capabilities
1867 .iter()
1868 .flat_map(|declarations| declarations.requires.iter())
1869 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1870 .map(|requirement| requirement.capability.as_str())
1871 .collect::<BTreeSet<_>>();
1872 if required.is_empty() {
1873 return None;
1874 }
1875 let module_id = registration.manifest.module_id.as_str();
1876 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1877 if required
1878 .iter()
1879 .any(|capability| verdict(capability).is_none())
1880 {
1881 self.refresh_capability_requirements();
1882 }
1883 required
1884 .into_iter()
1885 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1886 .map(str::to_string)
1887 }
1888
1889 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1890 self.capability_evaluator
1891 .statuses()
1892 .into_iter()
1893 .map(capability_requirement_status)
1894 .collect()
1895 }
1896
1897 fn deregister_connection(
1904 &self,
1905 connection_id: ConnectionId,
1906 reason: RegistrationEndReason,
1907 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1908 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
1909 spawn.end_primary_registration();
1910 }
1911 self.registry
1912 .deregister_connection_with_reason(connection_id, reason)
1913 }
1914
1915 fn rollback_unpublished_hello(
1920 &self,
1921 connection: ConnectionId,
1922 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1923 self.registry.deregister_connection_with_reason(
1924 connection,
1925 RegistrationEndReason::RegistrationFailed,
1926 )
1927 }
1928
1929 fn request_refusal_close(
1930 &self,
1931 connection: ConnectionId,
1932 sink: Option<&crate::FrameSink>,
1933 reason: CloseReason,
1934 ) {
1935 if let Some(sink) = sink {
1936 sink.terminate_ingress();
1937 }
1938 if self.claims.close_notifications_enabled() {
1939 self.forwarding.request_connection_close(connection, reason);
1940 }
1941 }
1942
1943 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1944 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1945 return None;
1946 }
1947 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1948 parse_client_control_request(&frame.body)
1949 else {
1950 return None;
1951 };
1952 Some(target_module_id(&target).to_string())
1953 }
1954
1955 pub(crate) fn route_open_capacity_refusal(
1956 &self,
1957 ctx: &RouteCtx,
1958 frame: &Frame,
1959 target_module_id: &str,
1960 in_flight: usize,
1961 limit: usize,
1962 ) -> Result<Frame, RouterError> {
1963 let caller = self.route_open_caller_from_frame(ctx, frame);
1966 self.route_open_admission_refusal_frame(
1967 ctx,
1968 &caller,
1969 frame,
1970 target_module_id,
1971 "open_admission_full",
1972 (in_flight, limit),
1973 format!(
1974 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1975 ),
1976 )
1977 }
1978
1979 fn route_open_target_capacity_refusal(
1980 &self,
1981 ctx: &RouteCtx,
1982 caller: &RouteOpenCaller,
1983 frame: &Frame,
1984 target_module_id: &str,
1985 in_flight: usize,
1986 ) -> Result<Frame, RouterError> {
1987 self.route_open_admission_refusal_frame(
1988 ctx,
1989 caller,
1990 frame,
1991 target_module_id,
1992 "target_binds_full",
1993 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1994 format!(
1995 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1996 ),
1997 )
1998 }
1999
2000 #[allow(clippy::too_many_arguments)]
2008 fn route_open_admission_refusal_frame(
2009 &self,
2010 ctx: &RouteCtx,
2011 caller: &RouteOpenCaller,
2012 frame: &Frame,
2013 target_module_id: &str,
2014 reason: &'static str,
2015 (in_flight, limit): (usize, usize),
2016 message: impl Into<String>,
2017 ) -> Result<Frame, RouterError> {
2018 let code = error_codes::TARGET_UNAVAILABLE;
2019 self.counters.increment_route_open_refused(code);
2020 info!(
2021 target: "control",
2022 code,
2023 reason,
2024 module_id = ?target_module_id,
2025 connection_id = ctx.connection_id.get(),
2026 principal = %caller.principal,
2027 scope = %caller.scope,
2028 harness = %caller.harness,
2029 in_flight,
2030 limit,
2031 "route.open refused"
2032 );
2033 control_error_frame(frame, code, message.into())
2034 }
2035
2036 #[cfg(test)]
2042 pub fn handle_control(
2043 &self,
2044 connection_id: ConnectionId,
2045 frame: Frame,
2046 ) -> Result<Vec<Frame>, RouterError> {
2047 match frame.header.ty {
2048 FrameType::Ping => Ok(vec![pong(&frame)?]),
2049 FrameType::Hello => self.handle_hello(connection_id, None, frame),
2050 FrameType::Goodbye => self.handle_goodbye(connection_id),
2051 ty => Ok(vec![control_error_frame(
2052 &frame,
2053 "unsupported_control_frame",
2054 format!("unsupported channel-0 frame {ty:?}"),
2055 )?]),
2056 }
2057 }
2058
2059 pub async fn handle_control_frame(
2060 &self,
2061 ctx: &RouteCtx,
2062 frame: Frame,
2063 ) -> Result<Vec<Frame>, RouterError> {
2064 self.handle_control_frame_timed(ctx, frame, None).await
2065 }
2066
2067 #[cfg(any(test, feature = "test-support"))]
2068 pub fn with_test_operator_app_policy(
2069 mut self,
2070 policy: crate::operator_apps::TestOperatorAppPolicy,
2071 ) -> Self {
2072 self.operator_app_policy = policy.into();
2073 self
2074 }
2075
2076 #[cfg(all(windows, any(test, feature = "test-support")))]
2080 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
2081 self.windows_verify = Some(verify);
2082 self
2083 }
2084
2085 #[cfg(all(windows, any(test, feature = "test-support")))]
2087 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
2088 self.windows_flash = Some(flash);
2089 self
2090 }
2091
2092 #[cfg(all(windows, any(test, feature = "test-support")))]
2096 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
2097 self.windows_track = Some(track);
2098 self
2099 }
2100
2101 pub(crate) fn record_captured_peer(
2102 &self,
2103 connection: ConnectionId,
2104 peer: subc_os::local_peer::CapturedPeer,
2105 ) {
2106 self.record_observed_peer(connection, peer.observation().clone());
2107 let identity = self.operator_app_policy.config_identity(peer.clone());
2108 #[cfg(windows)]
2109 let identity = identity.or_else(|| self.config_windows_launch_identity(&peer));
2110 if let Some(identity) = identity {
2111 self.config_desktop_stamps
2114 .lock()
2115 .unwrap_or_else(|p| p.into_inner())
2116 .insert(connection, identity);
2117 }
2118 self.captured_peers
2119 .lock()
2120 .unwrap_or_else(|p| p.into_inner())
2121 .insert(connection, peer.clone());
2122 #[cfg(windows)]
2123 if let Some(stamp) = self.stamp_windows_launch(&peer) {
2124 self.operator_apps
2125 .lock()
2126 .unwrap_or_else(|p| p.into_inner())
2127 .insert(connection, stamp);
2128 }
2129 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
2130 self.operator_apps
2131 .lock()
2132 .unwrap_or_else(|p| p.into_inner())
2133 .insert(connection, stamp);
2134 }
2135 }
2136
2137 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
2138 self.local_connections
2139 .lock()
2140 .unwrap_or_else(|p| p.into_inner())
2141 .insert(connection);
2142 }
2143
2144 fn connection_operator_app(
2147 &self,
2148 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
2149 ) -> Option<subc_protocol::peer::OperatorApp> {
2150 stamp
2151 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
2152 .map(|stamp| stamp.app.clone())
2153 }
2154
2155 fn handle_connection_describe(
2156 &self,
2157 ctx: &RouteCtx,
2158 frame: Frame,
2159 ) -> Result<Vec<Frame>, RouterError> {
2160 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
2161 return Ok(vec![control_error_frame(
2162 &frame,
2163 "invalid_control_body",
2164 "malformed connection.describe body",
2165 )?]);
2166 }
2167 let local = self
2168 .local_connections
2169 .lock()
2170 .unwrap_or_else(|p| p.into_inner())
2171 .contains(&ctx.connection_id);
2172 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
2173 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
2174 Ok(vec![control_response_body_frame(
2175 &frame,
2176 &subc_control::ConnectionDescribeReply::new(local, operator_app),
2177 ops::CONNECTION_DESCRIBE,
2178 )?])
2179 }
2180
2181 pub(crate) fn record_observed_peer(
2182 &self,
2183 connection: ConnectionId,
2184 peer: subc_os::local_peer::ObservedPeer,
2185 ) {
2186 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
2187 let peer = match peer {
2188 OsPeer::Unobserved => ObservedPeer::unobserved(),
2189 OsPeer::Observed(peer) => {
2190 let observed = match peer.user {
2191 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
2192 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
2193 };
2194 observed
2195 .with_exe(
2196 peer.exe
2197 .and_then(|path| path.into_os_string().into_string().ok()),
2198 )
2199 .with_signing_id(peer.signing_id)
2200 }
2201 };
2202 self.observed_peers
2203 .lock()
2204 .unwrap_or_else(|p| p.into_inner())
2205 .insert(connection, peer);
2206 }
2207
2208 pub(crate) fn record_caller_endpoints(
2209 &self,
2210 connection: ConnectionId,
2211 endpoints: Option<(SocketAddr, SocketAddr)>,
2212 ) {
2213 let mut callers = self
2214 .caller_endpoints
2215 .lock()
2216 .unwrap_or_else(|p| p.into_inner());
2217 callers.remove(&connection);
2218 if let Some(endpoints) = endpoints {
2219 callers.insert(connection, endpoints);
2220 }
2221 }
2222
2223 #[cfg(test)]
2224 pub(crate) fn with_caller_lookup_for_test(
2225 mut self,
2226 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2227 ) -> Self {
2228 self.caller_lookup = Some(Arc::new(lookup));
2229 self
2230 }
2231
2232 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2233 let endpoints = self
2234 .caller_endpoints
2235 .lock()
2236 .unwrap_or_else(|p| p.into_inner())
2237 .get(&connection)
2238 .copied();
2239 #[cfg(test)]
2240 let lookup = self.caller_lookup.clone();
2241 tokio::task::spawn_blocking(move || {
2242 #[cfg(test)]
2243 if let Some(lookup) = lookup {
2244 return lookup(endpoints);
2245 }
2246 endpoints
2247 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2248 .unwrap_or_default()
2249 })
2250 .await
2251 .unwrap_or_default()
2252 }
2253
2254 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2255 frame.header.channel == 0
2256 && frame.header.ty == FrameType::Request
2257 && self
2258 .forwarding
2259 .module_endpoint_for_connection(ctx.connection_id)
2260 .is_ok_and(|endpoint| endpoint.is_none())
2261 && parse_client_control_request(&frame.body)
2262 .ok()
2263 .is_some_and(|request| privileged_operation(&request).is_some())
2264 }
2265
2266 pub(crate) async fn handle_control_frame_timed(
2267 &self,
2268 ctx: &RouteCtx,
2269 frame: Frame,
2270 dispatch_started_at: Option<StdInstant>,
2271 ) -> Result<Vec<Frame>, RouterError> {
2272 let first = ctx.egress.first_ingress_frame();
2273 self.handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
2274 .await
2275 }
2276
2277 pub(crate) async fn handle_control_frame_ingress(
2278 &self,
2279 ctx: &RouteCtx,
2280 frame: Frame,
2281 dispatch_started_at: Option<StdInstant>,
2282 first: bool,
2283 ) -> Result<Vec<Frame>, RouterError> {
2284 if ctx.egress.ingress_terminal() {
2285 return Ok(Vec::new());
2286 }
2287 #[cfg(test)]
2288 self.claims.record_dispatch(frame.header.corr);
2289 let claim_binding = self.claims.binding(ctx.connection_id);
2290 let mut prefix_attempt = (frame.header.ty == FrameType::Hello
2291 && claim_binding
2292 .as_ref()
2293 .is_some_and(|binding| binding.prefix_module_id().is_some()))
2294 .then(|| self.prefix_hello_attempt(ctx));
2295 if claim_binding
2296 .as_ref()
2297 .is_some_and(|binding| binding.prefix_module_id().is_some())
2298 {
2299 let registered = self
2300 .forwarding
2301 .module_endpoint_for_connection(ctx.connection_id)
2302 .map_err(RouterError::Forwarding)?
2303 .is_some();
2304 if !registered && frame.header.ty != FrameType::Hello {
2305 return self
2306 .refuse_claim_connection(ctx, &frame, "claim_hello_required")
2307 .await;
2308 }
2309 if registered && frame.header.ty == FrameType::Goodbye {
2310 ctx.egress.terminate_ingress();
2311 let result = self.handle_goodbye(ctx.connection_id);
2312 self.claims.connection_closed(ctx.connection_id);
2313 self.request_refusal_close(
2314 ctx.connection_id,
2315 Some(&ctx.egress),
2316 CloseReason::new("claim_registration_ended", "prefix registration ended"),
2317 );
2318 return result;
2319 }
2320 if registered && frame.header.ty == FrameType::Request {
2321 let allowed =
2322 serde_json::from_slice::<ControlOpProbe>(&frame.body).is_ok_and(|probe| {
2323 matches!(
2324 probe.op.as_str(),
2325 "catalog.update"
2326 | "catalog.update_role_declarations"
2327 | "scope.describe"
2328 | "supervisor.live_roots"
2329 | "activity.record"
2330 )
2331 });
2332 if !allowed {
2333 return Ok(vec![control_error_frame(
2334 &frame,
2335 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2336 "prefix registrations have only brokered module operations",
2337 )?]);
2338 }
2339 }
2340 if registered
2341 && frame.header.ty == FrameType::Push
2342 && !serde_json::from_slice::<ControlOpProbe>(&frame.body)
2343 .is_ok_and(|probe| probe.op == "route.status")
2344 {
2345 return Ok(vec![control_error_frame(
2346 &frame,
2347 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2348 "prefix registrations may push only route.status for routes they serve",
2349 )?]);
2350 }
2351 }
2352 match frame.header.ty {
2353 FrameType::Ping => Ok(vec![pong(&frame)?]),
2354 FrameType::Hello => {
2355 if claim_binding
2356 .as_ref()
2357 .is_some_and(|binding| binding.prefix_module_id().is_none())
2358 {
2359 return self
2360 .refuse_claim_connection(ctx, &frame, "claim_hello_not_allowed")
2361 .await;
2362 }
2363 let channel_spawn = self.supervisor.registration_spawn(ctx.connection_id);
2367 if let Some(spawn) = &channel_spawn {
2368 spawn.wait_launch().await;
2369 if !spawn.launch_ready() {
2372 ctx.egress.terminate_ingress();
2373 return Ok(Vec::new());
2374 }
2375 #[cfg(test)]
2376 self.pause_hello_ingress_for_test().await;
2377 } else if claim_binding.is_none() {
2378 if let Ok(hello) = serde_json::from_slice::<serde_json::Value>(&frame.body) {
2379 if let (Some(module), Some(nonce)) = (
2380 hello
2381 .get("manifest")
2382 .and_then(|value| value.get("module_id"))
2383 .and_then(|value| value.as_str()),
2384 hello.get("launch_nonce").and_then(|value| value.as_str()),
2385 ) {
2386 if !self
2387 .config_runtime
2388 .as_ref()
2389 .is_some_and(|owner| owner.module_is_gated(module))
2390 && !self.supervisor.wait_program_binding(module, nonce).await
2391 {
2392 self.request_refusal_close(
2393 ctx.connection_id,
2394 Some(&ctx.egress),
2395 CloseReason::new(
2396 "program_refused",
2397 "the spawned program binding failed",
2398 ),
2399 );
2400 return Ok(vec![control_error_frame(
2401 &frame,
2402 if self
2403 .config_runtime
2404 .as_ref()
2405 .is_some_and(|owner| owner.is_recovering())
2406 {
2407 "config_gated"
2408 } else {
2409 "program_refused"
2410 },
2411 "the spawned program binding failed",
2412 )?]);
2413 }
2414 }
2415 }
2416 }
2417 let handler = self.clone();
2418 let connection = ctx.connection_id;
2419 let egress = ctx.egress.clone();
2420 let correlation = frame.header.corr;
2421 let dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
2422 let span = tracing::Span::current();
2423 let hello_binding = claim_binding.clone();
2424 let replies = tokio::task::spawn_blocking(move || {
2425 tracing::dispatcher::with_default(&dispatcher, || {
2426 span.in_scope(|| {
2427 handler.handle_hello_checked(
2428 connection,
2429 Some(egress),
2430 frame,
2431 true,
2432 hello_binding,
2433 channel_spawn,
2434 )
2435 })
2436 })
2437 })
2438 .await
2439 .map_err(|error| RouterError::backend(0, correlation, error.to_string()))??;
2440 #[cfg(test)]
2441 if prefix_attempt.is_some() {
2442 self.claims.pause_prefix_commit_for_test(false).await;
2443 }
2444 if let Some(attempt) = &mut prefix_attempt {
2445 let accepted = !replies
2446 .iter()
2447 .any(|reply| reply.header.ty == FrameType::Error)
2448 && self
2449 .registry
2450 .get_module_by_connection(ctx.connection_id)
2451 .map_err(|err| RouterError::backend(0, correlation, err.to_string()))?
2452 .is_some();
2453 if accepted {
2454 attempt.accepted = true;
2455 } else {
2456 ctx.egress.terminate_ingress();
2457 for reply in replies {
2458 ctx.egress.send_flushed(reply).await?;
2459 }
2460 return Ok(Vec::new());
2461 }
2462 }
2463 Ok(replies)
2464 }
2465 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2466 FrameType::Cancel => {
2467 if self
2473 .agent_declarations
2474 .cancel(ctx.connection_id, frame.header.corr)
2475 || self
2476 .forwarding
2477 .operator_confirms()
2478 .cancel(ctx.connection_id, frame.header.corr)
2479 || self.cancel_config_approval(ctx.connection_id, frame.header.corr)
2480 || self
2481 .supervisor
2482 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2483 || self.cancel_activity_subscription(ctx, &frame)?
2484 || self.cancel_catalog_subscription(ctx.connection_id, frame.header.corr)
2485 {
2486 Ok(Vec::new())
2487 } else {
2488 Ok(vec![control_error_frame(
2489 &frame,
2490 "unknown_subscription",
2491 "no subscription has this correlation id",
2492 )?])
2493 }
2494 }
2495 FrameType::Request => {
2496 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2499 let claim_present = op.as_ref().is_some_and(|probe| probe.op == "claim.present");
2500 if let Some(refusal) = self
2501 .recovery_request_refusal(
2502 ctx,
2503 &frame,
2504 op.as_ref().map(|probe| probe.op.as_str()),
2505 )
2506 .await?
2507 {
2508 if claim_present {
2509 self.reclaim_presented_expiry(&frame);
2510 return self
2511 .refuse_claim_connection(ctx, &frame, "config_gated")
2512 .await;
2513 }
2514 return Ok(vec![refusal]);
2515 }
2516 if claim_present {
2517 return self.handle_claim_present(ctx, &frame, first).await;
2518 }
2519 if op.as_ref().is_some_and(|probe| probe.op == "claim.mint") {
2520 return self.handle_claim_mint(ctx, &frame).await;
2521 }
2522 if let Some(refusal) = self.registration_operation_refusal(
2523 ctx,
2524 &frame,
2525 op.as_ref().map(|probe| probe.op.as_str()),
2526 )? {
2527 return Ok(vec![refusal]);
2528 }
2529 if op
2530 .as_ref()
2531 .is_some_and(|probe| probe.op == ops::CATALOG_WATCH)
2532 {
2533 return self.handle_catalog_watch(ctx, frame);
2534 }
2535 if op
2536 .as_ref()
2537 .is_some_and(|probe| probe.op == MODULE_TO_SUBC_OP_CATALOG_UPDATE)
2538 && subc_protocol::provider_declarations::catalog_update_has_provider_declarations(&frame.body)
2539 .unwrap_or(false)
2540 {
2541 return Ok(vec![control_error_frame(
2542 &frame,
2543 error_codes::CATALOG_UPDATE_FROZEN_DECLARATIONS,
2544 "provider_declarations are fixed at HELLO; restart the module to change them",
2545 )?]);
2546 }
2547 if let Some(probe) = op
2548 .as_ref()
2549 .filter(|probe| config_gate::is_module_config_op(&probe.op))
2550 {
2551 return self
2552 .handle_module_config_request(ctx, frame, &probe.op)
2553 .await;
2554 }
2555 if op
2556 .as_ref()
2557 .is_some_and(|probe| config_gate::is_config_op(&probe.op))
2558 {
2559 return self.handle_config_gate_request(ctx, frame).await;
2560 }
2561 if op
2562 .as_ref()
2563 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2564 {
2565 return self.handle_activity_record(ctx, frame).await;
2566 }
2567 if op.as_ref().is_some_and(|probe| {
2568 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2569 }) {
2570 return self.handle_activity_read(ctx, frame).await;
2571 }
2572 if op
2573 .as_ref()
2574 .is_some_and(|probe| probe.op == "operator.confirm")
2575 {
2576 return self.handle_operator_confirm(ctx, frame);
2577 }
2578 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2579 return self
2580 .observe_control_wait(
2581 ctx,
2582 None,
2583 ops::APP_OPEN,
2584 self.handle_app_open(ctx, frame),
2585 )
2586 .await;
2587 }
2588 if op
2589 .as_ref()
2590 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2591 {
2592 return self.handle_connection_describe(ctx, frame);
2593 }
2594 if op
2595 .as_ref()
2596 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2597 {
2598 return self.handle_scope_served_by(ctx.connection_id, frame);
2599 }
2600 if self
2601 .forwarding
2602 .module_endpoint_for_connection(ctx.connection_id)
2603 .map_err(RouterError::Forwarding)?
2604 .is_some()
2605 {
2606 if !is_known_module_request_op(&frame.body) {
2607 return Ok(vec![control_error_frame(
2608 &frame,
2609 "unsupported_control_frame",
2610 "module-originated channel-0 REQUEST is not supported",
2611 )?]);
2612 }
2613 let request = match parse_module_control_request_from_module(&frame.body) {
2614 Ok(request) => request,
2615 Err((err, ControlRequestBodyError::UnknownOp)) => {
2616 return Ok(vec![control_error_frame(
2617 &frame,
2618 "unsupported_control_frame",
2619 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2620 )?])
2621 }
2622 Err((err, ControlRequestBodyError::InvalidBody)) => {
2623 return Ok(vec![control_error_frame(
2624 &frame,
2625 "invalid_control_body",
2626 format!("malformed module control body: {err}"),
2627 )?])
2628 }
2629 };
2630 let op = module_control_request_op(&request);
2631 let corr = frame.header.corr;
2632 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2633 if let ModuleControlRequestFromModule::AgentCommands {
2634 started,
2635 finished,
2636 known,
2637 } = request
2638 {
2639 return self
2640 .agent_declarations
2641 .enqueue(
2642 self,
2643 ctx,
2644 frame,
2645 crate::resource_totals::Declaration {
2646 started,
2647 finished,
2648 known,
2649 },
2650 )
2651 .await;
2652 }
2653 let result =
2654 self.handle_module_control_request(ctx.connection_id, frame, request);
2655 log_slow_control_dispatch(
2656 dispatch_started_at,
2657 op,
2658 ctx.connection_id,
2659 corr,
2660 None,
2661 );
2662 return result;
2663 }
2664
2665 if is_known_module_request_op(&frame.body) {
2666 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2667 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2668 let code = if is_agents {
2669 "not_supervised"
2670 } else if op
2671 .as_ref()
2672 .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2673 {
2674 error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2675 } else {
2676 "not_registered"
2677 };
2678 return Ok(vec![control_error_frame(
2679 &frame,
2680 code,
2681 "module-originated control requests require the module's own registered connection",
2682 )?]);
2683 }
2684
2685 let request = match parse_client_control_request(&frame.body) {
2686 Ok(request) => request,
2687 Err((err, ControlRequestBodyError::UnknownOp)) => {
2688 return Ok(vec![control_error_frame(
2689 &frame,
2690 "unknown_control_op",
2691 format!("unknown client control op: {err}"),
2692 )?])
2693 }
2694 Err((err, ControlRequestBodyError::InvalidBody)) => {
2695 return Ok(vec![control_error_frame(
2696 &frame,
2697 "invalid_control_body",
2698 format!("malformed client control body: {err}"),
2699 )?])
2700 }
2701 };
2702 let op = client_control_request_op(&request);
2703 let corr = frame.header.corr;
2704 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2705 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2706 #[cfg(test)]
2707 if let Some(delay) = self.control_dispatch_delay {
2708 tokio::time::sleep(delay).await;
2709 }
2710 let result = self
2711 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2712 .await;
2713 if let Some(timings) = route_open_timings.as_mut() {
2714 timings.finish();
2715 }
2716 log_slow_control_dispatch(
2717 dispatch_started_at,
2718 op,
2719 ctx.connection_id,
2720 corr,
2721 route_open_timings.as_ref(),
2722 );
2723 result
2724 }
2725 FrameType::Push => {
2726 let Some(endpoint) = self
2727 .forwarding
2728 .module_endpoint_for_connection(ctx.connection_id)
2729 .map_err(RouterError::Forwarding)?
2730 else {
2731 return Ok(vec![control_error_frame(
2732 &frame,
2733 "unsupported_control_frame",
2734 "client-originated channel-0 PUSH is not supported",
2735 )?]);
2736 };
2737 self.handle_status_update(endpoint, frame)
2738 }
2739 FrameType::Response | FrameType::Error
2740 if self
2741 .forwarding
2742 .module_endpoint_for_connection(ctx.connection_id)
2743 .map_err(RouterError::Forwarding)?
2744 .is_some() =>
2745 {
2746 self.handle_module_relay_response(ctx.connection_id, frame)
2747 }
2748 ty => Ok(vec![control_error_frame(
2749 &frame,
2750 "unsupported_control_frame",
2751 format!("unsupported channel-0 frame {ty:?}"),
2752 )?]),
2753 }
2754 }
2755
2756 #[cfg(test)]
2757 fn pause_cleanup_for_test(&self, stage: u8) {
2758 let mut slot = self.cleanup_pause.lock().unwrap_or_else(|p| p.into_inner());
2759 if slot.as_ref().is_some_and(|(at, _)| *at == stage) {
2760 let (_, (reached, release)) = slot.take().unwrap();
2761 drop(slot);
2762 let _ = reached.send(());
2763 let _ = release.recv_timeout(Duration::from_secs(5));
2764 }
2765 }
2766
2767 #[cfg(all(test, unix))]
2770 pub(crate) fn after_nonce_attestation_for_test(&self, hook: impl FnOnce() + Send + 'static) {
2771 *self
2772 .after_nonce_attestation
2773 .lock()
2774 .unwrap_or_else(|p| p.into_inner()) = Some(Box::new(hook));
2775 }
2776
2777 #[cfg(test)]
2778 pub(super) fn pause_after_nonce_attestation_for_test(&self) {
2779 let hook = self
2780 .after_nonce_attestation
2781 .lock()
2782 .unwrap_or_else(|p| p.into_inner())
2783 .take();
2784 if let Some(hook) = hook {
2785 hook();
2786 }
2787 }
2788
2789 #[cfg(test)]
2790 async fn pause_hello_ingress_for_test(&self) {
2791 let pause = self
2792 .hello_ingress_pause
2793 .lock()
2794 .unwrap_or_else(|p| p.into_inner())
2795 .take();
2796 if let Some((reached, release)) = pause {
2797 let _ = reached.send(());
2798 let _ = release.await;
2799 }
2800 }
2801
2802 pub fn cleanup_connection(
2803 &self,
2804 connection_id: ConnectionId,
2805 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2806 self.agent_declarations.close(connection_id);
2820 self.close_catalog_subscriptions(connection_id);
2821 #[cfg(test)]
2822 self.pause_cleanup_for_test(0);
2823 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
2824 spawn.retire_registration_channel();
2825 }
2826 self.supervisor
2827 .forget_registration_connection(connection_id);
2828 #[cfg(test)]
2829 self.pause_cleanup_for_test(1);
2830 self.local_connections
2831 .lock()
2832 .unwrap_or_else(|p| p.into_inner())
2833 .remove(&connection_id);
2834 self.caller_endpoints
2835 .lock()
2836 .unwrap_or_else(|p| p.into_inner())
2837 .remove(&connection_id);
2838 self.build_class_warned
2839 .lock()
2840 .unwrap_or_else(|p| p.into_inner())
2841 .remove(&connection_id);
2842 self.observed_peers
2843 .lock()
2844 .unwrap_or_else(|p| p.into_inner())
2845 .remove(&connection_id);
2846 self.captured_peers
2847 .lock()
2848 .unwrap_or_else(|p| p.into_inner())
2849 .remove(&connection_id);
2850 self.config_desktop_stamps
2851 .lock()
2852 .unwrap_or_else(|p| p.into_inner())
2853 .remove(&connection_id);
2854 self.operator_apps
2855 .lock()
2856 .unwrap_or_else(|p| p.into_inner())
2857 .remove(&connection_id);
2858 self.forwarding
2859 .operator_confirms()
2860 .connection_closed(connection_id);
2861 let cleanup = self.cleanup_connection_with_end_reason(
2862 connection_id,
2863 RegistrationEndReason::ConnectionClosed,
2864 );
2865 self.claims.connection_closed(connection_id);
2868 cleanup
2869 }
2870
2871 fn cleanup_connection_with_end_reason(
2872 &self,
2873 connection_id: ConnectionId,
2874 requested_reason: RegistrationEndReason,
2875 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2876 #[cfg(test)]
2877 if let Some(pause) = self
2878 .activity_request_pause
2879 .lock()
2880 .unwrap()
2881 .as_ref()
2882 .filter(|pause| pause.connection == connection_id)
2883 {
2884 pause
2885 .cleanup_started
2886 .store(true, std::sync::atomic::Ordering::Release);
2887 }
2888 if let Some(runtime) = &self.config_runtime {
2889 runtime.disconnected(connection_id);
2890 }
2891 self.agent_declarations.close(connection_id);
2892 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2893 self.registry
2894 .get_module_by_connection(connection_id)?
2895 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2896 .and_then(|module| module.registration_end_reason().ok().flatten())
2897 .unwrap_or(requested_reason)
2898 } else {
2899 requested_reason
2900 };
2901 let crash_closed = self
2902 .registry
2903 .get_module_by_connection(connection_id)?
2904 .and_then(|registration| {
2905 self.forwarding
2906 .module_endpoint_for_connection(connection_id)
2907 .ok()
2908 .flatten()
2909 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2910 .map(|routes| (registration.manifest.module_id, routes))
2911 });
2912 let crash_closed = crash_closed.map(|(module_id, routes)| {
2913 let terminal = match self.supervisor.get(&module_id) {
2914 None => false,
2915 Some(module) => match module.will_recover_after_connection_loss() {
2916 Ok(will_recover) => !will_recover,
2917 Err(err) => {
2918 warn!(
2919 %module_id,
2920 error = %err,
2921 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2922 );
2923 false
2924 }
2925 },
2926 };
2927 let (reason, drained) = match self
2931 .forwarding
2932 .daemon_drain_result_for_connection(connection_id)
2933 {
2934 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2935 Ok(None) => (RouteCloseReason::Crash, false),
2936 Err(err) => {
2937 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2938 (RouteCloseReason::Crash, false)
2939 }
2940 };
2941 (module_id, routes, reason, drained, terminal)
2942 });
2943 let registrations = self.deregister_connection(connection_id, end_reason);
2944 let ended_authority = self
2947 .forwarding
2948 .target_authority_for_connection(connection_id)
2949 .ok()
2950 .flatten();
2951 let cleanup = if crash_closed.is_some() {
2952 self.forwarding.cleanup_connection_counted(connection_id)
2953 } else {
2954 self.forwarding
2960 .cleanup_connection(connection_id)
2961 .map(|released| crate::forwarding::ConnectionCleanup {
2962 released,
2963 abandoned_relays: 0,
2964 })
2965 };
2966 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2971 let abandoned = cleanup
2972 .as_ref()
2973 .map_or(0, |cleanup| cleanup.abandoned_relays);
2974 send_route_control_pushes(
2975 &self.forwarding,
2976 routes,
2977 ClientControlPush::RouteClosed {
2978 module_id,
2979 channels: Vec::new(),
2980 reason,
2981 drained,
2982 abandoned,
2983 excluded_subscriptions: 0,
2984 terminal: Some(terminal),
2985 },
2986 );
2987 }
2988 if let Ok(cleanup) = cleanup {
2989 self.emit_route_goodbyes(cleanup.released);
2990 }
2991 if let Some(record) = ended_authority {
2992 self.change_spawn_authority(&record);
2993 }
2994 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2998 crate::supervise::notify_registration_release();
2999 self.capability_evaluator.wake_deadline_loop();
3000 self.refresh_capability_requirements();
3001 }
3002 self.supervisor.remove_spawn_subscribers(connection_id);
3003 self.activity_streams
3004 .lock()
3005 .unwrap_or_else(|p| p.into_inner())
3006 .retain(|(id, _), task| {
3007 if *id == connection_id {
3008 task.abort();
3009 false
3010 } else {
3011 true
3012 }
3013 });
3014 self.hello_launch_nonces
3017 .lock()
3018 .unwrap_or_else(|poisoned| poisoned.into_inner())
3019 .forget(connection_id);
3020 self.scopes
3021 .write()
3022 .unwrap_or_else(|poisoned| poisoned.into_inner())
3023 .release_connection(connection_id);
3024 registrations
3025 }
3026
3027 pub(crate) fn handle_route_goodbye(
3028 &self,
3029 connection_id: ConnectionId,
3030 route_channel: u16,
3031 route_epoch: u32,
3032 ) -> Result<bool, RouterError> {
3033 debug!(
3034 connection_id = connection_id.get(),
3035 route_channel, route_epoch, "handling route GOODBYE"
3036 );
3037 let RouteRelease::Removed(released_route) = self
3038 .forwarding
3039 .release_client_route(connection_id, route_channel, route_epoch)
3040 .map_err(RouterError::Forwarding)?
3041 else {
3042 return Ok(false);
3043 };
3044 self.emit_route_goodbyes(vec![released_route]);
3045 Ok(true)
3046 }
3047
3048 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
3049 scope_expiry::ScopeExpiryContext::new(self).emit_route_goodbyes(released_routes);
3050 }
3051
3052 fn send_abandoned_route_bind_goodbye(
3064 &self,
3065 module_sink: &crate::FrameSink,
3066 negotiated_ver: u8,
3067 module_channel: u16,
3068 module_epoch: u32,
3069 ) {
3070 let frame = match Frame::build_with_version(
3071 negotiated_ver,
3072 FrameType::Goodbye,
3073 control_flags(),
3074 module_channel,
3075 module_epoch,
3076 0,
3077 Vec::new(),
3078 ) {
3079 Ok(frame) => frame,
3080 Err(err) => {
3081 warn!(
3082 route_channel = module_channel,
3083 error = %err,
3084 "failed to build GOODBYE for abandoned route.bind"
3085 );
3086 return;
3087 }
3088 };
3089 crate::forwarding::send_module_route_goodbye(
3090 &self.counters,
3091 module_sink,
3092 frame,
3093 None,
3094 "abandoned route.bind",
3095 );
3096 }
3097
3098 #[cfg(test)]
3099 fn handle_hello(
3100 &self,
3101 connection_id: ConnectionId,
3102 egress: Option<crate::router::FrameSink>,
3103 frame: Frame,
3104 ) -> Result<Vec<Frame>, RouterError> {
3105 let binding = self.claims.binding(connection_id);
3106 let channel_spawn = self.supervisor.registration_spawn(connection_id);
3107 self.handle_hello_checked(connection_id, egress, frame, false, binding, channel_spawn)
3108 }
3109
3110 fn handle_hello_checked(
3111 &self,
3112 connection_id: ConnectionId,
3113 sink: Option<crate::FrameSink>,
3114 frame: Frame,
3115 wait_for_gate: bool,
3116 prefix_binding: Option<claims::ClaimBinding>,
3117 channel_spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3118 ) -> Result<Vec<Frame>, RouterError> {
3119 let primary_sink = channel_spawn.as_ref().and_then(|_| sink.clone());
3120 #[cfg(test)]
3121 if prefix_binding.is_some() {
3122 self.claims.pause_prefix_hello_for_test(5);
3123 match self.claims.take_prefix_hello_failure_for_test() {
3124 Some("error") => {
3125 return Err(RouterError::backend(
3126 0,
3127 frame.header.corr,
3128 "injected HELLO failure",
3129 ))
3130 }
3131 Some("panic") => panic!("injected HELLO task panic"),
3132 _ => {}
3133 }
3134 }
3135 if let Some(binding) = &prefix_binding {
3136 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3137 let module = value
3138 .as_ref()
3139 .and_then(|value| value["manifest"]["module_id"].as_str());
3140 if !binding.admit_prefix_hello(module)
3141 || !self.prefix_binding_live(binding, connection_id)
3142 || !module.is_some_and(|module| {
3143 self.supervisor
3144 .prefix_claim_authorized(&binding.registration.module_id, module)
3145 })
3146 || binding.registration.check().is_err()
3147 {
3148 return Ok(vec![control_error_frame(
3149 &frame,
3150 "claim_prefix_refused",
3151 "HELLO must name the prefix claim's exact module id once",
3152 )?]);
3153 }
3154 }
3155 struct RefuseUnregisteredChannel {
3156 spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3157 sink: Option<crate::FrameSink>,
3158 }
3159 impl Drop for RefuseUnregisteredChannel {
3160 fn drop(&mut self) {
3161 if let Some(spawn) = &self.spawn {
3162 if !spawn.registered() {
3163 if let Some(sink) = &self.sink {
3164 sink.terminate_ingress();
3165 }
3166 spawn.retire_registration_channel();
3167 }
3168 }
3169 }
3170 }
3171 let _channel_attempt = RefuseUnregisteredChannel {
3172 spawn: channel_spawn.clone(),
3173 sink: sink.clone(),
3174 };
3175 if let Some(spawn) = &channel_spawn {
3176 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3177 if let Err(code) = spawn.admit_hello(
3178 value
3179 .as_ref()
3180 .and_then(|value| value["manifest"]["module_id"].as_str()),
3181 ) {
3182 return Ok(vec![control_error_frame(
3183 &frame,
3184 code,
3185 "registration channel HELLO refused",
3186 )?]);
3187 }
3188 }
3189 debug!(
3190 connection_id = connection_id.get(),
3191 corr = frame.header.corr,
3192 "handling HELLO"
3193 );
3194 if self
3198 .registry
3199 .get_module_by_connection(connection_id)
3200 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3201 .is_some()
3202 {
3203 return Ok(vec![control_error_frame(
3204 &frame,
3205 "invalid_hello",
3206 "connection is already registered as a module",
3207 )?]);
3208 }
3209 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
3210 Ok(value) => value,
3211 Err(err) => {
3212 return Ok(vec![control_error_frame(
3213 &frame,
3214 "invalid_hello",
3215 format!("malformed HELLO body: {err}"),
3216 )?])
3217 }
3218 };
3219 if let Err(detail) = catalog::validate_hello_declarations(&hello_value) {
3220 return Ok(vec![control_error_frame(
3221 &frame,
3222 error_codes::INVALID_PROVIDER_DECLARATIONS,
3223 detail,
3224 )?]);
3225 }
3226 if let Some(module) = hello_value
3227 .get("manifest")
3228 .and_then(|value| value.get("module_id"))
3229 .and_then(|value| value.as_str())
3230 {
3231 let recovering = self
3232 .config_runtime
3233 .as_ref()
3234 .is_some_and(|owner| owner.check_recovery().unwrap_or(true));
3235 if recovering
3239 && channel_spawn.as_ref().is_none_or(|spawn| {
3240 !self.supervisor.has_live_verified_channel_binding(spawn)
3241 || spawn.check().is_err()
3242 })
3243 && !(channel_spawn.is_none()
3244 && hello_value
3245 .get("launch_nonce")
3246 .and_then(|value| value.as_str())
3247 .is_some_and(|nonce| {
3248 self.supervisor.spawned_consumer_authorized(module, nonce)
3249 && self
3250 .supervisor
3251 .has_live_verified_program_binding(module, nonce)
3252 }))
3253 {
3254 return Ok(vec![control_error_frame(
3255 &frame,
3256 "config_gated",
3257 "recovery admits only previously bound supervised launches",
3258 )?]);
3259 }
3260 if self
3261 .config_runtime
3262 .as_ref()
3263 .is_some_and(|owner| owner.module_is_gated(module))
3264 && (channel_spawn
3265 .as_ref()
3266 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3267 || self.config_runtime.as_ref().is_some_and(|owner| {
3268 !recovering && owner.check_module_hold(module).is_err()
3269 }))
3270 {
3271 return Ok(vec![control_error_frame(
3272 &frame,
3273 "reserved_module",
3274 "a gated identity belongs only to its supervised child",
3275 )?]);
3276 }
3277 }
3278 if channel_spawn.is_none() {
3279 if let (Some(module), Some(nonce)) = (
3280 hello_value
3281 .get("manifest")
3282 .and_then(|value| value.get("module_id"))
3283 .and_then(|value| value.as_str()),
3284 hello_value
3285 .get("launch_nonce")
3286 .and_then(|value| value.as_str()),
3287 ) {
3288 let binding = self.supervisor.has_program_binding_record(module, nonce);
3289 let required = self
3290 .config_runtime
3291 .as_ref()
3292 .is_some_and(|owner| owner.module_is_gated(module));
3293 if let Some(error) = (binding || required)
3294 .then(|| {
3295 self.supervisor
3296 .check_current_program_binding(module, nonce)
3297 .err()
3298 })
3299 .flatten()
3300 {
3301 let code = if error.code == "program_refused" {
3302 "program_refused"
3303 } else {
3304 "config_delivery_deferred"
3305 };
3306 self.request_refusal_close(
3307 connection_id,
3308 sink.as_ref(),
3309 CloseReason::new(code, error.detail.clone()),
3310 );
3311 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3312 }
3313 }
3314 }
3315 if let Err(err) = validate_hello_display_name(&hello_value) {
3316 return Ok(vec![control_error_frame(
3317 &frame,
3318 "invalid_manifest",
3319 err.to_string(),
3320 )?]);
3321 }
3322 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
3323 return Ok(vec![control_error_frame(
3324 &frame,
3325 "invalid_capability_grammar",
3326 err.to_string(),
3327 )?]);
3328 }
3329 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
3330 return Ok(vec![control_error_frame(
3331 &frame,
3332 "invalid_manifest",
3333 err.to_string(),
3334 )?]);
3335 }
3336 if let Err(err) = validate_hello_event_declarations(&hello_value) {
3337 return Ok(vec![control_error_frame(
3338 &frame,
3339 "invalid_event_declaration",
3340 err.to_string(),
3341 )?]);
3342 }
3343 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
3344 return Ok(vec![control_error_frame(
3345 &frame,
3346 "invalid_manifest",
3347 err.to_string(),
3348 )?]);
3349 }
3350 if let Some(provenance) = hello_value
3351 .get("manifest")
3352 .and_then(|manifest| manifest.get("provenance"))
3353 {
3354 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
3355 return Ok(vec![control_error_frame(
3356 &frame,
3357 "invalid_manifest",
3358 format!("malformed manifest provenance: {err}"),
3359 )?]);
3360 }
3361 }
3362 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
3363 let module_id = hello_value["manifest"]["module_id"]
3364 .as_str()
3365 .unwrap_or("<unknown>");
3366 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
3367 }
3368 if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
3369 let module_id = hello_value
3370 .get("manifest")
3371 .and_then(|m| m.get("module_id"))
3372 .and_then(serde_json::Value::as_str)
3373 .unwrap_or("unknown");
3374 warn!(module_id, %reason, "dropping operation_deferral from HELLO");
3375 }
3376 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
3377 let module_id = hello_value["manifest"]["module_id"]
3378 .as_str()
3379 .unwrap_or("<unknown>");
3380 warn!(module_id, %reason, "dropping flow_operations from HELLO");
3381 }
3382 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
3383 let module_id = hello_value["manifest"]["module_id"]
3384 .as_str()
3385 .unwrap_or("<unknown>");
3386 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
3387 }
3388 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
3389 Ok(hello) => hello,
3390 Err(err) => {
3391 return Ok(vec![control_error_frame(
3392 &frame,
3393 "invalid_hello",
3394 format!("malformed HELLO body: {err}"),
3395 )?])
3396 }
3397 };
3398
3399 if hello.protocol_ver != hello.manifest.protocol_ver {
3400 return Ok(vec![control_error_frame(
3401 &frame,
3402 "invalid_manifest",
3403 format!(
3404 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
3405 hello.protocol_ver, hello.manifest.protocol_ver
3406 ),
3407 )?]);
3408 }
3409
3410 if hello.manifest.module_id.trim().is_empty() {
3411 return Ok(vec![control_error_frame(
3412 &frame,
3413 "invalid_manifest",
3414 "manifest module_id must not be empty",
3415 )?]);
3416 }
3417
3418 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
3419 Ok(negotiated_ver) => negotiated_ver,
3420 Err(message) => {
3421 return Ok(vec![control_error_frame(
3422 &frame,
3423 "version_unsupported",
3424 message,
3425 )?])
3426 }
3427 };
3428
3429 let _publication_gate = if let Some(owner) = &self.config_runtime {
3435 let gate = owner
3436 .module_gate(&hello.manifest.module_id)
3437 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3438 Some(if wait_for_gate {
3439 gate.blocking_lock_owned()
3440 } else {
3441 match gate.try_lock_owned() {
3442 Ok(guard) => guard,
3443 Err(_) => {
3444 return Ok(vec![control_error_frame(
3445 &frame,
3446 "config_delivery_deferred",
3447 "module admission is busy",
3448 )?])
3449 }
3450 }
3451 })
3452 } else {
3453 None
3454 };
3455 #[cfg(test)]
3456 if let Some(hold) = &self.hello_before_publication_hold {
3457 hold.wait();
3458 hold.wait();
3459 }
3460 let module = &hello.manifest.module_id;
3461 let required = self
3462 .config_runtime
3463 .as_ref()
3464 .is_some_and(|owner| owner.module_is_gated(module));
3465 let recovering = self
3466 .config_runtime
3467 .as_ref()
3468 .is_some_and(|owner| owner.is_recovering());
3469 if recovering
3477 && channel_spawn.as_ref().is_none_or(|spawn| {
3478 !self.supervisor.has_live_verified_channel_binding(spawn) || spawn.check().is_err()
3479 })
3480 && !(channel_spawn.is_none()
3481 && hello.launch_nonce.as_deref().is_some_and(|nonce| {
3482 self.supervisor.spawned_consumer_authorized(module, nonce)
3483 && self
3484 .supervisor
3485 .has_live_verified_program_binding(module, nonce)
3486 }))
3487 {
3488 return Ok(vec![control_error_frame(
3489 &frame,
3490 "config_gated",
3491 "recovery admits only previously bound supervised launches",
3492 )?]);
3493 }
3494 if required
3495 && (channel_spawn
3496 .as_ref()
3497 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3498 || self
3499 .config_runtime
3500 .as_ref()
3501 .is_some_and(|owner| !recovering && owner.check_module_hold(module).is_err()))
3502 {
3503 return Ok(vec![control_error_frame(
3504 &frame,
3505 "reserved_module",
3506 "gated admission requires the current supervised child's registration channel and verified image binding",
3507 )?]);
3508 }
3509 if let Some(spawn) = &channel_spawn {
3510 if let Err(code) = spawn.check() {
3511 return Ok(vec![control_error_frame(
3512 &frame,
3513 code,
3514 "registration launch is no longer live",
3515 )?]);
3516 }
3517 } else if let Some(nonce) = hello.launch_nonce.as_deref() {
3518 if required || self.supervisor.has_program_binding_record(module, nonce) {
3519 if let Err(error) = self.supervisor.check_current_program_binding(module, nonce) {
3520 let code = if error.code == "program_refused" {
3521 "program_refused"
3522 } else {
3523 "config_delivery_deferred"
3524 };
3525 self.request_refusal_close(
3526 connection_id,
3527 sink.as_ref(),
3528 CloseReason::new(code, error.detail.clone()),
3529 );
3530 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3531 }
3532 }
3533 }
3534
3535 let swap_admission = match &channel_spawn {
3543 Some(spawn) if spawn.is_candidate() => SwapHelloAdmission::Candidate,
3544 Some(_) => SwapHelloAdmission::NotSwapping,
3545 None if prefix_binding.is_some() => SwapHelloAdmission::NotSwapping,
3546 None => self
3547 .supervisor
3548 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref()),
3549 };
3550 if swap_admission == SwapHelloAdmission::ProtocolNone {
3551 warn!(
3552 module_id = %hello.manifest.module_id,
3553 connection_id = connection_id.get(),
3554 "HELLO refused: the module_id belongs to a protocol:none module, which never registers"
3555 );
3556 return Ok(vec![control_error_frame(
3559 &frame,
3560 "swap_token_invalid",
3561 format!(
3562 "module_id '{}' is a protocol:none module; a protocol:none module may not register",
3563 hello.manifest.module_id
3564 ),
3565 )?]);
3566 }
3567 if swap_admission == SwapHelloAdmission::ChannelRequired {
3568 warn!(
3569 module_id = %hello.manifest.module_id,
3570 connection_id = connection_id.get(),
3571 "HELLO refused: the module_id is gated and the HELLO did not arrive over its registration channel"
3572 );
3573 return Ok(vec![control_error_frame(
3576 &frame,
3577 "swap_token_invalid",
3578 format!(
3579 "module_id '{}' is gated; it must register over its supervised registration channel",
3580 hello.manifest.module_id
3581 ),
3582 )?]);
3583 }
3584 if swap_admission == SwapHelloAdmission::Refused {
3585 warn!(
3586 module_id = %hello.manifest.module_id,
3587 connection_id = connection_id.get(),
3588 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
3589 );
3590 return Ok(vec![control_error_frame(
3591 &frame,
3592 "swap_token_invalid",
3593 format!(
3594 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
3595 hello.manifest.module_id
3596 ),
3597 )?]);
3598 }
3599 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
3600
3601 let verification = if swap_candidate && channel_spawn.is_none() {
3606 ReservedHelloVerification {
3607 nonce_verified: true,
3608 prefix_owner: None,
3609 }
3610 } else {
3611 let verification = if let Some(binding) = &prefix_binding {
3612 if !self.supervisor.prefix_claim_authorized(
3613 &binding.registration.module_id,
3614 &hello.manifest.module_id,
3615 ) {
3616 return Ok(vec![control_error_frame(
3617 &frame,
3618 "claim_prefix_refused",
3619 "prefix ownership changed",
3620 )?]);
3621 }
3622 Ok(ReservedHelloVerification {
3623 nonce_verified: true,
3624 prefix_owner: Some(binding.registration.module_id.clone()),
3625 })
3626 } else if channel_spawn.is_some() {
3627 self.supervisor
3628 .channel_hello_verification(&hello.manifest.module_id)
3629 } else {
3630 self.supervisor.reserved_hello_verification(
3631 &hello.manifest.module_id,
3632 hello.launch_nonce.as_deref(),
3633 )
3634 };
3635 match verification {
3636 Ok(verified) => verified,
3637 Err(rejection) => {
3638 let message = match rejection {
3639 ReservedHelloRejection::Exact { module_id } => format!(
3640 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
3641 ),
3642 ReservedHelloRejection::Prefix {
3643 prefix,
3644 owner_module_id,
3645 } => format!(
3646 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
3647 hello.manifest.module_id
3648 ),
3649 };
3650 return Ok(vec![control_error_frame(
3651 &frame,
3652 "reserved_module",
3653 message,
3654 )?]);
3655 }
3656 }
3657 };
3658
3659 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
3660 &hello.manifest.module_id,
3661 hello.manifest.capabilities.as_ref(),
3662 );
3663 if let Some(refusal) = reserved_capability_refusals.first() {
3664 let capability = refusal.capability.clone();
3665 let bound_module = refusal.claimants[0].clone();
3666 log_duplicate_claim_events(reserved_capability_refusals);
3667 return Ok(vec![control_error_frame(
3668 &frame,
3669 "reserved_capability",
3670 format!(
3671 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3672 capability, bound_module, hello.manifest.module_id
3673 ),
3674 )?]);
3675 }
3676
3677 if self
3680 .forwarding
3681 .connection_has_client_routes(connection_id)
3682 .map_err(RouterError::Forwarding)?
3683 {
3684 return Ok(vec![control_error_frame(
3685 &frame,
3686 "invalid_hello",
3687 "connection has open client routes and cannot also register as a module",
3688 )?]);
3689 }
3690
3691 self.hello_launch_nonces
3699 .lock()
3700 .unwrap_or_else(|poisoned| poisoned.into_inner())
3701 .record(
3702 connection_id,
3703 if prefix_binding.is_some() || channel_spawn.is_some() {
3704 None
3705 } else {
3706 hello.launch_nonce.as_deref()
3707 },
3708 );
3709 let control_ops = effective_module_control_ops(hello.control_ops);
3710 let prefix_submodule = verification.prefix_owner.is_some();
3711 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
3714 #[cfg(test)]
3715 struct PublicationFinished(Arc<std::sync::Barrier>);
3716 #[cfg(test)]
3717 impl Drop for PublicationFinished {
3718 fn drop(&mut self) {
3719 self.0.wait();
3720 }
3721 }
3722 #[cfg(test)]
3723 let _publication_finished = self.hello_after_launch_check_hold.as_ref().map(|hold| {
3724 hold.wait();
3725 hold.wait();
3726 PublicationFinished(hold.clone())
3727 });
3728 let publication_spawn = channel_spawn
3729 .as_ref()
3730 .or_else(|| prefix_binding.as_ref().map(|binding| &binding.registration));
3731 #[cfg(test)]
3732 if prefix_binding.is_some() {
3733 self.claims.pause_prefix_hello_for_test(7);
3734 }
3735 let _registration_publication = match publication_spawn {
3736 Some(spawn) => Some(match spawn.begin_publication() {
3737 Ok(guard) => guard,
3738 Err(code) => {
3739 return Ok(vec![control_error_frame(
3740 &frame,
3741 code,
3742 "registration ended before HELLO publication",
3743 )?])
3744 }
3745 }),
3746 None => None,
3747 };
3748 let (_prefix_live, _prefix_ownership) = if let Some(binding) = &prefix_binding {
3749 let live = self.claims.lock_prefix_binding(binding, connection_id);
3750 let ownership = self.supervisor.prefix_claim_publication(
3751 &binding.registration.module_id,
3752 &hello.manifest.module_id,
3753 );
3754 if live.is_none()
3755 || ownership.is_none()
3756 || !self.claim_primary_registered(binding.spawn, &binding.registration)
3757 || binding.prefix_module_id() != Some(hello.manifest.module_id.as_str())
3758 {
3759 return Ok(vec![control_error_frame(
3760 &frame,
3761 "claim_prefix_refused",
3762 "prefix authority ended before publication",
3763 )?]);
3764 }
3765 (live, ownership)
3766 } else {
3767 (None, None)
3768 };
3769 if let Some(spawn) = publication_spawn {
3770 self.forwarding
3771 .bind_spawn(connection_id, spawn)
3772 .map_err(RouterError::Forwarding)?;
3773 }
3774 let hello_authority = match (&channel_spawn, &prefix_binding) {
3784 (Some(spawn), _) => Some(spawn.authority.clone()),
3785 (None, None) => hello.launch_nonce.as_deref().and_then(|nonce| {
3786 self.supervisor
3787 .spawned_consumer_record(&hello.manifest.module_id, nonce)
3788 }),
3789 (None, Some(_)) => None,
3790 }
3791 .map(|record| HelloAuthority {
3792 record,
3793 requested: hello
3794 .manifest
3795 .provider_declarations
3796 .clone()
3797 .unwrap_or_default(),
3798 froze: Default::default(),
3799 });
3800 #[cfg(test)]
3801 if prefix_binding.is_some() {
3802 self.claims.pause_prefix_hello_for_test(8);
3803 }
3804 if swap_candidate {
3805 let module_id = hello.manifest.module_id.clone();
3806 let mut differing_declarations = Default::default();
3807 let reply = self.register_swap_candidate(
3808 connection_id,
3809 sink,
3810 &frame,
3811 hello.manifest,
3812 negotiated_ver,
3813 control_ops,
3814 hello_ack,
3815 hello_authority.as_ref(),
3816 &mut differing_declarations,
3817 #[cfg(windows)]
3818 hello.launch_nonce.as_deref(),
3819 )?;
3820 if let Some(spawn) = &channel_spawn {
3821 if self
3822 .registry
3823 .get_module_by_connection(connection_id)
3824 .ok()
3825 .flatten()
3826 .is_some()
3827 {
3828 if let Some(sink) = primary_sink {
3829 spawn.bind_primary_sink(sink);
3830 }
3831 spawn.hello_accepted();
3832 }
3833 } else if self
3834 .registry
3835 .get_module_by_connection(connection_id)
3836 .ok()
3837 .flatten()
3838 .is_some()
3839 {
3840 self.supervisor
3841 .retire_unused_registration_channel(&module_id, true);
3842 }
3843 if self
3844 .registry
3845 .get_module_by_connection(connection_id)
3846 .ok()
3847 .flatten()
3848 .is_some()
3849 {
3850 self.file_development_widening(
3851 hello_authority.map(|authority| authority.record),
3852 differing_declarations,
3853 );
3854 }
3855 return Ok(reply);
3856 }
3857 let registration = match self
3864 .registry
3865 .register_unpublished_verified_with_control_ops(
3866 hello.manifest,
3867 negotiated_ver,
3868 connection_id,
3869 control_ops,
3870 verification.nonce_verified,
3871 ) {
3872 Ok(registration) => registration,
3873 Err(RegistryError::DuplicateModuleId { module_id }) => {
3874 return Ok(vec![control_error_frame(
3875 &frame,
3876 "duplicate_module_id",
3877 format!(
3878 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
3879 ),
3880 )?])
3881 }
3882 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3883 return Ok(vec![control_error_frame(
3884 &frame,
3885 "invalid_module_id",
3886 err.to_string(),
3887 )?])
3888 }
3889 Err(err) => {
3890 return Ok(vec![control_error_frame(
3891 &frame,
3892 "registry_error",
3893 err.to_string(),
3894 )?])
3895 }
3896 };
3897 let differing_declarations =
3898 match self.admit_hello_authority(connection_id, hello_authority.as_ref()) {
3899 Ok(differing) => differing,
3900 Err(err) => {
3901 if matches!(
3902 self.rollback_unpublished_hello(connection_id),
3903 Ok(r) if !r.is_empty()
3904 ) {
3905 crate::supervise::notify_registration_release();
3906 }
3907 return Err(RouterError::Forwarding(err));
3908 }
3909 };
3910 #[cfg(test)]
3911 self.hold_between_registry_and_forwarding();
3912
3913 let mut reply = Vec::new();
3914 let publication = self
3917 .registry
3918 .publish_registration_after(connection_id, || {
3919 if let Some(sink) = sink {
3920 let concurrency = manifest_concurrency(®istration.manifest);
3936 self.forwarding.register_module_connection_acked(
3937 connection_id,
3938 registration.manifest.module_id.clone(),
3939 negotiated_ver,
3940 concurrency,
3941 sink,
3942 crate::forwarding::ModuleHelloAck {
3943 frame: hello_ack,
3944 verified_prefix_owner: verification.prefix_owner,
3945 },
3946 )?;
3947 } else {
3948 reply.push(hello_ack);
3951 }
3952 #[cfg(test)]
3953 self.hold_between_ack_and_publication();
3954 Ok(())
3955 })
3956 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3957 let published = match publication {
3958 Ok(published) => published,
3959 Err(err) => {
3960 if matches!(
3967 self.rollback_unpublished_hello(connection_id),
3968 Ok(r) if !r.is_empty()
3969 ) {
3970 crate::supervise::notify_registration_release();
3971 }
3972 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
3973 return Ok(vec![control_error_frame(
3974 &frame,
3975 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
3976 "invalid_hello"
3977 } else {
3978 forwarding_error_code(&err)
3979 },
3980 err.to_string(),
3981 )?]);
3982 }
3983 };
3984 if !published {
3985 debug!(
3986 module_id = %registration.manifest.module_id,
3987 connection_id = connection_id.get(),
3988 "registration was removed before it could be published"
3989 );
3990 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
3991 return Ok(reply);
3992 }
3993
3994 drop(_prefix_ownership);
3995 drop(_prefix_live);
3996
3997 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
4006 info!(
4007 module_id = %registration.manifest.module_id,
4008 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
4009 );
4010 }
4011 if registration.manifest.registration_channel == Some(false)
4012 && self
4013 .supervisor
4014 .module_is_gated(®istration.manifest.module_id)
4015 && self
4016 .config_runtime
4017 .as_ref()
4018 .and_then(|owner| owner.registration_declaration(®istration.manifest.module_id))
4019 .as_deref()
4020 == Some("declared")
4021 {
4022 warn!(module_id = %registration.manifest.module_id,
4023 "HELLO registration_channel is false despite the approved channel declaration; registration remains admitted");
4024 }
4025
4026 self.apply_registration_capabilities(®istration);
4027
4028 self.supervisor
4029 .record_activity_registration(®istration.manifest.module_id);
4030
4031 info!(
4032 module_id = %registration.manifest.module_id,
4033 module_version = %registration.manifest.module_version,
4034 negotiated_ver,
4035 routable_provider = manifest_provides_routable_role(®istration.manifest),
4036 connection_id = connection_id.get(),
4037 "module registered"
4038 );
4039
4040 if let Some(spawn) = &channel_spawn {
4041 if let Some(sink) = primary_sink {
4042 spawn.bind_primary_sink(sink);
4043 }
4044 spawn.hello_accepted();
4045 } else if !prefix_submodule {
4046 self.supervisor
4047 .retire_unused_registration_channel(®istration.manifest.module_id, false);
4048 }
4049 self.file_development_widening(
4050 hello_authority.map(|authority| authority.record),
4051 differing_declarations,
4052 );
4053 #[cfg(windows)]
4054 if let Some(nonce) = hello.launch_nonce.as_deref() {
4055 self.supervisor
4056 .program_hello_accepted(®istration.manifest.module_id, nonce);
4057 }
4058
4059 #[cfg(test)]
4060 if let Some(hold) = &self.hello_inside_publication_hold {
4061 hold.wait();
4062 hold.wait();
4063 }
4064
4065 Ok(reply)
4066 }
4067
4068 #[allow(clippy::too_many_arguments)]
4085 fn register_swap_candidate(
4086 &self,
4087 connection_id: ConnectionId,
4088 sink: Option<crate::FrameSink>,
4089 frame: &Frame,
4090 manifest: ModuleManifest,
4091 negotiated_ver: u8,
4092 control_ops: Vec<String>,
4093 hello_ack: Frame,
4094 hello_authority: Option<&HelloAuthority>,
4095 differing_declarations: &mut ProviderDeclarations,
4096 #[cfg(windows)] launch_nonce: Option<&str>,
4097 ) -> Result<Vec<Frame>, RouterError> {
4098 let module_id = manifest.module_id.clone();
4099 self.supervisor
4102 .with_swap_candidate_admission(&module_id, || {
4103 let reply = self.publish_swap_candidate(
4104 connection_id,
4105 sink,
4106 frame,
4107 manifest,
4108 negotiated_ver,
4109 control_ops,
4110 hello_ack,
4111 hello_authority,
4112 differing_declarations,
4113 #[cfg(windows)]
4114 launch_nonce,
4115 );
4116 let admitted = self
4117 .registry
4118 .get_candidate(&module_id)
4119 .ok()
4120 .flatten()
4121 .is_some_and(|candidate| candidate.connection_id == connection_id);
4122 (reply, admitted)
4123 })
4124 .unwrap_or_else(|| {
4125 Ok(vec![control_error_frame(
4126 frame,
4127 "swap_token_invalid",
4128 "swap ended before candidate HELLO publication",
4129 )?])
4130 })
4131 }
4132
4133 #[allow(clippy::too_many_arguments)]
4134 fn publish_swap_candidate(
4135 &self,
4136 connection_id: ConnectionId,
4137 sink: Option<crate::FrameSink>,
4138 frame: &Frame,
4139 manifest: ModuleManifest,
4140 negotiated_ver: u8,
4141 control_ops: Vec<String>,
4142 hello_ack: Frame,
4143 hello_authority: Option<&HelloAuthority>,
4144 differing_declarations: &mut ProviderDeclarations,
4145 #[cfg(windows)] launch_nonce: Option<&str>,
4146 ) -> Result<Vec<Frame>, RouterError> {
4147 let module_id = manifest.module_id.clone();
4148 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
4149 manifest,
4150 negotiated_ver,
4151 connection_id,
4152 control_ops,
4153 ) {
4154 Ok(registration) => registration,
4155 Err(RegistryError::DuplicateModuleId { module_id }) => {
4156 return Ok(vec![control_error_frame(
4157 frame,
4158 "duplicate_module_id",
4159 format!(
4160 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
4161 ),
4162 )?])
4163 }
4164 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
4165 return Ok(vec![control_error_frame(
4166 frame,
4167 "invalid_module_id",
4168 err.to_string(),
4169 )?])
4170 }
4171 Err(err) => {
4172 return Ok(vec![control_error_frame(
4173 frame,
4174 "registry_error",
4175 err.to_string(),
4176 )?])
4177 }
4178 };
4179 *differing_declarations = match self.admit_hello_authority(connection_id, hello_authority) {
4180 Ok(differing) => differing,
4181 Err(err) => {
4182 if matches!(
4183 self.rollback_unpublished_hello(connection_id),
4184 Ok(r) if !r.is_empty()
4185 ) {
4186 crate::supervise::notify_registration_release();
4187 }
4188 return Err(RouterError::Forwarding(err));
4189 }
4190 };
4191 #[cfg(test)]
4192 self.hold_between_registry_and_forwarding();
4193 let mut reply = Vec::new();
4194 let publication = self
4195 .registry
4196 .publish_registration_after(connection_id, || {
4197 if let Some(sink) = sink {
4198 let concurrency = manifest_concurrency(®istration.manifest);
4202 self.forwarding.register_candidate_module_connection_acked(
4203 connection_id,
4204 module_id.clone(),
4205 negotiated_ver,
4206 concurrency,
4207 sink,
4208 hello_ack,
4209 )?;
4210 } else {
4211 reply.push(hello_ack);
4212 }
4213 #[cfg(test)]
4214 self.hold_between_ack_and_publication();
4215 Ok(())
4216 })
4217 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4218 let published = match publication {
4219 Ok(published) => published,
4220 Err(err) => {
4221 if matches!(
4222 self.rollback_unpublished_hello(connection_id),
4223 Ok(r) if !r.is_empty()
4224 ) {
4225 crate::supervise::notify_registration_release();
4226 }
4227 self.refuse_hello_authority(connection_id, hello_authority);
4228 return Ok(vec![control_error_frame(
4229 frame,
4230 forwarding_error_code(&err),
4231 err.to_string(),
4232 )?]);
4233 }
4234 };
4235 if !published {
4236 debug!(
4237 module_id = %module_id,
4238 connection_id = connection_id.get(),
4239 "swap candidate registration was removed before it could be published"
4240 );
4241 self.refuse_hello_authority(connection_id, hello_authority);
4242 return Ok(reply);
4243 }
4244 info!(
4245 module_id = %module_id,
4246 module_version = %registration.manifest.module_version,
4247 negotiated_ver,
4248 ready = registration.ready,
4249 connection_id = connection_id.get(),
4250 "swap candidate registered; not routable until cutover"
4251 );
4252 #[cfg(windows)]
4253 if let Some(nonce) = launch_nonce {
4254 self.supervisor.program_hello_accepted(&module_id, nonce);
4255 }
4256 Ok(reply)
4257 }
4258
4259 fn admit_hello_authority(
4265 &self,
4266 connection: ConnectionId,
4267 authority: Option<&HelloAuthority>,
4268 ) -> Result<ProviderDeclarations, ForwardingError> {
4269 let Some(authority) = authority else {
4270 return Ok(ProviderDeclarations::new());
4271 };
4272 let froze = authority
4273 .record
4274 .freeze_declarations(authority.requested.clone());
4275 authority
4276 .froze
4277 .store(froze.is_some(), std::sync::atomic::Ordering::Relaxed);
4278 self.forwarding
4279 .bind_target_authority(connection, authority.record.clone())
4280 .inspect_err(|_| self.refuse_hello_authority(connection, Some(authority)))?;
4281 Ok(froze.unwrap_or_default())
4282 }
4283
4284 fn refuse_hello_authority(&self, connection: ConnectionId, authority: Option<&HelloAuthority>) {
4289 let Some(authority) = authority else {
4290 return;
4291 };
4292 if let Err(err) = self.forwarding.unbind_target_authority(connection) {
4293 warn!(error = %err, "failed to unbind a refused HELLO from its spawn");
4294 }
4295 if authority
4296 .froze
4297 .swap(false, std::sync::atomic::Ordering::Relaxed)
4298 {
4299 authority.record.unfreeze_refused_declarations();
4300 }
4301 }
4302
4303 fn build_hello_ack(
4304 &self,
4305 frame: &Frame,
4306 negotiated_ver: u8,
4307 module_id: &str,
4308 ) -> Result<Frame, RouterError> {
4309 let ack = ModuleHelloAckBody {
4310 negotiated_ver,
4311 subc_ops: module_subc_ops(),
4312 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
4313 storage: self
4314 .storage_config
4315 .as_ref()
4316 .map(|cfg| cfg.descriptor_for(module_id)),
4317 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4318 };
4319 let body = serde_json::to_vec(&ack).map_err(|err| {
4320 RouterError::backend(
4321 0,
4322 frame.header.corr,
4323 format!("failed to encode HELLO_ACK: {err}"),
4324 )
4325 })?;
4326
4327 Frame::build_with_version(
4328 negotiated_ver,
4329 FrameType::HelloAck,
4330 control_flags(),
4331 0,
4332 0,
4333 frame.header.corr,
4334 body,
4335 )
4336 .map_err(RouterError::FrameBuild)
4337 }
4338
4339 async fn handle_client_control_request(
4340 &self,
4341 ctx: &RouteCtx,
4342 frame: Frame,
4343 request: ClientControlRequest,
4344 route_open_timings: &mut Option<RouteOpenTimings>,
4345 ) -> Result<Vec<Frame>, RouterError> {
4346 if let Some(owner) = &self.config_runtime {
4347 let module = match &request {
4348 ClientControlRequest::SupervisorRestart { module_id, .. }
4349 | ClientControlRequest::SupervisorReload { module_id }
4350 | ClientControlRequest::SupervisorSwap { module_id, .. }
4351 | ClientControlRequest::SupervisorSetEnabled {
4352 module_id,
4353 enabled: true,
4354 } => Some(module_id.as_str()),
4355 _ => None,
4356 };
4357 let rescan = matches!(
4358 &request,
4359 ClientControlRequest::SupervisorRescan { preview: false }
4360 | ClientControlRequest::SupervisorRescanChecked { preview: false, .. }
4361 );
4362 let held = if module.is_some() || rescan {
4363 let module = module.map(str::to_owned);
4364 owner
4365 .on_blocking(move |owner| owner.checked_admission_hold(module.as_deref()))
4366 .await
4367 } else {
4368 Ok(())
4369 };
4370 if let Err(error) = held {
4371 if error.code != "config_delivery_deferred" {
4372 let code = if error.code == "store_tampered" {
4373 "store_tampered"
4374 } else {
4375 "config_gated"
4376 };
4377 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
4378 }
4379 }
4380 }
4381 if let Some(operation) = privileged_operation(&request) {
4382 let caller = self.resolve_caller(ctx.connection_id).await;
4383 let op = client_control_request_op(&request);
4384 let text = prompt_text(operation, &caller);
4385 match self.privileged_operator_prompt {
4386 PrivilegedOperatorPrompt::Log => {
4387 info!(target: "subc_daemon::privileged_prompt", operation = op,
4388 target = ?operation, caller = %caller.description(), summary = %text,
4389 "privileged request would prompt the operator");
4390 }
4391 PrivilegedOperatorPrompt::Enforce => {
4392 let key =
4393 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
4394 let stamp = self
4395 .operator_apps
4396 .lock()
4397 .unwrap_or_else(|p| p.into_inner())
4398 .get(&ctx.connection_id)
4399 .cloned();
4400 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
4401 &self.operator_app_policy,
4402 &self.launch_table,
4403 stamp.as_ref(),
4404 )
4405 .then(|| self.shortcut_context(stamp, None, None));
4406 let outcome = self
4407 .forwarding
4408 .operator_confirms()
4409 .approve_privileged_with_shortcut(
4410 ctx,
4411 &frame,
4412 op,
4413 text,
4414 caller.description(),
4415 key,
4416 shortcut,
4417 )
4418 .await;
4419 if outcome.code != "confirmed" {
4420 return Ok(vec![outcome.frame(&frame)]);
4421 }
4422 }
4423 }
4424 }
4425 match request {
4426 ClientControlRequest::DaemonShutdownIntent { intent } => {
4427 if intent.as_str() != Some("restart") {
4428 return Ok(vec![control_error_frame(
4429 &frame,
4430 "invalid_params",
4431 "intent must be 'restart'",
4432 )?]);
4433 }
4434 if let Some(owner) = &self.config_runtime {
4439 match owner
4440 .on_blocking(|owner| owner.assign_daemon_restart())
4441 .await
4442 {
4443 Ok(Some(generation)) => {
4444 info!(
4445 generation,
4446 "daemon restart will warm up approved generation"
4447 )
4448 }
4449 Ok(None) => {}
4450 Err(error) => warn!(
4451 %error,
4452 "daemon restart intent not recorded; the next start boots the last active generation"
4453 ),
4454 }
4455 }
4456 let response = ClientControlResponse::DaemonShutdownIntent {
4457 intent: "restart".to_owned(),
4458 expires_at_ms: self.supervisor.announce_daemon_restart(),
4459 };
4460 Ok(vec![control_response_body_frame(
4461 &frame,
4462 &response,
4463 "ClientControlResponse::DaemonShutdownIntent",
4464 )?])
4465 }
4466 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
4467 ClientControlRequest::CatalogList { module_id } => {
4468 self.handle_catalog_list(frame, module_id)
4469 }
4470 ClientControlRequest::RouteOpen {
4471 target,
4472 identity,
4473 consumer_identity,
4474 consumer_capabilities,
4475 role_versions,
4476 busy_refusals,
4477 admission_facts,
4478 scope,
4479 ..
4480 } => {
4481 let extensions = match serde_json::from_slice::<subc_control::RouteOpenRequestExt>(
4482 &frame.body,
4483 ) {
4484 Ok(extensions) => extensions,
4485 Err(err) => {
4486 return Ok(vec![control_error_frame(
4487 &frame,
4488 "invalid_control_body",
4489 format!("malformed route.open extensions: {err}"),
4490 )?]);
4491 }
4492 };
4493 self.handle_route_open(
4494 ctx,
4495 frame,
4496 RouteOpenRequest {
4497 target,
4498 identity,
4499 consumer_identity,
4500 consumer_capabilities,
4501 role_versions,
4502 busy_refusals,
4503 admission_facts,
4504 scope,
4505 accepts_close_reasons: extensions.accepts_close_reasons,
4506 },
4507 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
4508 )
4509 .await
4510 }
4511 ClientControlRequest::RoutePoll {
4512 route_channel,
4513 route_epoch,
4514 kind,
4515 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
4516 ClientControlRequest::SupervisorList {} => {
4517 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
4518 self.agent_declarations.refuse(
4519 self,
4520 ctx,
4521 control_error_frame(
4522 &frame,
4523 "busy",
4524 "supervisor list work is at capacity; retry",
4525 )?,
4526 );
4527 return Ok(Vec::new());
4528 };
4529 let handler = self.clone();
4531 let sink = ctx.egress.clone();
4532 tokio::spawn(
4533 async move {
4534 let _permit = permit;
4535 let result = handler.handle_supervisor_list(frame.clone()).await;
4536 let replies = match result {
4537 Ok(replies) => replies,
4538 Err(error) => {
4539 control_error_frame(&frame, "backend_error", error.to_string())
4540 .into_iter()
4541 .collect()
4542 }
4543 };
4544 for reply in replies {
4545 if !matches!(
4546 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
4547 .await,
4548 Ok(Ok(()))
4549 ) {
4550 break;
4551 }
4552 }
4553 }
4554 .with_current_subscriber(),
4555 );
4556 Ok(Vec::new())
4557 }
4558 ClientControlRequest::SupervisorSpawnSnapshot {} => {
4559 self.handle_supervisor_spawn_snapshot(frame)
4560 }
4561 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
4562 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
4563 }
4564 ClientControlRequest::SupervisorRestart {
4565 module_id,
4566 drain_timeout_ms,
4567 } => {
4568 self.log_supervisor_request_received(
4569 ctx,
4570 frame.header.corr,
4571 ops::SUPERVISOR_RESTART,
4572 Some(&module_id),
4573 None,
4574 )?;
4575 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4576 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
4577 .await
4578 }
4579 ClientControlRequest::SupervisorSwap {
4580 module_id,
4581 ready_timeout_ms,
4582 } => {
4583 self.log_supervisor_request_received(
4584 ctx,
4585 frame.header.corr,
4586 ops::SUPERVISOR_SWAP,
4587 Some(&module_id),
4588 None,
4589 )?;
4590 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4591 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
4592 .await
4593 }
4594 ClientControlRequest::SupervisorReload { module_id } => {
4595 self.log_supervisor_request_received(
4596 ctx,
4597 frame.header.corr,
4598 ops::SUPERVISOR_RELOAD,
4599 Some(&module_id),
4600 None,
4601 )?;
4602 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4603 self.handle_supervisor_reload(frame, module_id, requester)
4604 .await
4605 }
4606 ClientControlRequest::SupervisorRescan { preview } => {
4607 if !preview {
4608 self.log_supervisor_request_received(
4609 ctx,
4610 frame.header.corr,
4611 ops::SUPERVISOR_RESCAN,
4612 None,
4613 None,
4614 )?;
4615 }
4616 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4617 let operation = self.handle_supervisor_rescan(
4618 frame,
4619 if preview {
4620 RescanMode::Preview
4621 } else {
4622 RescanMode::Execute
4623 },
4624 requester,
4625 );
4626 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
4627 .await
4628 }
4629 ClientControlRequest::SupervisorRescanChecked {
4630 preview,
4631 candidate,
4632 expect,
4633 } => {
4634 self.observe_control_wait(
4635 ctx,
4636 None,
4637 ops::SUPERVISOR_RESCAN,
4638 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
4639 )
4640 .await
4641 }
4642 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
4643 self.log_supervisor_request_received(
4644 ctx,
4645 frame.header.corr,
4646 ops::SUPERVISOR_RELEASE_RESERVED,
4647 Some(&module_id),
4648 None,
4649 )?;
4650 self.handle_supervisor_release_reserved(frame, module_id)
4651 .await
4652 }
4653 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
4654 self.log_supervisor_request_received(
4655 ctx,
4656 frame.header.corr,
4657 ops::SUPERVISOR_SET_ENABLED,
4658 Some(&module_id),
4659 Some(enabled),
4660 )?;
4661 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4662 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
4663 .await
4664 }
4665 ClientControlRequest::SupervisorHealthProbe { module_id } => {
4666 self.handle_supervisor_health_probe(ctx, frame, module_id)
4667 .await
4668 }
4669 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
4670 ClientControlRequest::SupervisorRoutes { module_id } => {
4671 self.handle_supervisor_routes(frame, module_id)
4672 }
4673 ClientControlRequest::SupervisorProvenance { module_id } => {
4674 self.handle_supervisor_provenance(ctx, frame, module_id)
4675 .await
4676 }
4677 ClientControlRequest::SupervisorStderrTail {
4678 module_id,
4679 max_lines,
4680 max_bytes,
4681 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
4682 ClientControlRequest::SupervisorTerminals { module_id } => {
4683 self.handle_supervisor_terminals(ctx, frame, module_id)
4684 .await
4685 }
4686 }
4687 }
4688
4689 fn log_supervisor_request_received(
4690 &self,
4691 ctx: &RouteCtx,
4692 corr: u64,
4693 op: &'static str,
4694 module_id: Option<&str>,
4695 enabled: Option<bool>,
4696 ) -> Result<(), RouterError> {
4697 let caller = self.supervisor_requester(ctx, corr)?;
4698 let caller = principal_label(&caller);
4699 info!(
4702 target: "control",
4703 op,
4704 module_id,
4705 enabled,
4706 connection_id = ctx.connection_id.get(),
4707 caller = %caller,
4708 "supervisor request received"
4709 );
4710 Ok(())
4711 }
4712
4713 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
4714 Ok(self
4715 .registry
4716 .get_module_by_connection(ctx.connection_id)
4717 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4718 .map(|registration| Principal::Reserved {
4719 module_id: registration.manifest.module_id,
4720 })
4721 .unwrap_or(Principal::Direct))
4722 }
4723
4724 fn handle_module_control_request(
4725 &self,
4726 connection_id: ConnectionId,
4727 frame: Frame,
4728 request: ModuleControlRequestFromModule,
4729 ) -> Result<Vec<Frame>, RouterError> {
4730 match request {
4731 ModuleControlRequestFromModule::RouteAuthorize {
4732 principal,
4733 scope,
4734 target,
4735 expected,
4736 } => self.handle_route_authorize(
4737 connection_id,
4738 frame,
4739 principal,
4740 scope,
4741 target,
4742 expected,
4743 ),
4744 ModuleControlRequestFromModule::CatalogUpdate {
4745 provides,
4746 capabilities,
4747 ready,
4748 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
4749 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
4750 self.handle_catalog_update_role_declarations(
4751 connection_id,
4752 frame,
4753 role_declarations,
4754 )
4755 }
4756 ModuleControlRequestFromModule::LiveRoots {} => {
4757 let registered = self
4758 .registry
4759 .get_module_by_connection(connection_id)
4760 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4761 let Some(registration) = registered else {
4762 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
4763 };
4764 let response = self
4765 .forwarding
4766 .live_roots(®istration.manifest.module_id)
4767 .map_err(RouterError::Forwarding)?;
4768 Ok(vec![control_response_body_frame(
4769 &frame,
4770 &response,
4771 "ModuleControlResponseToModule::LiveRoots",
4772 )?])
4773 }
4774 ModuleControlRequestFromModule::AgentCommands {
4775 started,
4776 finished,
4777 known,
4778 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
4779 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
4780 self.handle_scope_sync(connection_id, frame, generation, scopes)
4781 }
4782 ModuleControlRequestFromModule::ScopeApply {
4783 generation,
4784 upsert,
4785 end,
4786 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
4787 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
4788 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
4789 }
4790 _ => Ok(vec![control_error_frame(
4791 &frame,
4792 "unsupported_control_frame",
4793 "unsupported module control operation",
4794 )?]),
4795 }
4796 }
4797
4798 fn handle_scope_served_by(
4799 &self,
4800 connection_id: ConnectionId,
4801 frame: Frame,
4802 ) -> Result<Vec<Frame>, RouterError> {
4803 let registered = self
4804 .registry
4805 .get_module_by_connection(connection_id)
4806 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4807 let Some(registered) = registered else {
4808 return Ok(vec![control_error_frame(
4809 &frame,
4810 "not_registered",
4811 "scope.served_by requires an active module registration owned by this connection",
4812 )?]);
4813 };
4814 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
4815 {
4816 Ok(request) => request,
4817 Err(err) => {
4818 return Ok(vec![control_error_frame(
4819 &frame,
4820 "invalid_control_body",
4821 format!("malformed scope.served_by body: {err}"),
4822 )?])
4823 }
4824 };
4825 let answer = self
4826 .scopes
4827 .read()
4828 .unwrap_or_else(|poisoned| poisoned.into_inner())
4829 .served_by(
4830 ®istered.manifest.module_id,
4831 &request.scope_ref,
4832 request.scope_epoch,
4833 &request.module_id,
4834 );
4835 let reply = subc_control::ScopeServedByReply::new(
4836 answer,
4837 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4838 );
4839 Ok(vec![control_response_body_frame(
4840 &frame,
4841 &reply,
4842 "ScopeServedByReply",
4843 )?])
4844 }
4845
4846 fn handle_operator_confirm(
4847 &self,
4848 ctx: &RouteCtx,
4849 frame: Frame,
4850 ) -> Result<Vec<Frame>, RouterError> {
4851 use crate::operator_confirm::{audit, Outcome};
4852 let registration = self
4853 .registry
4854 .get_module_by_connection(ctx.connection_id)
4855 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4856 let Some(registration) = registration else {
4857 let outcome = Outcome::refusal("not_registered");
4858 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4859 return Ok(vec![outcome.frame(&frame)]);
4860 };
4861 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
4862 Ok(request) => request,
4863 Err(_) => {
4864 let outcome = Outcome::refusal("invalid_control_body");
4865 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4866 return Ok(vec![outcome.frame(&frame)]);
4867 }
4868 };
4869 let declared = matches!(
4870 request.kind,
4871 Some(subc_protocol::session::OperatorConfirmKind::Operation)
4872 ) && request.operation.as_ref().is_some_and(|operation| {
4873 registration
4874 .manifest
4875 .confirm_operations
4876 .as_ref()
4877 .is_some_and(|operations| operations.contains(operation))
4878 });
4879 let module_id = registration.manifest.module_id;
4880 let nonce = self
4884 .hello_launch_nonces
4885 .lock()
4886 .unwrap_or_else(|p| p.into_inner())
4887 .nonce(ctx.connection_id)
4888 .map(str::to_owned);
4889 let nonce_proven = self
4892 .confirm_channel_proven(ctx.connection_id, &module_id)
4893 .unwrap_or_else(|| {
4894 nonce.as_deref().is_some_and(|nonce| {
4895 self.supervisor
4896 .spawned_consumer_authorized(&module_id, nonce)
4897 })
4898 });
4899 let confirms = self.forwarding.operator_confirms();
4900 self.forwarding
4901 .with_operator_route(
4902 ctx.connection_id,
4903 request.route_channel,
4904 request.route_epoch,
4905 |binding| {
4906 let shortcut = binding
4907 .filter(|binding| {
4908 nonce_proven
4909 && crate::operator_confirm::valid_summary(&request.summary)
4910 && declared
4911 && crate::operator_confirm::ShortcutContext::route_eligible(
4912 &self.operator_app_policy,
4913 &self.launch_table,
4914 binding,
4915 )
4916 })
4917 .map(|binding| {
4918 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
4919 &self.operator_app_policy,
4920 binding,
4921 );
4922 let stamp = if phone.is_some() {
4923 None
4924 } else {
4925 binding.operator_app.clone()
4926 };
4927 self.shortcut_context(
4928 stamp,
4929 phone,
4930 Some((request.route_channel, request.route_epoch)),
4931 )
4932 });
4933 confirms.admit_with_shortcut(
4934 ctx,
4935 frame,
4936 module_id,
4937 nonce_proven,
4938 request,
4939 binding,
4940 shortcut,
4941 )
4942 },
4943 )
4944 .map_err(RouterError::Forwarding)
4945 }
4946
4947 fn shortcut_context(
4948 &self,
4949 stamp: Option<crate::operator_apps::OperatorAppStamp>,
4950 phone_fingerprint: Option<String>,
4951 route: Option<(u16, u32)>,
4952 ) -> crate::operator_confirm::ShortcutContext {
4953 crate::operator_confirm::ShortcutContext {
4954 ledger: self.activity.clone(),
4955 forwarding: self.forwarding.clone(),
4956 launches: self.launch_table.clone(),
4957 stamp,
4958 phone_fingerprint,
4959 route,
4960 connections: self.operator_apps.clone(),
4961 #[cfg(test)]
4962 hooks: Default::default(),
4963 }
4964 }
4965
4966 #[cfg(any(test, feature = "test-support"))]
4968 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
4969 self.operator_app_policy.phone_enabled = enabled;
4970 self
4971 }
4972
4973 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
4976 let mut scopes = ScopeRecords::from(scopes);
4977 for (index, record) in scopes.records.iter_mut().enumerate() {
4978 if let Some(refusal) = Self::canonicalize_workdir(record) {
4979 scopes.refusals.insert(index, refusal);
4980 }
4981 }
4982 scopes
4983 }
4984
4985 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
4986 let workdir = record.workdir.as_mut()?;
4987 let path = std::path::Path::new(workdir);
4988 if !path.is_absolute() {
4989 return Some((
4990 error_codes::INVALID_WORKDIR,
4991 "workdir must be absolute".to_string(),
4992 ));
4993 }
4994 match ProjectRootId::from_path_allowing_missing(path) {
4995 Ok(root) => match root.as_path().to_str() {
4996 Some(canonical) => {
4997 *workdir = canonical.to_string();
4998 None
4999 }
5000 None => Some((
5001 error_codes::INVALID_WORKDIR,
5002 "canonical workdir is not UTF-8".to_string(),
5003 )),
5004 },
5005 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
5006 }
5007 }
5008
5009 fn handle_scope_sync(
5013 &self,
5014 connection_id: ConnectionId,
5015 frame: Frame,
5016 generation: u64,
5017 scopes: Vec<ScopeRecord>,
5018 ) -> Result<Vec<Frame>, RouterError> {
5019 self.handle_scope_change(connection_id, frame, generation, scopes, None)
5020 }
5021
5022 fn handle_scope_change(
5023 &self,
5024 connection_id: ConnectionId,
5025 frame: Frame,
5026 generation: u64,
5027 scopes: Vec<ScopeRecord>,
5028 end: Option<Vec<ScopeEnd>>,
5029 ) -> Result<Vec<Frame>, RouterError> {
5030 let is_apply = end.is_some();
5031 let Some(registration) = self
5032 .registry
5033 .get_module_by_connection(connection_id)
5034 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5035 else {
5036 return Ok(vec![control_error_frame(
5037 &frame,
5038 "not_registered",
5039 "scope.sync and scope.apply require an active module registration owned by this connection",
5040 )?]);
5041 };
5042 let owner = registration.manifest.module_id;
5043 let scopes = Self::prepare_scope_records(scopes);
5044 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
5045 let is_current_launch = |connection: ConnectionId| {
5046 self.connection_is_current_launch(&owner, connection, current_nonce.as_deref())
5047 };
5048 let mut table = self
5053 .scopes
5054 .write()
5055 .unwrap_or_else(|poisoned| poisoned.into_inner());
5056 let outcome = match end {
5057 Some(end) => table.apply(
5058 &owner,
5059 connection_id,
5060 is_current_launch,
5061 generation,
5062 scopes,
5063 end,
5064 ),
5065 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
5066 };
5067 let drained = match &outcome {
5068 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
5069 Err(_) => Vec::new(),
5070 };
5071 let has_deadlines = outcome.is_ok() && table.has_deadlines();
5072 drop(table);
5073 if has_deadlines {
5074 self.scope_deadline_added.notify_one();
5075 }
5076 match outcome {
5077 Ok(applied) => {
5078 let counts = ScopeOutcomeCounts::of(&applied.results);
5079 info!(
5080 owner = %owner,
5081 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5082 generation,
5083 records = applied.results.len(),
5084 created = counts.created,
5085 replaced = counts.replaced,
5086 updated = counts.updated,
5087 unchanged = counts.unchanged,
5088 refused = counts.refused,
5089 ended = applied.ended.len(),
5090 tag_changes = applied.tag_changes.len(),
5091 routes_closed = drained.len(),
5092 "scope change accepted"
5093 );
5094 for refused in applied
5100 .results
5101 .iter()
5102 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
5103 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
5104 {
5105 warn!(
5106 owner = %owner,
5107 generation,
5108 scope_ref = %refused.scope_ref,
5109 scope_epoch = refused.scope_epoch,
5110 code = refused.code.as_deref().unwrap_or(""),
5111 "scope record refused"
5112 );
5113 }
5114 self.close_scope_drained_routes(drained);
5115 let response = if is_apply {
5116 ModuleControlResponseToModule::ScopeApply {
5117 generation,
5118 results: applied.results,
5119 end_results: applied.end_results,
5120 ended: applied.ended,
5121 }
5122 } else {
5123 ModuleControlResponseToModule::ScopeSync {
5124 generation,
5125 results: applied.results,
5126 ended: applied.ended,
5127 }
5128 };
5129 Ok(vec![control_response_body_frame(
5130 &frame,
5131 &response,
5132 "scope change response",
5133 )?])
5134 }
5135 Err(refusal) => {
5136 info!(
5137 owner = %owner,
5138 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5139 generation,
5140 code = refusal.code,
5141 "scope change refused"
5142 );
5143 Ok(vec![control_error_frame(
5144 &frame,
5145 refusal.code,
5146 refusal.message,
5147 )?])
5148 }
5149 }
5150 }
5151
5152 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
5154 scope_expiry::ScopeExpiryContext::new(self).close_scope_drained_routes(drained);
5155 }
5156
5157 fn handle_route_authorize(
5163 &self,
5164 connection_id: ConnectionId,
5165 frame: Frame,
5166 principal: Principal,
5167 scope: subc_protocol::scope::RouteAuthorizeScope,
5168 target: String,
5169 expected: subc_protocol::scope::RouteAuthorizeExpected,
5170 ) -> Result<Vec<Frame>, RouterError> {
5171 use crate::scopes::admission::{admit_route, AdmissionMode};
5172 use subc_protocol::scope::{
5173 RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
5174 };
5175 let registry = self
5176 .registry
5177 .route_admission_snapshot()
5178 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5179 let owner_configured = match &scope.owner {
5180 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5181 _ => false,
5182 };
5183 let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
5184 let answer = {
5185 let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
5186 let permitted = registry
5187 .iter()
5188 .find(|r| r.connection_id == connection_id)
5189 .is_some_and(|r| {
5190 scopes.is_federated_owner(&Principal::Reserved {
5191 module_id: r.manifest.module_id.clone(),
5192 }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
5193 caps.provides
5194 .iter()
5195 .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
5196 })
5197 });
5198 if !permitted {
5199 drop(scopes);
5200 return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
5201 "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
5202 }
5203 admit_route(
5204 &scopes,
5205 ®istry,
5206 &principal,
5207 &target,
5208 Some(&selector),
5209 owner_configured,
5210 AdmissionMode::Authorize(&expected),
5211 )
5212 };
5213 let response = match answer {
5214 Ok(admission) => {
5215 let mut response = RouteAuthorizeResponse::new(
5216 RouteAuthorizeOutcome::Authorized,
5217 "route admission authorized",
5218 );
5219 response.content_version = admission.content_version;
5220 response
5221 }
5222 Err(refusal) => refusal.response(),
5223 };
5224 Ok(vec![control_response_body_frame(
5225 &frame,
5226 &response,
5227 "RouteAuthorizeResponse",
5228 )?])
5229 }
5230
5231 fn handle_scope_describe(
5234 &self,
5235 connection_id: ConnectionId,
5236 frame: Frame,
5237 owner: Principal,
5238 scope_ref: String,
5239 ) -> Result<Vec<Frame>, RouterError> {
5240 let registered = self
5241 .registry
5242 .get_module_by_connection(connection_id)
5243 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5244 let Some(registered) = registered else {
5245 return Ok(vec![control_error_frame(
5246 &frame,
5247 "not_registered",
5248 "scope.describe requires an active module registration owned by this connection",
5249 )?]);
5250 };
5251 let scopes = self
5252 .scopes
5253 .read()
5254 .unwrap_or_else(|poisoned| poisoned.into_inner());
5255 if scopes.is_federated_owner(&owner)
5256 && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
5257 {
5258 return Ok(vec![control_error_frame(
5259 &frame,
5260 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
5261 "scope.describe of the federated owner requires scope-federated/v1",
5262 )?]);
5263 }
5264 let mut description = scopes.describe(&owner, &scope_ref);
5265 drop(scopes);
5266 if owner
5267 != (Principal::Reserved {
5268 module_id: registered.manifest.module_id.clone(),
5269 })
5270 {
5271 if let Some(facts) = description
5272 .stamp
5273 .as_mut()
5274 .and_then(|stamp| stamp.attributes.federated.as_mut())
5275 {
5276 facts.stamp_required = false;
5277 }
5278 }
5279 if !registered
5282 .manifest
5283 .capabilities
5284 .as_ref()
5285 .is_some_and(|caps| {
5286 caps.provides
5287 .iter()
5288 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5289 })
5290 {
5291 if let Some(stamp) = description.stamp.as_mut() {
5292 stamp.attributes.on_behalf_of_agent = None;
5293 }
5294 }
5295 if !registered
5296 .manifest
5297 .capabilities
5298 .as_ref()
5299 .is_some_and(|caps| {
5300 caps.provides
5301 .iter()
5302 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5303 })
5304 {
5305 if let Some(stamp) = description.stamp.as_mut() {
5306 stamp.attributes.flow_version = None;
5307 stamp.attributes.flow_code_hash = None;
5308 stamp.attributes.flow_install_id = None;
5309 }
5310 }
5311 let owner_configured = match &owner {
5312 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5317 _ => false,
5318 };
5319 let response = ModuleControlResponseToModule::ScopeDescribe {
5320 status: description.status,
5321 scope_epoch: description.scope_epoch,
5322 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
5323 owner_synced: description.owner_synced,
5324 owner_configured,
5325 scope: description.stamp,
5326 };
5327 Ok(vec![control_response_body_frame(
5328 &frame,
5329 &response,
5330 "ModuleControlResponseToModule::ScopeDescribe",
5331 )?])
5332 }
5333
5334 fn handle_catalog_update_role_declarations(
5335 &self,
5336 connection_id: ConnectionId,
5337 frame: Frame,
5338 role_declarations: serde_json::Value,
5339 ) -> Result<Vec<Frame>, RouterError> {
5340 if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
5341 return Ok(vec![control_error_frame(
5342 &frame,
5343 "invalid_role_declarations",
5344 err.to_string(),
5345 )?]);
5346 }
5347 let declarations = serde_json::from_value(role_declarations)
5348 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5349 if self
5350 .registry
5351 .replace_role_declarations_for_connection(connection_id, declarations)
5352 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5353 .is_none()
5354 {
5355 return Ok(vec![control_error_frame(
5356 &frame,
5357 "not_registered",
5358 "catalog.update_role_declarations requires an active module registration owned by this connection",
5359 )?]);
5360 }
5361 control_response_body_frame(
5362 &frame,
5363 &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
5364 "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
5365 )
5366 .map(|frame| vec![frame])
5367 }
5368
5369 fn handle_catalog_update(
5370 &self,
5371 connection_id: ConnectionId,
5372 frame: Frame,
5373 provides: Vec<ProviderRole>,
5374 capabilities: Option<CapabilityDeclarations>,
5375 ready: Option<bool>,
5376 ) -> Result<Vec<Frame>, RouterError> {
5377 self.refresh_capability_requirements();
5378 let Some(registration) = self
5379 .registry
5380 .get_module_by_connection(connection_id)
5381 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5382 else {
5383 return Ok(vec![control_error_frame(
5384 &frame,
5385 "not_registered",
5386 "catalog.update requires an active module registration owned by this connection",
5387 )?]);
5388 };
5389
5390 if let Some(message) =
5391 catalog_update_frozen_field_message(®istration.manifest, &provides)
5392 {
5393 return Ok(vec![control_error_frame(
5394 &frame,
5395 "catalog_update_frozen_field",
5396 message,
5397 )?]);
5398 }
5399
5400 let mut candidate = registration.manifest.clone();
5401 candidate.provides = provides.clone();
5402 candidate.capabilities = capabilities
5403 .clone()
5404 .or_else(|| registration.manifest.capabilities.clone());
5405 if let Some(rich_ui) = &candidate.rich_ui {
5406 if let Err(err) = rich_ui.validate() {
5407 return Ok(vec![control_error_frame(
5408 &frame,
5409 "invalid_manifest",
5410 err.to_string(),
5411 )?]);
5412 }
5413 }
5414 if let Err(err) = candidate.validate_capability_grammar() {
5415 return Ok(vec![control_error_frame(
5416 &frame,
5417 "invalid_capability_grammar",
5418 err.to_string(),
5419 )?]);
5420 }
5421
5422 let mut conflicts = self
5425 .capability_evaluator
5426 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
5427 if let Some(conflict) = conflicts.first() {
5428 let message = format!(
5429 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
5430 conflict.capability, conflict.claimants[0], candidate.module_id
5431 );
5432 for conflict in &mut conflicts {
5433 conflict.source = DuplicateClaimSource::CatalogUpdate;
5434 }
5435 log_duplicate_claim_events(conflicts);
5436 return Ok(vec![control_error_frame(
5437 &frame,
5438 "reserved_capability",
5439 message,
5440 )?]);
5441 }
5442
5443 let updated = match self.registry.replace_catalog_for_connection(
5444 connection_id,
5445 provides,
5446 capabilities,
5447 ready,
5448 ) {
5449 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => {
5450 return Ok(vec![control_error_frame(
5451 &frame,
5452 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5453 "scope-federated/v1 is fixed at HELLO; restart the module to change it",
5454 )?])
5455 }
5456 Err(crate::registry::RegistryError::CallerAttestationCapabilityChanged) => {
5457 return Ok(vec![control_error_frame(
5458 &frame,
5459 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5460 "caller-attestation/v1 is fixed at HELLO; restart the module to change it",
5461 )?])
5462 }
5463 result => {
5464 result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5465 }
5466 };
5467 if updated.is_none() {
5468 return Ok(vec![control_error_frame(
5469 &frame,
5470 "not_registered",
5471 "catalog.update requires an active module registration owned by this connection",
5472 )?]);
5473 }
5474 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
5475 log_duplicate_claim_events(
5476 self.capability_evaluator
5477 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
5478 );
5479 }
5480 if capability_census_trigger(
5481 registration.manifest.capabilities.as_ref(),
5482 updated
5483 .as_ref()
5484 .and_then(|entry| entry.manifest.capabilities.as_ref()),
5485 ) {
5486 self.enforce_capability_denies();
5487 }
5488 self.refresh_capability_requirements();
5489
5490 let response = ModuleControlResponseToModule::CatalogUpdate {};
5491 control_response_body_frame(
5492 &frame,
5493 &response,
5494 "ModuleControlResponseToModule::CatalogUpdate",
5495 )
5496 .map(|frame| vec![frame])
5497 }
5498
5499 pub(crate) fn with_activity_ledger_config(
5500 mut self,
5501 run_dir: &Path,
5502 incarnation: String,
5503 config: crate::activity::ActivityConfig,
5504 ) -> Self {
5505 self.activity = Some(crate::activity::ActivityLedger::open_config(
5506 run_dir.join("activity.jsonl"),
5507 incarnation,
5508 config,
5509 ));
5510 self.supervisor.set_activity(self.activity.clone());
5511 self
5512 }
5513
5514 pub(crate) async fn configure_activity(
5515 &self,
5516 config: crate::activity::ActivityConfig,
5517 ) -> Result<(), &'static str> {
5518 match &self.activity {
5519 Some(ledger) => ledger.configure(config).await,
5520 None => Ok(()),
5521 }
5522 }
5523
5524 fn cancel_activity_subscription(
5525 &self,
5526 ctx: &RouteCtx,
5527 frame: &Frame,
5528 ) -> Result<bool, RouterError> {
5529 let task = self
5530 .activity_streams
5531 .lock()
5532 .unwrap_or_else(|p| p.into_inner())
5533 .remove(&(ctx.connection_id, frame.header.corr));
5534 let Some(task) = task else {
5535 return Ok(false);
5536 };
5537 task.abort();
5538 let end = Frame::build_with_version(
5539 response_version(frame),
5540 FrameType::StreamEnd,
5541 control_flags(),
5542 0,
5543 0,
5544 frame.header.corr,
5545 Vec::new(),
5546 )
5547 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
5548 let sink = ctx.egress.clone();
5549 tokio::spawn(async move {
5550 let _ = sink.send(end).await;
5551 });
5552 Ok(true)
5553 }
5554
5555 async fn handle_activity_read(
5556 &self,
5557 ctx: &RouteCtx,
5558 frame: Frame,
5559 ) -> Result<Vec<Frame>, RouterError> {
5560 #[cfg(test)]
5561 {
5562 let pause = self
5563 .activity_request_pause
5564 .lock()
5565 .unwrap()
5566 .clone()
5567 .filter(|pause| pause.connection == ctx.connection_id);
5568 if let Some(pause) = pause {
5569 let _cancelled = ActivityRequestCancellation(pause.clone());
5570 pause.started.notify_one();
5571 std::future::pending::<()>().await;
5572 }
5573 }
5574 use subc_control::{
5575 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
5576 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
5577 };
5578 #[derive(Deserialize)]
5579 struct Identity {
5580 #[serde(default)]
5581 consumer_identity: Option<ConsumerIdentity>,
5582 }
5583 let mut bound_spawn = None;
5587 let legacy = match self.activity_reader(ctx.connection_id) {
5588 channel_operations::ActivityReader::Module { module_id, spawn } => {
5589 bound_spawn = Some(spawn);
5590 Some(Ok(Principal::Reserved { module_id }))
5591 }
5592 channel_operations::ActivityReader::Refused => {
5594 return Ok(vec![control_error_frame(
5595 &frame,
5596 ACTIVITY_READ_FORBIDDEN,
5597 "helper, relay and retired connections cannot read activity",
5598 )?]);
5599 }
5600 channel_operations::ActivityReader::Legacy => None,
5601 };
5602 let principal = match legacy {
5606 Some(principal) => principal,
5607 None => match self
5608 .forwarding
5609 .module_id_for_connection(ctx.connection_id)
5610 .map_err(RouterError::Forwarding)?
5611 {
5612 Some(module_id) => Ok(Principal::Reserved { module_id }),
5613 None => {
5614 let identity: Identity = match serde_json::from_slice(&frame.body) {
5615 Ok(identity) => identity,
5616 Err(_) => {
5617 return Ok(vec![control_error_frame(
5618 &frame,
5619 ACTIVITY_MALFORMED,
5620 "malformed activity read",
5621 )?])
5622 }
5623 };
5624 self.route_open_principal(
5625 ctx.connection_id,
5626 &frame,
5627 identity.consumer_identity,
5628 )?
5629 }
5630 },
5631 };
5632 let admitted = matches!(principal, Ok(Principal::Direct))
5633 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
5634 if !admitted {
5635 return Ok(vec![control_error_frame(
5636 &frame,
5637 ACTIVITY_READ_FORBIDDEN,
5638 "activity reads require an operator or trusted mirror principal",
5639 )?]);
5640 }
5641 let Some(ledger) = &self.activity else {
5642 return Ok(vec![control_error_frame(
5643 &frame,
5644 ACTIVITY_UNAVAILABLE,
5645 "activity ledger unavailable",
5646 )?]);
5647 };
5648 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
5649 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
5650 {
5651 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
5652 Ok(request) => ledger.query(request).await,
5653 Err(_) => Err(ACTIVITY_MALFORMED),
5654 };
5655 return match result {
5656 Ok(reply) => Ok(vec![control_response_body_frame(
5657 &frame,
5658 &reply,
5659 ops::ACTIVITY_QUERY,
5660 )?]),
5661 Err(code) => Ok(vec![control_error_frame(
5662 &frame,
5663 code,
5664 "activity query refused",
5665 )?]),
5666 };
5667 }
5668 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
5669 Ok(request) => request,
5670 Err(_) => {
5671 return Ok(vec![control_error_frame(
5672 &frame,
5673 ACTIVITY_MALFORMED,
5674 "malformed activity subscription",
5675 )?])
5676 }
5677 };
5678 let mut subscription = match ledger.subscribe(request.since).await {
5679 Ok(subscription) => subscription,
5680 Err(code) => {
5681 return Ok(vec![control_error_frame(
5682 &frame,
5683 code,
5684 "activity subscription refused",
5685 )?])
5686 }
5687 };
5688 let key = (ctx.connection_id, frame.header.corr);
5689 let sink = ctx.egress.clone();
5690 let mut streams = self
5691 .activity_streams
5692 .lock()
5693 .unwrap_or_else(|p| p.into_inner());
5694 streams.retain(|_, task| !task.is_finished());
5695 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
5696 return Ok(vec![control_error_frame(
5697 &frame,
5698 "duplicate_subscription",
5699 "activity subscription correlation id already in use",
5700 )?]);
5701 }
5702 let stream = async move {
5703 loop {
5704 let entry = if let Some(entry) = subscription.replay.pop_front() {
5705 Some(entry)
5706 } else {
5707 subscription.receiver.recv().await
5708 };
5709 let Some(entry) = entry else {
5710 break;
5711 };
5712 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
5713 return;
5714 };
5715 let Ok(next) = Frame::build_with_version(
5716 response_version(&frame),
5717 FrameType::StreamData,
5718 control_flags(),
5719 0,
5720 0,
5721 frame.header.corr,
5722 body,
5723 ) else {
5724 return;
5725 };
5726 if sink.send(next).await.is_err() {
5727 return;
5728 }
5729 }
5730 if let Some(code) = subscription.terminal_code() {
5733 if let Ok(end) =
5734 control_error_frame(&frame, code, "activity subscriber fell behind")
5735 {
5736 let _ = sink.send(end).await;
5737 }
5738 }
5739 };
5740 let task = tokio::spawn(async move {
5744 match bound_spawn {
5745 Some(spawn) => {
5746 tokio::select! {
5747 () = stream => {}
5748 () = spawn.wait_closed() => {}
5749 }
5750 }
5751 None => stream.await,
5752 }
5753 });
5754 streams.insert(key, task.abort_handle());
5755 Ok(Vec::new())
5756 }
5757
5758 async fn handle_activity_record(
5759 &self,
5760 ctx: &RouteCtx,
5761 frame: Frame,
5762 ) -> Result<Vec<Frame>, RouterError> {
5763 use subc_control::{
5764 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
5765 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
5766 };
5767 let result = async {
5768 let module = self
5769 .forwarding
5770 .module_id_for_connection(ctx.connection_id)
5771 .ok()
5772 .flatten()
5773 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
5774 let request: ActivityRecordRequest =
5775 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
5776 crate::activity::validate(&module, &request)?;
5777 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
5778 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
5779 let at_ms = std::time::SystemTime::now()
5780 .duration_since(std::time::UNIX_EPOCH)
5781 .unwrap_or_default()
5782 .as_millis() as u64;
5783 ledger.record(module, request, actor, at_ms).await
5784 }
5785 .await;
5786 match result {
5787 Ok(entry) => Ok(vec![control_response_body_frame(
5788 &frame,
5789 &ActivityRecordResponse { seq: entry.seq },
5790 "activity.record",
5791 )?]),
5792 Err(code) => Ok(vec![control_error_frame(
5793 &frame,
5794 code,
5795 "activity recording refused",
5796 )?]),
5797 }
5798 }
5799
5800 fn activity_actor(
5801 &self,
5802 connection: ConnectionId,
5803 module: &str,
5804 on: Option<&subc_control::ActivityOn>,
5805 ) -> Result<subc_control::ActivityActor, &'static str> {
5806 use crate::forwarding::{DataRoute, DataRouteState};
5807 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
5808 let principal = |id: &str| Principal::Reserved {
5809 module_id: id.to_owned(),
5810 };
5811 let mut actor = ActivityActor {
5812 principal: principal(module),
5813 scope_owner: None,
5814 agent_id: None,
5815 run_id: None,
5816 flow_id: None,
5817 project_root: None,
5818 workdir: None,
5819 project_id: None,
5820 };
5821 let scope = match on {
5822 None => return Ok(actor),
5823 Some(ActivityOn::Scope(scope)) => {
5824 let selector = subc_protocol::scope::ScopeSelector::new(
5825 principal(module),
5826 scope.scope_ref.clone(),
5827 Some(scope.epoch),
5828 );
5829 self.scopes
5830 .read()
5831 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5832 .attribute(&principal(module), module, &selector, true)
5833 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5834 }
5835 Some(ActivityOn::Route(route)) => {
5836 let DataRoute::Module(DataRouteState::Bound(binding)) = self
5837 .forwarding
5838 .lookup_data_route(connection, route.channel, route.epoch)
5839 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5840 else {
5841 return Err(ACTIVITY_ACTOR_UNKNOWN);
5842 };
5843 if binding.module_id != module {
5844 return Err(ACTIVITY_ACTOR_UNKNOWN);
5845 }
5846 actor.principal = binding.principal.clone();
5847 actor.project_root = binding
5848 .project_root
5849 .as_ref()
5850 .map(|root| root.as_path().to_path_buf());
5851 let Some(bound) = &binding.scope else {
5852 return Ok(actor);
5853 };
5854 let owner = principal(&bound.owner);
5855 let selector = subc_protocol::scope::ScopeSelector::new(
5856 owner.clone(),
5857 bound.scope_ref.clone(),
5858 Some(bound.tag.scope_epoch),
5859 );
5860 let admitted = self
5861 .scopes
5862 .read()
5863 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5864 .attribute(&owner, module, &selector, true)
5865 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
5866 if admitted.tag != bound.tag {
5867 return Err(ACTIVITY_ACTOR_UNKNOWN);
5868 }
5869 admitted
5870 }
5871 };
5872 actor.scope_owner = Some(scope.stamp.owner);
5873 actor.agent_id = scope.stamp.attributes.agent_id;
5874 actor.run_id = scope.stamp.attributes.run_id;
5875 actor.flow_id = scope.stamp.attributes.flow_id;
5876 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
5877 actor.project_id = scope.stamp.project_id;
5878 Ok(actor)
5879 }
5880
5881 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5882 self.refresh_capability_requirements();
5883 let mut counters = self.counters.snapshot();
5889 let losses = self.resource_tracker.loss_diagnostics();
5890 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
5891 if let Some(obj) = counters.as_object_mut() {
5892 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
5893 }
5894 }
5895 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
5896 obj.insert(
5897 "activity_unparseable_lines".into(),
5898 ledger.read_errors().into(),
5899 );
5900 obj.insert(
5901 "activity_lifecycle_append_failures".into(),
5902 ledger.lifecycle_failures().into(),
5903 );
5904 obj.insert(
5905 "activity_started_digest_skips".into(),
5906 ledger.lifecycle_digest_skips().into(),
5907 );
5908 }
5909 if let (Ok((connections_with_routes, max)), Some(obj)) = (
5910 self.forwarding.client_route_concentration(),
5911 counters.as_object_mut(),
5912 ) {
5913 obj.insert(
5914 "client_connections_with_routes".into(),
5915 connections_with_routes.into(),
5916 );
5917 obj.insert("max_routes_on_one_connection".into(), max.into());
5918 }
5919 if let (Some(open_breakers), Some(obj)) = (
5925 self.route_bind_breakers.open_snapshot(),
5926 counters.as_object_mut(),
5927 ) {
5928 obj.insert("route_bind_breakers_open".into(), open_breakers);
5929 }
5930 if let (Some(reason), Some(obj)) = (
5935 self.local_endpoint_unavailable.clone(),
5936 counters.as_object_mut(),
5937 ) {
5938 obj.insert("local_endpoint_unavailable".into(), reason.into());
5939 }
5940 let response = ClientControlResponse::ServerDescribe {
5941 protocol_ver: PROTOCOL_VERSION,
5942 subc_ops: subc_ops(),
5943 capabilities: self.describe_capabilities(),
5944 connected_clients: self.connected_clients.count(),
5945 counters: Some(counters),
5946 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
5947 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
5948 capability_requirements: self.capability_requirement_statuses(),
5949 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
5950 config_path: self
5951 .rescan
5952 .as_ref()
5953 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
5954 };
5955 Ok(vec![control_response_body_frame(
5956 &frame,
5957 &response,
5958 "ClientControlResponse::ServerDescribe",
5959 )?])
5960 }
5961
5962 fn handle_catalog_list(
5963 &self,
5964 frame: Frame,
5965 module_id: Option<String>,
5966 ) -> Result<Vec<Frame>, RouterError> {
5967 let (generation, modules) = self
5968 .registry
5969 .list_modules_with_nonce_verification()
5970 .map_err(|err| {
5971 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
5972 })?;
5973 let entries = modules
5974 .into_iter()
5975 .filter(|(registration, _)| {
5976 module_id
5977 .as_deref()
5978 .map(|wanted| registration.manifest.module_id == wanted)
5979 .unwrap_or(true)
5980 })
5981 .map(|(registration, nonce_verified)| {
5982 let entry = self
5983 .catalog_discovery_entry(®istration, module_id.is_some())
5984 .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
5985 let not_ready = self.not_ready_reason(®istration);
5986 let mut entry = entry;
5987 entry.roles = registration.manifest.provides;
5988 entry.control_ops = registration.control_ops;
5989 Ok(entry
5990 .with_ready(not_ready.is_none())
5991 .with_display_name(registration.manifest.display_name)
5992 .with_not_ready(not_ready)
5993 .with_module_version(Some(registration.manifest.module_version))
5994 .with_provenance(registration.manifest.provenance)
5995 .with_capabilities(registration.manifest.capabilities)
5996 .with_self_signals(registration.manifest.self_signals)
5997 .with_events(registration.manifest.events)
5998 .with_operation_delivery(registration.manifest.operation_delivery)
5999 .with_operation_payload(registration.manifest.operation_payload)
6000 .with_flow_operations(registration.manifest.flow_operations)
6001 .with_operation_deferral(registration.manifest.operation_deferral)
6002 .with_confirm_operations(registration.manifest.confirm_operations)
6003 .with_rich_ui(registration.manifest.rich_ui)
6004 .with_role_declarations(registration.manifest.role_declarations)
6005 .with_nonce_verified(nonce_verified))
6006 })
6007 .collect::<Result<Vec<_>, RouterError>>()?;
6008 let response = ClientControlResponse::CatalogList {
6009 generation,
6010 modules: entries,
6011 subc_ops: subc_ops(),
6012 };
6013 Ok(vec![control_response_body_frame(
6014 &frame,
6015 &response,
6016 "ClientControlResponse::CatalogList",
6017 )?])
6018 }
6019
6020 fn route_open_caller_from_frame(&self, ctx: &RouteCtx, frame: &Frame) -> RouteOpenCaller {
6024 let Ok(ClientControlRequest::RouteOpen {
6025 identity,
6026 consumer_identity,
6027 scope,
6028 ..
6029 }) = parse_client_control_request(&frame.body)
6030 else {
6031 return RouteOpenCaller::unknown();
6032 };
6033 let principal = self
6034 .attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref())
6035 .map(|attested| attested.principal);
6036 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
6037 }
6038
6039 fn route_open_principal(
6040 &self,
6041 connection_id: ConnectionId,
6042 frame: &Frame,
6043 consumer_identity: Option<ConsumerIdentity>,
6044 ) -> Result<Result<Principal, Frame>, RouterError> {
6045 let attested = self
6046 .attest_route_open_consumer(connection_id, consumer_identity.as_ref())
6047 .map(|attested| attested.principal);
6048 self.route_open_principal_attested(frame, consumer_identity, attested)
6049 }
6050
6051 fn route_open_principal_attested(
6055 &self,
6056 frame: &Frame,
6057 consumer_identity: Option<ConsumerIdentity>,
6058 attested: Option<Principal>,
6059 ) -> Result<Result<Principal, Frame>, RouterError> {
6060 if let Some(principal) = attested {
6061 return Ok(Ok(principal));
6062 }
6063 Ok(Err(control_error_frame(
6064 frame,
6065 "bad_consumer_identity",
6066 consumer_identity.map_or_else(
6067 || "connection claim is no longer authorized for routes".to_string(),
6068 |identity| format!(
6069 "consumer_identity for module_id '{}' conflicts with its connection claim or did not match a supervised launch nonce",
6070 identity.module_id
6071 ),
6072 ),
6073 )?))
6074 }
6075
6076 #[allow(clippy::too_many_arguments)]
6083 fn route_open_refusal_frame(
6084 &self,
6085 ctx: &RouteCtx,
6086 caller: &RouteOpenCaller,
6087 frame: &Frame,
6088 module_id: &str,
6089 reason: &'static str,
6090 code: &'static str,
6091 message: impl Into<String>,
6092 ) -> Result<Frame, RouterError> {
6093 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
6094 let detail = if matches!(
6095 reason,
6096 "registration_not_active"
6097 | "supervisor_not_live"
6098 | "no_forwarding_connection"
6099 | "reloading"
6100 ) {
6101 let expected_back = (reason == "reloading").then_some(true);
6104 self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
6105 } else {
6106 None
6107 };
6108 control_error_body_frame(
6109 frame,
6110 ErrorBody {
6111 code: code.to_string(),
6112 message: message.into(),
6113 detail,
6114 },
6115 )
6116 }
6117
6118 fn route_open_recovery_detail(
6119 &self,
6120 module_id: &str,
6121 corr: u64,
6122 detail: Option<serde_json::Value>,
6123 expected_back: Option<bool>,
6124 ) -> Result<Option<serde_json::Value>, RouterError> {
6125 let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
6126 return Ok(detail);
6127 };
6128 let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
6129 detail["state"] = serde_json::json!(status.state.to_string());
6130 if let Some(expected_back) = expected_back {
6131 detail["expected_back"] = serde_json::json!(expected_back);
6132 } else if let Some(module) = self.supervisor.get(module_id) {
6133 match module.will_recover_after_connection_loss() {
6134 Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
6135 Err(err) => warn!(
6136 %module_id,
6137 error = %err,
6138 "failed to read route.open recovery verdict; omitting expected_back"
6139 ),
6140 }
6141 }
6142 Ok(Some(detail))
6143 }
6144
6145 #[allow(clippy::too_many_arguments)]
6154 fn route_bind_rejection(
6155 &self,
6156 ctx: &RouteCtx,
6157 caller: &RouteOpenCaller,
6158 frame: &Frame,
6159 target_module_id: &str,
6160 body: ErrorBody,
6161 daemon_authored: bool,
6162 ) -> Result<Vec<Frame>, RouterError> {
6163 if daemon_authored {
6169 let scope_code = match body.code.as_str() {
6170 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
6171 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
6172 _ => None,
6173 };
6174 if let Some(code) = scope_code {
6175 self.observe_route_open_refusal(
6176 ctx,
6177 caller,
6178 target_module_id,
6179 "scope_changed_before_commit",
6180 code,
6181 );
6182 return Ok(vec![control_error_body_frame(frame, body)?]);
6183 }
6184 }
6185 self.counters
6186 .increment_route_open_refused("module_rejected");
6187 info!(
6188 target: "control",
6189 code = "module_rejected",
6190 module_code = ?body.code,
6191 module_id = ?target_module_id,
6192 connection_id = ctx.connection_id.get(),
6193 principal = %caller.principal,
6194 scope = %caller.scope,
6195 harness = %caller.harness,
6196 "route.open refused"
6197 );
6198 let refusal = if daemon_authored {
6199 control_error_body_frame(frame, body)?
6200 } else {
6201 control_relayed_error_body_frame(frame, body)?
6202 };
6203 Ok(vec![refusal])
6204 }
6205
6206 #[allow(clippy::too_many_arguments)]
6224 fn route_open_breaker_refusal_frame(
6225 &self,
6226 ctx: &RouteCtx,
6227 caller: &RouteOpenCaller,
6228 frame: &Frame,
6229 module_id: &str,
6230 consecutive_timeouts: u32,
6231 retry_in: Duration,
6232 probe_in_flight: bool,
6233 ) -> Result<Frame, RouterError> {
6234 self.counters
6235 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
6236 debug!(
6237 target: "control",
6238 code = "module_timeout",
6239 module_id = ?module_id,
6240 connection_id = ctx.connection_id.get(),
6241 principal = %caller.principal,
6242 scope = %caller.scope,
6243 harness = %caller.harness,
6244 consecutive_timeouts,
6245 retry_in_ms = retry_in.as_millis() as u64,
6246 probe_in_flight,
6247 "route.open refused by open bind-relay breaker"
6248 );
6249 let detail = if probe_in_flight {
6256 "one test route is already being tried; retry once it settles".to_string()
6257 } else {
6258 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
6259 };
6260 control_error_frame(
6261 frame,
6262 "module_timeout",
6263 format!(
6264 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
6265 timed out in a row); {detail}; its established routes are unaffected"
6266 ),
6267 )
6268 }
6269
6270 fn observe_route_open_refusal(
6283 &self,
6284 ctx: &RouteCtx,
6285 caller: &RouteOpenCaller,
6286 module_id: &str,
6287 reason: &'static str,
6288 code: &'static str,
6289 ) {
6290 self.counters.increment_route_open_refused(code);
6291 info!(
6292 target: "control",
6293 code,
6294 reason,
6295 module_id = ?module_id,
6296 connection_id = ctx.connection_id.get(),
6297 principal = %caller.principal,
6298 scope = %caller.scope,
6299 harness = %caller.harness,
6300 "route.open refused"
6301 );
6302 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
6303 self.route_outages.record_not_serving(module_id, reason);
6304 }
6305 }
6306
6307 fn observe_route_open_accept(
6352 &self,
6353 ctx: &RouteCtx,
6354 module_id: &str,
6355 principal: &str,
6356 scope: &str,
6357 ) {
6358 self.route_outages.record_accepted(module_id);
6359 self.counters.increment_route_open_accepted(principal);
6360 info!(
6361 target: "control",
6362 principal,
6363 module_id,
6364 connection_id = ctx.connection_id.get(),
6365 scope = %scope,
6366 "route.open accepted"
6367 );
6368 }
6369
6370 fn supervised_absent_route_open_refusal_frame(
6371 &self,
6372 ctx: &RouteCtx,
6373 caller: &RouteOpenCaller,
6374 frame: &Frame,
6375 module_id: &str,
6376 code: &'static str,
6377 status: &crate::supervise::ModuleStatus,
6378 ) -> Result<Frame, RouterError> {
6379 self.counters.increment_route_open_refused(code);
6380 info!(
6381 target: "control",
6382 code,
6383 reason = "supervised_not_registered",
6384 module_id = ?module_id,
6385 connection_id = ctx.connection_id.get(),
6386 principal = %caller.principal,
6387 scope = %caller.scope,
6388 harness = %caller.harness,
6389 state = %status.state,
6390 enabled = status.enabled,
6391 live = status.live,
6392 "route.open refused"
6393 );
6394 self.route_outages
6398 .record_not_serving(module_id, "supervised_not_registered");
6399 let message = if let Some(pid) = status.waiting_for_previous_pid {
6400 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
6401 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
6402 })
6403 } else {
6404 format!(
6405 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
6406 status.state, status.enabled, status.live
6407 )
6408 };
6409 let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
6413 control_error_body_frame(
6414 frame,
6415 ErrorBody {
6416 code: code.to_string(),
6417 message,
6418 detail: self.route_open_recovery_detail(
6419 module_id,
6420 frame.header.corr,
6421 None,
6422 expected_back,
6423 )?,
6424 },
6425 )
6426 }
6427
6428 async fn handle_route_open(
6429 &self,
6430 ctx: &RouteCtx,
6431 frame: Frame,
6432 request: RouteOpenRequest,
6433 timings: &mut RouteOpenTimings,
6434 ) -> Result<Vec<Frame>, RouterError> {
6435 let RouteOpenRequest {
6436 target,
6437 mut identity,
6438 consumer_identity,
6439 consumer_capabilities,
6440 role_versions,
6441 busy_refusals,
6442 admission_facts,
6443 scope,
6444 accepts_close_reasons,
6445 } = request;
6446 let target_module_id = target_module_id(&target).to_string();
6447 timings.module_id.clone_from(&target_module_id);
6448 if self
6449 .registry
6450 .get_module_by_connection(ctx.connection_id)
6451 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6452 .is_some()
6453 {
6454 return Ok(vec![control_error_frame(
6455 &frame,
6456 "invalid_request",
6457 "module connections cannot open client routes",
6458 )?]);
6459 }
6460 debug!(
6461 connection_id = ctx.connection_id.get(),
6462 corr = frame.header.corr,
6463 module_id = %target_module_id,
6464 "handling route.open"
6465 );
6466
6467 let attested =
6473 self.attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref());
6474 let mut caller = RouteOpenCaller::new(
6475 attested.as_ref().map(|attested| &attested.principal),
6476 scope.as_ref(),
6477 &identity,
6478 );
6479
6480 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
6486 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
6487 self.observe_route_open_refusal(
6488 ctx,
6489 &caller,
6490 &target_module_id,
6491 "invalid_role_versions",
6492 error_codes::INVALID_REQUEST,
6493 );
6494 return Ok(vec![control_error_body_frame(
6495 &frame,
6496 ErrorBody {
6497 code: error_codes::INVALID_REQUEST.to_string(),
6498 message: error.to_string(),
6499 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
6500 },
6501 )?]);
6502 }
6503
6504 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
6505 if status.waiting_for_previous_pid.is_some() {
6506 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6507 ctx,
6508 &caller,
6509 &frame,
6510 &target_module_id,
6511 "module_warming",
6512 &status,
6513 )?]);
6514 }
6515 }
6516
6517 let admission_registry = self
6536 .registry
6537 .route_admission_snapshot()
6538 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6539 let Some(registration) =
6540 crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
6541 else {
6542 if let Some((status, warming)) =
6543 self.supervisor_status(&target_module_id, frame.header.corr)?
6544 {
6545 if status.protocol == ModuleProtocol::None {
6552 return Ok(vec![self.route_open_refusal_frame(
6553 ctx,
6554 &caller,
6555 &frame,
6556 &target_module_id,
6557 "protocol_none",
6558 error_codes::MODULE_NO_PROTOCOL,
6559 format!(
6560 "module_id '{target_module_id}' is declared protocol: none; \
6561 it speaks no subc wire and serves no routes"
6562 ),
6563 )?]);
6564 }
6565 let code = if warming {
6566 "module_warming"
6567 } else {
6568 "target_unavailable"
6569 };
6570 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6571 ctx,
6572 &caller,
6573 &frame,
6574 &target_module_id,
6575 code,
6576 &status,
6577 )?]);
6578 }
6579 if let Some(removed_ago_ms) =
6580 self.supervisor.removal_tombstone_age_ms(&target_module_id)
6581 {
6582 return Ok(vec![self.route_open_refusal_frame(
6583 ctx,
6584 &caller,
6585 &frame,
6586 &target_module_id,
6587 "removed",
6588 error_codes::MODULE_REMOVED,
6589 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
6590 )?]);
6591 }
6592 return Ok(vec![self.route_open_refusal_frame(
6593 ctx,
6594 &caller,
6595 &frame,
6596 &target_module_id,
6597 "not_registered",
6598 error_codes::UNKNOWN_MODULE,
6599 format!("module_id '{target_module_id}' is not registered"),
6600 )?]);
6601 };
6602
6603 if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
6607 .is_err()
6608 {
6609 self.counters
6610 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
6611 info!(
6612 target: "control",
6613 code = error_codes::MODULE_WARMING,
6614 module_id = ?target_module_id,
6615 connection_id = ctx.connection_id.get(),
6616 principal = %caller.principal,
6617 scope = %caller.scope,
6618 harness = %caller.harness,
6619 reason = "declared_not_ready",
6620 "route.open refused"
6621 );
6622 self.route_outages
6625 .record_not_serving(&target_module_id, "declared_not_ready");
6626 return Ok(vec![control_error_body_frame(
6629 &frame,
6630 ErrorBody {
6631 code: error_codes::MODULE_WARMING.to_string(),
6632 message: format!(
6633 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
6634 ),
6635 detail: self.route_open_recovery_detail(
6636 &target_module_id,
6637 frame.header.corr,
6638 Some(serde_json::json!({ "reason": "declared_not_ready" })),
6639 Some(true),
6640 )?,
6641 },
6642 )?]);
6643 }
6644
6645 if let Some(capability) = self.first_unprovided_required_capability(registration) {
6664 self.counters
6665 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
6666 info!(
6667 target: "control",
6668 code = error_codes::MODULE_WARMING,
6669 module_id = ?target_module_id,
6670 connection_id = ctx.connection_id.get(),
6671 principal = %caller.principal,
6672 scope = %caller.scope,
6673 harness = %caller.harness,
6674 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6675 capability = %capability,
6676 "route.open refused"
6677 );
6678 return Ok(vec![control_error_body_frame(
6679 &frame,
6680 ErrorBody {
6681 code: error_codes::MODULE_WARMING.to_string(),
6682 message: format!(
6683 "module_id '{target_module_id}' requires capability '{capability}', \
6684 which no registered module provides; retry"
6685 ),
6686 detail: Some(serde_json::json!({
6687 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6688 "capability": capability,
6689 })),
6690 },
6691 )?]);
6692 }
6693
6694 if !target_has_required_role(&target, ®istration.manifest.provides) {
6695 return Ok(vec![self.route_open_refusal_frame(
6696 ctx,
6697 &caller,
6698 &frame,
6699 &target_module_id,
6700 "role_not_provided",
6701 "target_unavailable",
6702 format!("module_id '{target_module_id}' does not provide the requested target"),
6703 )?]);
6704 }
6705
6706 if registration.state != ChannelState::Active {
6707 return Ok(vec![self.route_open_refusal_frame(
6708 ctx,
6709 &caller,
6710 &frame,
6711 &target_module_id,
6712 "registration_not_active",
6713 "target_unavailable",
6714 format!("module_id '{target_module_id}' is not active"),
6715 )?]);
6716 }
6717
6718 if self
6719 .forwarding
6720 .module_is_draining(&target_module_id)
6721 .map_err(RouterError::Forwarding)?
6722 {
6723 return Ok(vec![self.route_open_refusal_frame(
6724 ctx,
6725 &caller,
6726 &frame,
6727 &target_module_id,
6728 "reloading",
6729 "module_reloading",
6730 format!("module_id '{target_module_id}' is reloading"),
6731 )?]);
6732 }
6733
6734 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
6735 process_liveness.process_live(&target_module_id) == Some(false)
6736 }) {
6737 if process_liveness.process_replacing(&target_module_id) {
6744 return Ok(vec![self.route_open_refusal_frame(
6745 ctx,
6746 &caller,
6747 &frame,
6748 &target_module_id,
6749 "reloading",
6750 "module_reloading",
6751 format!("module_id '{target_module_id}' is reloading"),
6752 )?]);
6753 }
6754 return Ok(vec![self.route_open_refusal_frame(
6755 ctx,
6756 &caller,
6757 &frame,
6758 &target_module_id,
6759 "supervisor_not_live",
6760 "target_unavailable",
6761 format!("module_id '{target_module_id}' is not live"),
6762 )?]);
6763 }
6764
6765 if !self
6766 .forwarding
6767 .has_live_module_connection(&target_module_id)
6768 .map_err(RouterError::Forwarding)?
6769 {
6770 return Ok(vec![self.route_open_refusal_frame(
6771 ctx,
6772 &caller,
6773 &frame,
6774 &target_module_id,
6775 "no_forwarding_connection",
6776 "target_unavailable",
6777 format!("module_id '{target_module_id}' has no live forwarding connection"),
6778 )?]);
6779 }
6780
6781 if let Some(error) =
6782 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
6783 {
6784 self.observe_route_open_refusal(
6785 ctx,
6786 &caller,
6787 &target_module_id,
6788 "op_not_allowed",
6789 "op_not_allowed",
6790 );
6791 return Ok(vec![error]);
6792 }
6793
6794 let principal = match (&attested, consumer_identity.is_some()) {
6795 (None, false) => {
6798 self.observe_route_open_refusal(
6799 ctx,
6800 &caller,
6801 &target_module_id,
6802 "registration_channel_closed",
6803 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6804 );
6805 return Ok(vec![control_error_frame(
6806 &frame,
6807 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6808 "the claim binding's launch has ended",
6809 )?]);
6810 }
6811 _ => match self.route_open_principal_attested(
6812 &frame,
6813 consumer_identity,
6814 attested.as_ref().map(|attested| attested.principal.clone()),
6815 )? {
6816 Ok(principal) => principal,
6817 Err(error) => {
6818 self.observe_route_open_refusal(
6819 ctx,
6820 &caller,
6821 &target_module_id,
6822 "bad_consumer_identity",
6823 "bad_consumer_identity",
6824 );
6825 return Ok(vec![error]);
6826 }
6827 },
6828 };
6829 let attested = attested.expect("an attested principal was resolved above");
6830 timings.principal = principal_label(&principal);
6831
6832 let Ok(mut caller_capture) = attested.capture() else {
6835 self.observe_route_open_refusal(
6836 ctx,
6837 &caller,
6838 &target_module_id,
6839 "registration_channel_closed",
6840 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6841 );
6842 return Ok(vec![control_error_frame(
6843 &frame,
6844 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6845 "the opener's launch has ended",
6846 )?]);
6847 };
6848 let caller_module_id = match &attested.spawn {
6852 Some((record, _)) => Some(record.module_id().to_owned()),
6853 None => match &principal {
6854 Principal::Reserved { module_id } => Some(module_id.clone()),
6855 _ => None,
6856 },
6857 };
6858
6859 let build_class = identity.build_class.take().filter(|build_class| {
6862 if caller_module_id.as_deref() == Some("callosum") {
6863 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
6864 } else {
6865 if self
6866 .build_class_warned
6867 .lock()
6868 .unwrap_or_else(|p| p.into_inner())
6869 .insert(ctx.connection_id)
6870 {
6871 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
6872 "ignoring build_class from a non-Callosum opener");
6873 }
6874 false
6875 }
6876 });
6877
6878 if let Some(opening_registration) = self.caller_registration(&attested, &admission_registry)
6882 {
6883 let opening_module_id = opening_registration.manifest.module_id.as_str();
6884 if let Some(capability) =
6885 denied_capability(&opening_registration.manifest, ®istration.manifest)
6886 {
6887 warn!(
6888 opening_module_id,
6889 target_module_id,
6890 capability,
6891 "refusing route.open because an attested capability deny edge matches"
6892 );
6893 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6894 &frame,
6895 &target_module_id,
6896 "capability_deny_edge",
6897 "capability_forbidden",
6898 format!(
6899 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
6900 ),
6901 )?]);
6902 }
6903 }
6904
6905 if admission_facts.is_some() {
6906 let carrier_matches = caller_module_id.is_some()
6907 && self.admission_facts_carrier_module_id.as_deref() == caller_module_id.as_deref();
6908 if !carrier_matches {
6909 return Ok(vec![self.route_open_refusal_frame(
6910 ctx,
6911 &caller,
6912 &frame,
6913 &target_module_id,
6914 "admission_facts_carrier_not_permitted",
6915 "admission_facts_not_permitted",
6916 "admission facts may only be carried by the configured reserved module",
6917 )?]);
6918 }
6919
6920 let target_allowed = self
6921 .admission_facts_targets
6922 .as_ref()
6923 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
6924 if !target_allowed {
6925 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6926 &frame,
6927 &target_module_id,
6928 "admission_facts_target_not_listed",
6929 "admission_facts_target_not_allowed",
6930 format!(
6931 "admission facts are not permitted for target module_id '{target_module_id}'"
6932 ),
6933 )?]);
6934 }
6935
6936 }
6940
6941 let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
6947 None => (None, None, false),
6948 Some(selector) => {
6949 let owner_configured = match &selector.owner {
6950 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
6956 _ => false,
6957 };
6958 let mut scopes = self
6959 .scopes
6960 .write()
6961 .unwrap_or_else(|poisoned| poisoned.into_inner());
6962 let admitted = crate::scopes::admission::admit_route(
6963 &scopes,
6964 &admission_registry,
6965 &principal,
6966 &target_module_id,
6967 Some(selector),
6968 owner_configured,
6969 crate::scopes::admission::AdmissionMode::Open,
6970 )
6971 .map(|route| {
6972 let admission = route.scope.expect("scoped admission");
6973 let bound = scopes.reserve_bind(
6974 &admission,
6975 &principal,
6976 ctx.connection_id,
6977 selector.hold_presence,
6978 );
6979 (admission, bound)
6980 });
6981 match admitted {
6982 Ok((admission, bound)) => {
6983 (Some(bound), Some(admission.stamp), admission.stamp_required)
6984 }
6985 Err(refusal) => {
6986 return Ok(vec![self.route_open_refusal_frame(
6987 ctx,
6988 &caller,
6989 &frame,
6990 &target_module_id,
6991 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
6992 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
6993 refusal.message,
6994 )?]);
6995 }
6996 }
6997 }
6998 };
6999 if let Some(stamp) = &scope_stamp {
7002 caller.scope = route_open_scope_label(stamp);
7003 }
7004 if scope_stamp.as_ref().is_some_and(scope_acts_for_another) {
7007 caller_capture = caller_capture.map(crate::spawn_authority::CallerCapture::relayed);
7008 }
7009
7010 timings.enter(RouteOpenPhase::ProjectRoot);
7018 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
7019 Ok(project) => project,
7020 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
7021 };
7022 timings.enter(RouteOpenPhase::Admission);
7023 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
7024 Ok(root) => root,
7025 Err(err) => {
7026 return Ok(vec![control_error_frame(
7027 &frame,
7028 "invalid_project_root",
7029 err.to_string(),
7030 )?])
7031 }
7032 };
7033 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
7039
7040 let _concurrency_guard = match self
7051 .route_bind_concurrency
7052 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
7053 {
7054 Ok(guard) => guard,
7055 Err(in_flight) => {
7056 return Ok(vec![self.route_open_target_capacity_refusal(
7057 ctx,
7058 &caller,
7059 &frame,
7060 &target_module_id,
7061 in_flight,
7062 )?]);
7063 }
7064 };
7065
7066 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
7069 RouteBindAdmission::Admitted { guard, probe } => {
7070 if probe {
7071 info!(
7072 module_id = %target_module_id,
7073 connection_id = ctx.connection_id.get(),
7074 "route.bind breaker half-open: admitting one probe"
7075 );
7076 }
7077 guard
7078 }
7079 RouteBindAdmission::Refused {
7080 consecutive_timeouts,
7081 retry_in,
7082 probe_in_flight,
7083 } => {
7084 return Ok(vec![self.route_open_breaker_refusal_frame(
7085 ctx,
7086 &caller,
7087 &frame,
7088 &target_module_id,
7089 consecutive_timeouts,
7090 retry_in,
7091 probe_in_flight,
7092 )?]);
7093 }
7094 };
7095
7096 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
7101 let relay_deadline = Instant::now() + route_bind_relay_timeout;
7102 let operator_app = self
7103 .operator_apps
7104 .lock()
7105 .unwrap_or_else(|p| p.into_inner())
7106 .get(&ctx.connection_id)
7107 .cloned();
7108 timings.enter(RouteOpenPhase::EgressReservation);
7109 let pending = match self
7110 .forwarding
7111 .begin_route_bind_relay_for(
7112 ctx.connection_id,
7113 ctx.egress.clone(),
7114 response_version(&frame),
7115 frame.header.corr,
7116 &target_module_id,
7117 principal.clone(),
7118 bound_scope,
7119 project_root,
7120 identity.harness.clone(),
7121 build_class,
7122 operator_app.clone(),
7123 crate::forwarding::RouteAuthority {
7124 incarnation: self.supervisor.daemon_incarnation(),
7125 caller: caller_capture
7126 .as_ref()
7127 .map(|caller| caller.authority.pair()),
7128 },
7129 relay_deadline,
7130 busy_refusals == Some(true),
7131 accepts_close_reasons,
7132 )
7133 .await
7134 {
7135 Ok(pending) => pending,
7136 Err(err) => {
7137 return Ok(vec![self.route_open_refusal_frame(
7138 ctx,
7139 &caller,
7140 &frame,
7141 &target_module_id,
7142 "relay_reservation_failed",
7143 forwarding_error_code(&err),
7144 err.to_string(),
7145 )?])
7146 }
7147 };
7148 timings.enter(RouteOpenPhase::Admission);
7149 let crate::forwarding::PendingRouteBindRelay {
7150 endpoint,
7151 module_sink,
7152 negotiated_ver,
7153 client_channel,
7154 client_epoch,
7155 module_channel,
7156 module_epoch,
7157 corr: relay_corr,
7158 receiver,
7159 } = pending;
7160 let mut reservation =
7161 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
7162
7163 let relay_registration = self
7168 .registry
7169 .get_module_by_connection(endpoint.connection_id)
7170 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7171 let relay_capabilities = relay_registration
7172 .as_ref()
7173 .and_then(|registration| registration.manifest.capabilities.as_ref());
7174 if let Some(selector) = &scope {
7175 if let Err(refusal) = self
7178 .scopes
7179 .read()
7180 .unwrap_or_else(|poisoned| poisoned.into_inner())
7181 .check_bind_proof(&principal, selector)
7182 {
7183 reservation.release_and_disarm();
7184 return Ok(vec![self.route_open_refusal_frame(
7185 ctx,
7186 &caller,
7187 &frame,
7188 &target_module_id,
7189 refusal.code,
7190 refusal.code,
7191 refusal.message,
7192 )?]);
7193 }
7194 }
7195 if let Some(stamp) = scope_stamp.as_ref() {
7196 if let Err(refusal) = crate::scopes::check_target_required_stamp(
7197 stamp_required,
7198 &target_module_id,
7199 relay_capabilities,
7200 )
7201 .and_then(|()| {
7202 crate::scopes::check_target_flow_support(
7203 stamp,
7204 &target_module_id,
7205 relay_capabilities,
7206 )
7207 })
7208 .and_then(|()| {
7209 crate::scopes::check_target_agent_run_support(
7210 stamp,
7211 &target_module_id,
7212 relay_capabilities,
7213 )
7214 }) {
7215 reservation.release_and_disarm();
7216 return Ok(vec![self.route_open_refusal_frame(
7217 ctx,
7218 &caller,
7219 &frame,
7220 &target_module_id,
7221 refusal.code,
7222 refusal.code,
7223 refusal.message,
7224 )?]);
7225 }
7226 }
7227
7228 if scope_stamp
7231 .as_ref()
7232 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
7233 && !crate::scopes::target_supports_federated(relay_capabilities)
7234 {
7235 scope_stamp = None;
7237 }
7238 if !relay_capabilities.is_some_and(|caps| {
7239 caps.provides
7240 .iter()
7241 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
7242 }) {
7243 if let Some(stamp) = scope_stamp.as_mut() {
7244 stamp.workdir = None;
7245 stamp.project_id = None;
7246 }
7247 }
7248 let worker_run = scope_stamp
7253 .as_ref()
7254 .is_some_and(crate::scopes::is_worker_run)
7255 && relay_capabilities.is_some_and(|caps| {
7256 caps.provides
7257 .iter()
7258 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
7259 });
7260 if !worker_run
7261 && !relay_capabilities.is_some_and(|caps| {
7262 caps.provides
7263 .iter()
7264 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
7265 })
7266 {
7267 if let Some(stamp) = scope_stamp.as_mut() {
7268 stamp.attributes.on_behalf_of_agent = None;
7269 }
7270 }
7271 if !relay_capabilities.is_some_and(|caps| {
7272 caps.provides
7273 .iter()
7274 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
7275 }) {
7276 if let Some(stamp) = scope_stamp.as_mut() {
7277 stamp.attributes.flow_version = None;
7278 stamp.attributes.flow_code_hash = None;
7279 stamp.attributes.flow_install_id = None;
7280 }
7281 }
7282 let call_mode = if worker_run
7283 || relay_capabilities
7284 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
7285 {
7286 Some(derive_call_mode(scope_stamp.as_ref()))
7287 } else {
7288 if let Some(stamp) = scope_stamp.as_mut() {
7289 stamp.expires_at_ms = None;
7290 }
7291 None
7292 };
7293
7294 debug!(
7295 connection_id = ctx.connection_id.get(),
7296 client_channel,
7297 client_epoch,
7298 module_channel,
7299 module_epoch,
7300 "reserved route handle pair"
7301 );
7302 let route_principal_label = principal_label(&principal);
7305 let relay = ModuleControlRequest::RouteBind {
7306 route_channel: module_channel,
7307 epoch: module_epoch,
7308 target,
7309 identity,
7310 principal: Some(principal),
7311 consumer_capabilities,
7312 role_versions,
7313 admission_facts,
7314 scope: scope_stamp,
7315 call_mode,
7316 };
7317 let peer = RouteBindPeer::new()
7318 .with_observed_peer(
7319 relay_capabilities
7320 .is_some_and(|caps| {
7321 caps.provides
7322 .iter()
7323 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
7324 })
7325 .then(|| {
7326 self.observed_peers
7327 .lock()
7328 .unwrap_or_else(|p| p.into_inner())
7329 .get(&ctx.connection_id)
7330 .cloned()
7331 .unwrap_or_else(ObservedPeer::unobserved)
7332 }),
7333 )
7334 .with_operator_app(
7335 self.connection_operator_app(operator_app.as_ref())
7336 .filter(|_| {
7337 relay_capabilities.is_some_and(|caps| {
7338 caps.provides
7339 .iter()
7340 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
7341 })
7342 }),
7343 )
7344 .with_caller(caller_for_target(
7345 caller_capture.as_ref(),
7346 relay_capabilities,
7347 &target_module_id,
7348 ));
7349 let relay_body = crate::bind_compat::encode_bind(
7350 &relay,
7351 &peer,
7352 &legacy_bind_project,
7353 relay_capabilities,
7354 )
7355 .map_err(|err| {
7356 RouterError::backend(
7357 0,
7358 frame.header.corr,
7359 format!("failed to encode route.bind request: {err}"),
7360 )
7361 })?;
7362 let relay_frame = Frame::build_with_version(
7363 negotiated_ver,
7364 FrameType::Request,
7365 control_flags(),
7366 0,
7367 0,
7368 relay_corr,
7369 relay_body,
7370 )
7371 .map_err(RouterError::FrameBuild)?;
7372
7373 timings.enter(RouteOpenPhase::BindSend);
7374 let sent = module_sink.send(relay_frame).await;
7375 timings.enter(RouteOpenPhase::Admission);
7376 if let Err(err) = sent {
7377 reservation.release_and_disarm();
7378 return Ok(vec![self.route_open_refusal_frame(
7379 ctx,
7380 &caller,
7381 &frame,
7382 &target_module_id,
7383 "relay_send_failed",
7384 "target_unavailable",
7385 err.to_string(),
7386 )?]);
7387 }
7388
7389 #[cfg(test)]
7390 if let Some(hold) = &self.route_bind_relay_sent_hold {
7391 hold.wait().await;
7395 hold.wait().await;
7396 }
7397
7398 if self
7403 .forwarding
7404 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
7405 .map_err(RouterError::Forwarding)?
7406 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
7407 {
7408 self.send_abandoned_route_bind_goodbye(
7409 &module_sink,
7410 negotiated_ver,
7411 module_channel,
7412 module_epoch,
7413 );
7414 }
7415
7416 timings.enter(RouteOpenPhase::BindAckWait);
7417 let outcome = timeout_at(relay_deadline, receiver).await;
7418 timings.enter(RouteOpenPhase::Admission);
7419 match outcome {
7420 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
7421 reservation.disarm();
7422 if breaker.record_accepted() {
7423 info!(
7424 module_id = %target_module_id,
7425 "route.bind breaker closed: the probe was accepted"
7426 );
7427 }
7428 self.observe_route_open_accept(
7429 ctx,
7430 &target_module_id,
7431 &route_principal_label,
7432 &caller.scope,
7433 );
7434 Ok(Vec::new())
7435 }
7436 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
7437 reservation.release_and_disarm();
7438 breaker.record_inconclusive();
7442 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
7443 }
7444 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
7445 reservation.release_and_disarm();
7446 breaker.record_inconclusive();
7447 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
7448 }
7449 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
7450 reservation.release_and_disarm();
7451 breaker.record_inconclusive();
7452 tracing::warn!(
7459 module_id = %target_module_id,
7460 "route.bind relay abandoned: {message}"
7461 );
7462 Ok(vec![self.route_open_refusal_frame(
7463 ctx,
7464 &caller,
7465 &frame,
7466 &target_module_id,
7467 "relay_abandoned",
7468 "target_unavailable",
7469 message,
7470 )?])
7471 }
7472 Ok(Err(_)) => {
7473 reservation.release_and_disarm();
7474 breaker.record_inconclusive();
7475 Ok(vec![self.route_open_refusal_frame(
7476 ctx,
7477 &caller,
7478 &frame,
7479 &target_module_id,
7480 "relay_waiter_canceled",
7481 "target_unavailable",
7482 "route.bind relay waiter was canceled before the module responded",
7483 )?])
7484 }
7485 Err(_) => {
7486 reservation.release_and_disarm();
7487 if let Some(opened) = breaker.record_timeout(
7491 self.route_bind_breaker_threshold,
7492 self.route_bind_breaker_cooldown,
7493 ) {
7494 warn!(
7495 module_id = %target_module_id,
7496 consecutive_timeouts = opened.consecutive_timeouts,
7497 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
7498 reopened_after_probe = opened.reopened_after_probe,
7499 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
7500 );
7501 }
7502 tracing::warn!(
7510 module_id = %target_module_id,
7511 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
7512 "route.bind relay timed out: module did not ack within budget"
7513 );
7514 Ok(vec![self.route_open_refusal_frame(
7515 ctx,
7516 &caller,
7517 &frame,
7518 &target_module_id,
7519 "relay_timed_out",
7520 "module_timeout",
7521 format!(
7522 "module_id '{target_module_id}' did not answer route.bind within {:?}",
7523 route_bind_relay_timeout
7524 ),
7525 )?])
7526 }
7527 }
7528 }
7529
7530 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7531 let response = ClientControlResponse::SupervisorSpawnSnapshot {
7532 snapshot: self.supervisor.spawn_snapshot(),
7533 };
7534 Ok(vec![control_response_body_frame(
7535 &frame,
7536 &response,
7537 "ClientControlResponse::SupervisorSpawnSnapshot",
7538 )?])
7539 }
7540
7541 fn handle_supervisor_spawn_subscribe(
7542 &self,
7543 ctx: &RouteCtx,
7544 frame: Frame,
7545 since: Option<SpawnCursor>,
7546 ) -> Result<Vec<Frame>, RouterError> {
7547 match self.supervisor.subscribe_spawns(
7548 ctx.connection_id,
7549 frame.header.corr,
7550 response_version(&frame),
7551 since,
7552 ctx.egress.clone(),
7553 ) {
7554 Ok(()) => Ok(Vec::new()),
7555 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
7556 Ok(vec![control_error_body_frame(
7557 &frame,
7558 ErrorBody {
7559 code: "spawn_cursor_incarnation_mismatch".to_string(),
7560 message: "spawn cursor belongs to a different daemon incarnation"
7561 .to_string(),
7562 detail: Some(serde_json::json!({
7563 "current_daemon_incarnation": current
7564 })),
7565 },
7566 )?])
7567 }
7568 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
7569 &frame,
7570 ErrorBody {
7571 code: "spawn_cursor_too_old".to_string(),
7572 message: "spawn cursor predates the retained event ring".to_string(),
7573 detail: Some(serde_json::json!({
7574 "oldest_retained_cursor": oldest
7575 })),
7576 },
7577 )?]),
7578 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
7579 0,
7580 frame.header.corr,
7581 format!("failed to open supervisor spawn subscription: {error}"),
7582 )),
7583 }
7584 }
7585
7586 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7587 let handler = self.clone();
7590 let runtime = tokio::runtime::Handle::current();
7591 let subscriber = tracing::dispatcher::get_default(Clone::clone);
7594 let corr = frame.header.corr;
7595 tokio::task::spawn_blocking(move || {
7596 runtime.block_on(
7597 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
7598 .with_subscriber(subscriber),
7599 )
7600 })
7601 .await
7602 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
7603 }
7604
7605 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7606 use crate::config_gate::runtime::ApprovedDaemonView;
7607 let publication = self
7608 .config_runtime
7609 .as_ref()
7610 .and_then(|owner| owner.polled_snapshot().ok());
7611 let approved = publication
7612 .as_ref()
7613 .map(|view| view.approved_daemon.clone())
7614 .unwrap_or_else(|| {
7615 if self.config_runtime.is_some() {
7616 ApprovedDaemonView::Unavailable
7617 } else {
7618 ApprovedDaemonView::NotStamped
7619 }
7620 });
7621 let generation = self
7622 .registry
7623 .generation()
7624 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7625 let supervised = self.supervisor.list();
7626 let mut statuses = BTreeMap::new();
7627 let mut not_running = BTreeSet::new();
7628 for module in &supervised {
7629 let status = module.status_for_control("list").map_err(|err| {
7630 RouterError::backend(
7631 0,
7632 frame.header.corr,
7633 format!("failed to read supervisor status: {err}"),
7634 )
7635 })?;
7636 if !status.enabled || !status.process_alive {
7637 not_running.insert(module.module_id().to_owned());
7638 }
7639 statuses.insert(module.module_id().to_owned(), status);
7640 }
7641 let mut process_resources = BTreeMap::new();
7644 for module in &supervised {
7645 process_resources.insert(
7646 module.module_id().to_owned(),
7647 crate::subtree_resources::read_process(module.clone()).await,
7648 );
7649 }
7650 let mut image_readings = BTreeMap::new();
7653 for module in &supervised {
7654 image_readings.insert(
7655 module.module_id().to_owned(),
7656 module.running_image_agreement().await,
7657 );
7658 }
7659 let readings = self.resource_tracker.list(supervised.clone());
7660 let mut modules = Vec::new();
7661 for module in supervised {
7662 let status = statuses
7663 .remove(module.module_id())
7664 .expect("each supervised module has a status snapshot");
7665 let (configured, _) = module.configuration().map_err(|err| {
7666 RouterError::backend(
7667 0,
7668 frame.header.corr,
7669 format!("failed to read module configuration: {err}"),
7670 )
7671 })?;
7672 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
7673 RouterError::backend(
7674 0,
7675 frame.header.corr,
7676 format!("failed to read retry time: {err}"),
7677 )
7678 })?;
7679 let image = image_readings
7680 .remove(module.module_id())
7681 .expect("each supervised module has an image snapshot");
7682 let resources = process_resources.remove(module.module_id());
7683 let mut verdict =
7684 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
7685 verdict.spec = Some(launch_spec_agreement(
7686 &configured,
7687 status.process_alive,
7688 status.launch_spec_digest.as_ref(),
7689 ));
7690 let pending_reload = Some(verdict);
7691 let script = configured.script.as_ref().map(|path| {
7692 let approved = self
7693 .config_runtime
7694 .as_ref()
7695 .and_then(|owner| owner.approved_script(&status.module_id, path));
7696 crate::supervise::script::status(path, approved.as_ref())
7697 });
7698 let configured_enabled = match &approved {
7699 ApprovedDaemonView::Approved {
7700 config: approved, ..
7701 } => approved
7702 .modules
7703 .iter()
7704 .find(|module| module.module_id == status.module_id)
7705 .map(|module| module.enabled),
7706 ApprovedDaemonView::NotStamped => self
7707 .capability_evaluator
7708 .configured_enabled(&status.module_id),
7709 ApprovedDaemonView::Unavailable => None,
7710 };
7711 let admission_refused = status
7712 .admission_refused
7713 .clone()
7714 .filter(|_| status.enabled && !status.process_alive)
7715 .filter(|_| {
7716 publication
7717 .as_ref()
7718 .and_then(|view| view.registration_declaration(&status.module_id))
7719 .as_deref()
7720 == Some("undeclared")
7721 });
7722 modules.push(
7725 SupervisorEntry::new(
7726 status.module_id,
7727 status.state.to_string(),
7728 status.enabled,
7729 status.live,
7730 status.health.status,
7731 )
7732 .with_configured_enabled(configured_enabled)
7733 .with_script(script)
7734 .with_admission_refused(admission_refused)
7735 .with_next_retry_at_ms(next_retry_at_ms)
7736 .with_launch_nonce_env(Some(!cfg!(unix)))
7737 .with_launch_nonce_source(status.launch_nonce_source)
7738 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
7739 .with_protocol(status.protocol)
7740 .with_pending_reload(pending_reload)
7741 .with_last_probe_ms(status.health.last_probe_ms)
7742 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
7743 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
7744 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
7745 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
7746 .with_restart_count(Some(status.restart_count))
7747 .with_max_restarts(Some(status.max_restarts))
7748 .with_lifetime_restarts(Some(status.lifetime_restarts))
7749 .with_spawn_generation(Some(status.spawn_generation))
7750 .with_restart_window_secs(Some(status.restart_window.as_secs()))
7751 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
7752 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
7753 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
7754 .with_resources(resources),
7755 );
7756 }
7757 for module in &mut modules {
7758 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
7759 module.tree = tree.clone();
7760 module.cpu_last_hour = hour.clone();
7761 }
7762 module.tree_unavailable = readings
7763 .reasons
7764 .get(&module.module_id)
7765 .map(|reason| (*reason).to_owned());
7766 if module.tree.is_none() {
7767 module.tree_unavailable = Some(
7768 if not_running.contains(&module.module_id) {
7769 "not_running"
7770 } else if let Some(reason) = readings.reasons.get(&module.module_id) {
7771 reason
7772 } else {
7773 "not_yet_measured"
7774 }
7775 .to_owned(),
7776 );
7777 }
7778 }
7779 let response = ClientControlResponse::SupervisorList {
7780 generation,
7781 modules,
7782 daemon: readings.daemon,
7783 agent_commands: readings.agent_commands,
7784 };
7785 Ok(vec![control_response_body_frame(
7786 &frame,
7787 &response,
7788 "ClientControlResponse::SupervisorList",
7789 )?])
7790 }
7791
7792 fn handle_supervisor_stderr_tail(
7793 &self,
7794 frame: Frame,
7795 module_id: String,
7796 max_lines: Option<u32>,
7797 max_bytes: Option<u32>,
7798 ) -> Result<Vec<Frame>, RouterError> {
7799 let Some(module) = self.supervisor.get(&module_id) else {
7800 return Ok(vec![control_error_frame(
7801 &frame,
7802 "unknown_module",
7803 format!("module_id '{module_id}' is not supervised"),
7804 )?]);
7805 };
7806
7807 let snapshot = module.stderr_tail(
7808 max_lines.map(|value| value as usize),
7809 max_bytes.map(|value| value as usize),
7810 );
7811
7812 let response = ClientControlResponse::SupervisorStderrTail {
7813 module_id,
7814 tail: StderrTail {
7815 capture: match snapshot.capture {
7816 CaptureState::Captured => StderrCaptureState::Captured,
7817 CaptureState::Incomplete { reason } => {
7818 StderrCaptureState::Incomplete { reason }
7819 }
7820 CaptureState::NotCaptured { reason } => {
7821 StderrCaptureState::NotCaptured { reason }
7822 }
7823 },
7824 entries: snapshot
7825 .entries
7826 .into_iter()
7827 .map(|entry| match entry {
7828 TailEntry::Line {
7829 text,
7830 truncated,
7831 at_ms,
7832 } => StderrTailEntry::Line {
7833 text,
7834 truncated,
7835 at_ms,
7836 },
7837 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
7838 })
7839 .collect(),
7840 dropped_lines: snapshot.dropped_lines,
7841 },
7842 };
7843 Ok(vec![control_response_body_frame(
7844 &frame,
7845 &response,
7846 "ClientControlResponse::SupervisorStderrTail",
7847 )?])
7848 }
7849
7850 async fn observe_control_wait<F: std::future::Future>(
7851 &self,
7852 ctx: &RouteCtx,
7853 target: Option<&str>,
7854 kind: &'static str,
7855 future: F,
7856 ) -> F::Output {
7857 self.counters
7858 .observe_reader_wait(
7859 || {
7860 let module = self
7861 .forwarding
7862 .module_id_for_connection(ctx.connection_id)
7863 .ok()
7864 .flatten();
7865 let principal = if module.is_some() { "module" } else { "direct" };
7866 (
7867 self.resolve_reader_wait_target(kind, target)
7868 .or(module)
7869 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
7870 principal.into(),
7871 )
7872 },
7873 kind,
7874 future,
7875 )
7876 .await
7877 }
7878
7879 pub(crate) fn resolve_reader_wait_target(
7882 &self,
7883 op: &str,
7884 selector: Option<&str>,
7885 ) -> Option<String> {
7886 let selector = selector?;
7887 match op {
7888 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
7889 .supervisor
7890 .get(selector)
7891 .map(|module| module.module_id().to_owned()),
7892 ops::SUPERVISOR_HEALTH_PROBE => self
7893 .registry
7894 .get_module(selector)
7895 .ok()
7896 .flatten()
7897 .map(|registration| registration.manifest.module_id),
7898 _ => None,
7899 }
7900 }
7901
7902 async fn handle_supervisor_terminals(
7903 &self,
7904 ctx: &RouteCtx,
7905 frame: Frame,
7906 module_id: String,
7907 ) -> Result<Vec<Frame>, RouterError> {
7908 let Some(module) = self.supervisor.get(&module_id) else {
7909 return Ok(vec![control_error_frame(
7910 &frame,
7911 "unknown_module",
7912 format!("module_id '{module_id}' is not supervised"),
7913 )?]);
7914 };
7915
7916 let terminals = self
7919 .observe_control_wait(
7920 ctx,
7921 Some(module.module_id()),
7922 ops::SUPERVISOR_TERMINALS,
7923 module.read_durable_terminal_history(),
7924 )
7925 .await
7926 .map_err(|error| {
7927 RouterError::backend(
7928 0,
7929 frame.header.corr,
7930 format!("failed to read terminal history: {error}"),
7931 )
7932 })?;
7933 let response = ClientControlResponse::SupervisorTerminals {
7934 module_id,
7935 terminals,
7936 };
7937 Ok(vec![control_response_body_frame(
7938 &frame,
7939 &response,
7940 "ClientControlResponse::SupervisorTerminals",
7941 )?])
7942 }
7943
7944 fn handle_supervisor_routes(
7945 &self,
7946 frame: Frame,
7947 module_id: Option<String>,
7948 ) -> Result<Vec<Frame>, RouterError> {
7949 let modules = self
7950 .forwarding
7951 .route_census(module_id.as_deref())
7952 .map_err(RouterError::Forwarding)?
7953 .into_iter()
7954 .map(|(module_id, routes)| SupervisorRouteModule {
7955 module_id,
7956 routes: routes
7957 .into_iter()
7958 .map(|route| SupervisorRoute {
7959 consumer: match route.principal {
7960 Principal::Reserved { module_id } => {
7961 SupervisorRouteConsumer::Reserved { module_id }
7962 }
7963 Principal::Direct | Principal::Unverified => {
7964 SupervisorRouteConsumer::Direct {
7965 connection_id: route.goodbye_target.connection_id.get(),
7966 }
7967 }
7968 },
7969 age_ms: Instant::now()
7970 .saturating_duration_since(route.bound_at)
7971 .as_millis()
7972 .try_into()
7973 .unwrap_or(u64::MAX),
7974 draining: route.draining,
7975 drain_reason: route.drain_reason,
7976 })
7977 .collect(),
7978 })
7979 .collect();
7980 let response = ClientControlResponse::SupervisorRoutes { modules };
7981 Ok(vec![control_response_body_frame(
7982 &frame,
7983 &response,
7984 "ClientControlResponse::SupervisorRoutes",
7985 )?])
7986 }
7987
7988 fn running_registration_method(
7989 &self,
7990 module: &str,
7991 ) -> Option<subc_control::RegistrationMethod> {
7992 let registration = self.registry.get_module(module).ok()??;
7993 let connection = registration.connection_id;
7994 let channel = self.supervisor.registration_spawn(connection).is_some();
7995 let nonce = if channel {
7996 None
7997 } else {
7998 self.hello_launch_nonces
7999 .lock()
8000 .ok()?
8001 .nonce(connection)
8002 .map(str::to_owned)
8003 };
8004 if !channel && nonce.is_none() {
8005 return None;
8006 }
8007 self.supervisor
8008 .registration_snapshot_is_current_live(
8009 &self.registry,
8010 module,
8011 connection,
8012 nonce.as_deref(),
8013 )
8014 .then_some(if channel {
8015 subc_control::RegistrationMethod::Channel
8016 } else {
8017 subc_control::RegistrationMethod::Nonce
8018 })
8019 }
8020
8021 async fn handle_supervisor_provenance(
8022 &self,
8023 ctx: &RouteCtx,
8024 frame: Frame,
8025 module_id: Option<String>,
8026 ) -> Result<Vec<Frame>, RouterError> {
8027 let targeted = module_id.is_some();
8028 let selected = if let Some(module_id) = module_id {
8029 let Some(module) = self.supervisor.get(&module_id) else {
8030 return Ok(vec![control_error_frame(
8031 &frame,
8032 "unknown_module",
8033 format!("module_id '{module_id}' is not supervised"),
8034 )?]);
8035 };
8036 vec![module]
8037 } else {
8038 self.supervisor.list()
8039 };
8040
8041 let target = targeted.then(|| selected[0].module_id());
8042 let declarations = if let Some(owner) = &self.config_runtime {
8043 owner
8044 .on_blocking(|owner| owner.registration_declarations())
8045 .await
8046 .ok()
8047 } else {
8048 None
8049 };
8050 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
8051 let mut modules = Vec::with_capacity(selected.len());
8052 let mut extensions = Vec::with_capacity(selected.len());
8053 for module in &selected {
8054 let (status, observed_image) = module
8055 .status_and_running_image_agreement()
8056 .await
8057 .map_err(|err| {
8058 RouterError::backend(
8059 0,
8060 frame.header.corr,
8061 format!("failed to read supervisor status: {err}"),
8062 )
8063 })?;
8064 let module_declared = self
8065 .registry
8066 .get_module(&status.module_id)
8067 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
8068 .and_then(|registration| registration.manifest.provenance)
8069 .map(|build| ModuleDeclaredProvenance::Reported { build })
8070 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
8071 #[cfg(test)]
8072 let running_image = match &self.provenance_probe_override {
8073 Some(result) => result.clone(),
8074 None => observed_image,
8075 };
8076 #[cfg(not(test))]
8077 let running_image = observed_image;
8078 let registration = self.running_registration_method(&status.module_id);
8079 let mut extension = subc_control::SupervisorModuleProvenanceExt::new()
8080 .with_registration(registration);
8081 extension.registration_declaration = declarations
8082 .as_ref()
8083 .and_then(|entries| entries.get(&status.module_id))
8084 .cloned();
8085 if status.process_alive && status.registration_declaration.is_some() {
8086 extension.registration_declaration = status.registration_declaration.clone();
8087 }
8088 extensions.push(extension);
8089 modules.push(SupervisorModuleProvenance {
8090 module_id: status.module_id,
8091 module_declared,
8092 daemon_observed: SupervisorObservedProcess {
8093 launch_nonce_source: status.launch_nonce_source,
8094 pid: status.pid,
8095 spawned_at_ms: status.spawned_at_ms,
8096 spawned_from: status.spawned_from,
8097 running_image,
8098 },
8099 });
8100 }
8101 let daemon = SupervisorDaemonProvenance {
8102 daemon_build: self.daemon_provenance.build.clone(),
8103 daemon_observed: DaemonObservedProcess {
8104 pid: self.daemon_provenance.pid,
8105 started_at_ms: self
8106 .daemon_provenance
8107 .start_clock
8108 .map(|clock| clock.started_at_ms())
8109 .or(self.daemon_provenance.started_at_ms),
8110 running_image: self
8111 .daemon_provenance
8112 .probe
8113 .observe(
8114 self.daemon_provenance.pid,
8115 self.daemon_provenance.executable_path.as_deref(),
8116 self.daemon_provenance.executable_identity.clone(),
8117 self.daemon_provenance.process_start_time,
8118 )
8119 .await,
8120 },
8121 };
8122 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
8123 let mut response = serde_json::to_value(response)
8124 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
8125 for (entry, extension) in response["modules"]
8126 .as_array_mut()
8127 .expect("provenance modules are an array")
8128 .iter_mut()
8129 .zip(extensions)
8130 {
8131 extension.write_to(entry).map_err(|error| {
8132 RouterError::backend(0, frame.header.corr, error.to_string())
8133 })?;
8134 }
8135 Ok(vec![control_response_body_frame(
8136 &frame,
8137 &response,
8138 "ClientControlResponse::SupervisorProvenance",
8139 )?])
8140 })
8141 .await
8142 }
8143
8144 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
8145 self.refresh_capability_requirements();
8146 let publication = self
8147 .config_runtime
8148 .as_ref()
8149 .and_then(|owner| owner.polled_snapshot().ok());
8150 let generation = self
8151 .registry
8152 .generation()
8153 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
8154 let modules = self
8155 .supervisor
8156 .list()
8157 .into_iter()
8158 .map(|module| {
8159 let status = module.status_for_control("health").map_err(|err| {
8160 RouterError::backend(
8161 0,
8162 frame.header.corr,
8163 format!("failed to read supervisor health: {err}"),
8164 )
8165 })?;
8166 let module_id = status.module_id;
8167 let capability_detail = self
8168 .capability_evaluator
8169 .required_problem_detail(&module_id);
8170 let declaration_warning = self.registry.get_module(&module_id).ok().flatten()
8171 .filter(|registration| registration.manifest.registration_channel == Some(false))
8172 .filter(|_| publication.as_ref().and_then(|view| view.registration_declaration(&module_id))
8173 .as_deref().map(|next| if status.process_alive { status.registration_declaration.as_deref().unwrap_or(next) } else { next }) == Some("declared"))
8174 .map(|_| format!("{module_id}: HELLO registration_channel is false despite the approved channel declaration"));
8175 let capability_detail = append_capability_problem_detail(capability_detail, declaration_warning);
8176 Ok(SupervisorHealthEntry {
8177 module_id,
8178 status: status.health.status,
8179 detail: append_capability_problem_detail(
8180 status.health.detail,
8181 capability_detail,
8182 ),
8183 metrics: status.health.metrics,
8184 consecutive_failures: status.health.consecutive_failures,
8185 late_answer_count: status.health.late_answer_count,
8186 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
8187 last_action: status.health.last_action,
8188 last_action_ms: status.health.last_action_ms,
8189 last_probe_ms: status.health.last_probe_ms,
8190 })
8191 })
8192 .collect::<Result<Vec<_>, RouterError>>()?;
8193 let response = ClientControlResponse::SupervisorHealth {
8194 generation,
8195 modules,
8196 };
8197 Ok(vec![control_response_body_frame(
8198 &frame,
8199 &response,
8200 "ClientControlResponse::SupervisorHealth",
8201 )?])
8202 }
8203
8204 async fn handle_supervisor_restart(
8205 &self,
8206 frame: Frame,
8207 module_id: String,
8208 drain_timeout_ms: Option<u64>,
8209 requester: Principal,
8210 ) -> Result<Vec<Frame>, RouterError> {
8211 let operation_lock = self.supervisor.operation_lock();
8212 let _operation_guard = operation_lock.lock().await;
8213 let Some(module) = self.supervisor.get(&module_id) else {
8214 return Ok(vec![control_error_frame(
8215 &frame,
8216 "unknown_module",
8217 format!("module_id '{module_id}' is not supervised"),
8218 )?]);
8219 };
8220
8221 self.route_outages.mark_operator_action(&module_id);
8222 if let Err(err) = module
8223 .restart_requested(drain_timeout_ms, Some(requester))
8224 .await
8225 {
8226 self.route_outages
8227 .operator_action_ended_unrefused(&module_id);
8228 let (code, message) = match err {
8229 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8230 crate::supervise::SuperviseError::Disabled { .. } => {
8231 ("module_disabled", err.to_string())
8232 }
8233 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8234 ("swap_in_progress", err.to_string())
8235 }
8236 _ => (
8237 "target_unavailable",
8238 format!("failed to restart module_id '{module_id}': {err}"),
8239 ),
8240 };
8241 return Ok(vec![control_error_frame(&frame, code, message)?]);
8242 }
8243
8244 let response = ClientControlResponse::SupervisorAck {
8245 module_id,
8246 applied: true,
8247 };
8248 Ok(vec![control_response_body_frame(
8249 &frame,
8250 &response,
8251 "ClientControlResponse::SupervisorAck",
8252 )?])
8253 }
8254
8255 async fn handle_supervisor_swap(
8259 &self,
8260 frame: Frame,
8261 module_id: String,
8262 ready_timeout_ms: Option<u64>,
8263 requester: Principal,
8264 ) -> Result<Vec<Frame>, RouterError> {
8265 let module = {
8272 let operation_lock = self.supervisor.operation_lock();
8273 let _operation_guard = operation_lock.lock().await;
8274 self.supervisor.get(&module_id)
8275 };
8276 let Some(module) = module else {
8277 return Ok(vec![control_error_frame(
8278 &frame,
8279 "unknown_module",
8280 format!("module_id '{module_id}' is not supervised"),
8281 )?]);
8282 };
8283
8284 self.route_outages.mark_operator_action(&module_id);
8285 if let Err(err) = module
8286 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
8287 .await
8288 {
8289 self.route_outages
8290 .operator_action_ended_unrefused(&module_id);
8291 use crate::supervise::SuperviseError;
8292 let message = err.to_string();
8293 let error = match err {
8294 SuperviseError::ConfigHeld { .. } => ErrorBody::new("config_gated", message),
8295 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
8296 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
8297 code: "swap_refused".to_string(),
8298 message,
8299 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
8300 },
8301 SuperviseError::SwapFailed {
8302 arm,
8303 candidate_exit,
8304 ..
8305 } => ErrorBody {
8306 code: "swap_failed".to_string(),
8307 message,
8308 detail: Some(serde_json::json!({
8309 "arm": arm.as_str(),
8310 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
8311 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
8312 })),
8313 },
8314 _ => ErrorBody::new(
8315 "target_unavailable",
8316 format!("failed to swap module_id '{module_id}': {message}"),
8317 ),
8318 };
8319 return Ok(vec![control_error_body_frame(&frame, error)?]);
8320 }
8321 self.route_outages
8325 .operator_action_ended_unrefused(&module_id);
8326
8327 let response = ClientControlResponse::SupervisorAck {
8328 module_id,
8329 applied: true,
8330 };
8331 Ok(vec![control_response_body_frame(
8332 &frame,
8333 &response,
8334 "ClientControlResponse::SupervisorAck",
8335 )?])
8336 }
8337
8338 async fn handle_supervisor_reload(
8339 &self,
8340 frame: Frame,
8341 module_id: String,
8342 requester: Principal,
8343 ) -> Result<Vec<Frame>, RouterError> {
8344 let operation_lock = self.supervisor.operation_lock();
8345 let _operation_guard = operation_lock.lock().await;
8346 let Some(module) = self.supervisor.get(&module_id) else {
8347 return Ok(vec![control_error_frame(
8348 &frame,
8349 "unknown_module",
8350 format!("module_id '{module_id}' is not supervised"),
8351 )?]);
8352 };
8353
8354 self.route_outages.mark_operator_action(&module_id);
8355 if let Err(err) = module.reload_requested(Some(requester)).await {
8356 self.route_outages
8357 .operator_action_ended_unrefused(&module_id);
8358 let (code, message) = match err {
8359 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8360 crate::supervise::SuperviseError::Disabled { .. } => {
8361 ("module_disabled", err.to_string())
8362 }
8363 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8364 ("swap_in_progress", err.to_string())
8365 }
8366 _ => (
8367 "reload_failed",
8368 format!("failed to reload module_id '{module_id}': {err}"),
8369 ),
8370 };
8371 return Ok(vec![control_error_frame(&frame, code, message)?]);
8372 }
8373
8374 let response = ClientControlResponse::SupervisorAck {
8375 module_id,
8376 applied: true,
8377 };
8378 Ok(vec![control_response_body_frame(
8379 &frame,
8380 &response,
8381 "ClientControlResponse::SupervisorAck",
8382 )?])
8383 }
8384
8385 async fn handle_supervisor_rescan(
8389 &self,
8390 frame: Frame,
8391 mode: RescanMode,
8392 requester: Principal,
8393 ) -> Result<Vec<Frame>, RouterError> {
8394 let preview = mode.previews();
8395 let mut approved_preview = None;
8396 if matches!(mode, RescanMode::Preview) {
8397 if let Some(owner) = &self.config_runtime {
8398 match owner
8399 .on_blocking(|owner| {
8400 owner.observe_file_proposal()?;
8401 owner.approved_daemon_configuration()
8402 })
8403 .await
8404 {
8405 Ok(config) => approved_preview = config,
8406 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8407 }
8408 }
8409 }
8410 let Some(context) = self.rescan.clone() else {
8411 return Ok(vec![control_error_frame(
8412 &frame,
8413 "rescan_unavailable",
8414 "the daemon was not started with a reloadable config path".to_string(),
8415 )?]);
8416 };
8417
8418 let operation_lock = self.supervisor.operation_lock();
8419 let _operation_guard = operation_lock.lock().await;
8420 let loaded = match self.load_rescan_config(&mode, &context) {
8424 Ok(config) => config,
8425 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
8426 };
8427 let Some((config, activity_config)) = loaded else {
8435 return Ok(vec![control_error_frame(
8436 &frame,
8437 "invalid_daemon_config",
8438 format!(
8439 "daemon config not found at {}; refusing to rescan (an absent config would \
8440 retire every supervised module)",
8441 context.config_path.display()
8442 ),
8443 )?]);
8444 };
8445 let (
8446 configured_port,
8447 storage_config,
8448 admission_facts_carrier_module_id,
8449 admission_facts_targets,
8450 scope_authority_owners,
8451 federated_scope_owner,
8452 privileged_operator_prompt,
8453 modules,
8454 reserved_capabilities,
8455 ) = (
8456 config.port,
8457 config.storage,
8458 config.admission_facts_carrier_module_id,
8459 config.admission_facts_targets,
8460 config.scope_authority_owners,
8461 config.federated_scope_owner,
8462 config.privileged_operator_prompt,
8463 config.modules,
8464 config.reserved_capabilities,
8465 );
8466
8467 let mut restart_required = Vec::new();
8476 for section in RestartRequiredSection::ALL {
8477 let changed = match section {
8478 RestartRequiredSection::Port => configured_port != context.configured_port,
8479 RestartRequiredSection::Storage => storage_config != context.storage_config,
8480 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
8481 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
8482 }
8483 RestartRequiredSection::AdmissionFactsTargets => {
8484 admission_facts_targets != context.admission_facts_targets
8485 }
8486 RestartRequiredSection::ScopeAuthorityOwners => {
8487 scope_authority_owners != context.scope_authority_owners
8488 }
8489 RestartRequiredSection::FederatedScopeOwner => {
8490 federated_scope_owner != context.federated_scope_owner
8491 }
8492 RestartRequiredSection::PrivilegedOperatorPrompt => {
8493 privileged_operator_prompt != context.privileged_operator_prompt
8494 }
8495 };
8496 if changed {
8497 restart_required.push(section.label().to_string());
8498 }
8499 }
8500 if !restart_required.is_empty() {
8501 warn!(
8502 config_path = %context.config_path.display(),
8503 sections = %restart_required.join(", "),
8504 "daemon config changed outside the modules section; restart the daemon to apply those changes"
8505 );
8506 }
8507
8508 for configured in &modules {
8509 if let Err(err) = validate_spec(&configured.module_spec()) {
8510 return Ok(vec![control_error_frame(
8511 &frame,
8512 "invalid_daemon_config",
8513 format!("supervisor rescan rejected daemon config: {err}"),
8514 )?]);
8515 }
8516 }
8517
8518 let configured_capabilities = modules
8519 .iter()
8520 .map(|module| (module.module_id.clone(), module.enabled))
8521 .collect::<Vec<_>>();
8522 let preview_capability_warnings = if preview {
8523 let (_, registrations) = self.runtime_capability_snapshot()?;
8524 let current_modules = self
8525 .supervisor
8526 .list()
8527 .into_iter()
8528 .map(|module| module.module_id().to_string())
8529 .collect::<BTreeSet<_>>();
8530 let resulting_modules = configured_capabilities.clone();
8531 let removed = current_modules
8532 .into_iter()
8533 .filter(|module_id| {
8534 !resulting_modules
8535 .iter()
8536 .any(|(configured_id, _)| configured_id == module_id)
8537 })
8538 .collect::<Vec<_>>();
8539 let disabled = match &mode {
8542 RescanMode::Candidate(_) => resulting_modules
8543 .iter()
8544 .filter(|(_, enabled)| !enabled)
8545 .map(|(module_id, _)| module_id.clone())
8546 .collect::<Vec<_>>(),
8547 _ => Vec::new(),
8548 };
8549 self.capability_evaluator.preview_warnings(
8550 resulting_modules,
8551 &removed,
8552 &disabled,
8553 ®istrations,
8554 )
8555 } else {
8556 Vec::new()
8557 };
8558 let reconciled = match (&mode, approved_preview.as_ref()) {
8559 (RescanMode::Preview, Some(approved)) => Ok((
8560 config_gate::preview_approved_daemon(approved, &modules),
8561 Vec::new(),
8562 )),
8563 (RescanMode::Candidate(candidate), _) => self
8564 .preview_candidate(&context, modules, candidate)
8565 .map(|result| (result, Vec::new())),
8566 _ => {
8567 self.reconcile_supervised_modules(
8568 &context.supervisor,
8569 modules,
8570 preview,
8571 requester.clone(),
8572 )
8573 .await
8574 }
8575 };
8576 let (result, changed) = match reconciled {
8577 Ok(result) => result,
8578 Err(message) => {
8579 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
8580 }
8581 };
8582 if !preview {
8583 if let Some(ledger) = &self.activity {
8584 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
8585 ledger.config_changed(&result.added, &result.removed, &changed, requester);
8586 }
8587 if let Err(code) = self.configure_activity(activity_config).await {
8588 warn!(
8589 code,
8590 "activity retention could not be applied during rescan"
8591 );
8592 }
8593 self.capability_evaluator
8594 .configure(configured_capabilities, reserved_capabilities);
8595 self.capability_evaluator.wake_deadline_loop();
8596 self.refresh_capability_requirements();
8597 }
8598 let mut result = result;
8599 result.restart_required = restart_required;
8600 result.capability_warnings = preview_capability_warnings;
8601 let response = ClientControlResponse::SupervisorRescan { result };
8602 Ok(vec![control_response_body_frame(
8603 &frame,
8604 &response,
8605 "ClientControlResponse::SupervisorRescan",
8606 )?])
8607 }
8608
8609 async fn handle_supervisor_release_reserved(
8610 &self,
8611 frame: Frame,
8612 module_id: String,
8613 ) -> Result<Vec<Frame>, RouterError> {
8614 let Some(context) = self.rescan.clone() else {
8615 return Ok(vec![control_error_frame(
8616 &frame,
8617 "release_unavailable",
8618 "reserved-id release requires a daemon started with a reloadable config path",
8619 )?]);
8620 };
8621 let operation_lock = self.supervisor.operation_lock();
8622 let _operation_guard = operation_lock.lock().await;
8623 let approved = if let Some(owner) = &self.config_runtime {
8624 match owner
8627 .on_blocking(|owner| {
8628 owner.observe_file_proposal()?;
8629 owner.approved_daemon_configuration()
8630 })
8631 .await
8632 {
8633 Ok(config) => config,
8634 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8635 }
8636 } else {
8637 None
8638 };
8639 let loaded = if let Some(approved) = approved {
8640 Ok(Some(approved))
8641 } else {
8642 crate::daemon_config::load(&context.config_path)
8643 };
8644 let loaded = match loaded {
8645 Ok(Some(config)) => config,
8646 Ok(None) => {
8647 return Ok(vec![control_error_frame(
8648 &frame,
8649 "invalid_daemon_config",
8650 format!(
8651 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
8652 context.config_path.display()
8653 ),
8654 )?])
8655 }
8656 Err(err) => {
8657 return Ok(vec![control_error_frame(
8658 &frame,
8659 "invalid_daemon_config",
8660 format!("unable to verify reserved-id release against daemon config: {err}"),
8661 )?])
8662 }
8663 };
8664 if loaded
8665 .modules
8666 .iter()
8667 .any(|configured| configured.module_id == module_id)
8668 {
8669 return Ok(vec![control_error_frame(
8670 &frame,
8671 "reserved_module_configured",
8672 format!(
8673 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
8674 ),
8675 )?]);
8676 }
8677 if !self.supervisor.release_retained_reserved_gate(&module_id) {
8678 return Ok(vec![control_error_frame(
8679 &frame,
8680 "reserved_gate_not_retained",
8681 format!(
8682 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
8683 ),
8684 )?]);
8685 }
8686
8687 let response = ClientControlResponse::SupervisorAck {
8688 module_id,
8689 applied: true,
8690 };
8691 Ok(vec![control_response_body_frame(
8692 &frame,
8693 &response,
8694 "ClientControlResponse::SupervisorAck",
8695 )?])
8696 }
8697
8698 fn plan_rescan(
8702 &self,
8703 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8704 ) -> Result<RescanPlan, String> {
8705 let mut current = BTreeMap::new();
8706 for module in self.supervisor.list() {
8707 let (spec, health) = module.configuration().map_err(|err| {
8708 format!(
8709 "failed to read configuration for module_id '{}': {err}",
8710 module.module_id()
8711 )
8712 })?;
8713 let enabled = module
8714 .status()
8715 .map_err(|err| {
8716 format!(
8717 "failed to read status for module_id '{}': {err}",
8718 module.module_id()
8719 )
8720 })?
8721 .enabled;
8722 current.insert(
8723 module.module_id().to_string(),
8724 (module, spec, health, enabled),
8725 );
8726 }
8727 let configured = configured_modules
8728 .into_iter()
8729 .map(|module| (module.module_id.clone(), module))
8730 .collect::<BTreeMap<_, _>>();
8731
8732 let added = configured
8733 .keys()
8734 .filter(|module_id| !current.contains_key(*module_id))
8735 .cloned()
8736 .collect::<Vec<_>>();
8737 let removed = current
8738 .keys()
8739 .filter(|module_id| !configured.contains_key(*module_id))
8740 .cloned()
8741 .collect::<Vec<_>>();
8742 let mut changed_pending_reload = Vec::new();
8743 let mut configuration_changes = BTreeSet::new();
8744 let mut enabled_changes = BTreeSet::new();
8745 let mut unchanged = 0_u32;
8746
8747 for (module_id, configured_module) in &configured {
8748 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
8749 else {
8750 continue;
8751 };
8752 let launch_changed = *current_spec != configured_module.module_spec();
8756 let configuration_changed =
8757 launch_changed || *current_health != configured_module.health;
8758 let enabled_changed = *current_enabled != configured_module.enabled
8761 || self
8762 .capability_evaluator
8763 .configured_enabled(module_id)
8764 .is_some_and(|enabled| enabled != configured_module.enabled);
8765 if configuration_changed {
8766 configuration_changes.insert(module_id.clone());
8767 }
8768 if launch_changed {
8769 changed_pending_reload.push(module_id.clone());
8770 }
8771 if enabled_changed {
8772 enabled_changes.insert(module_id.clone());
8773 }
8774 if !configuration_changed && !enabled_changed {
8775 unchanged = unchanged.saturating_add(1);
8776 }
8777 }
8778 Ok(RescanPlan {
8779 current,
8780 configured,
8781 added,
8782 removed,
8783 changed_pending_reload,
8784 configuration_changes,
8785 enabled_changes,
8786 unchanged,
8787 })
8788 }
8789
8790 async fn reconcile_supervised_modules(
8799 &self,
8800 supervisor: &Supervisor,
8801 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8802 preview: bool,
8803 requester: Principal,
8804 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
8805 let plan = self.plan_rescan(configured_modules)?;
8806
8807 if preview {
8813 return Ok((plan.preview_result(), Vec::new()));
8814 }
8815 let changed = plan
8816 .configuration_changes
8817 .union(&plan.enabled_changes)
8818 .cloned()
8819 .collect::<Vec<_>>();
8820 let RescanPlan {
8821 current,
8822 configured,
8823 added,
8824 removed,
8825 changed_pending_reload,
8826 configuration_changes,
8827 enabled_changes,
8828 unchanged,
8829 } = plan;
8830 if !(added.is_empty()
8835 && removed.is_empty()
8836 && configuration_changes.is_empty()
8837 && enabled_changes.is_empty())
8838 {
8839 if let Some(context) = &self.rescan {
8840 context
8841 .generation
8842 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8843 }
8844 }
8845
8846 for module_id in &removed {
8847 let module = ¤t
8848 .get(module_id)
8849 .expect("removed module came from current supervisor state")
8850 .0;
8851 module
8852 .retire_requested(Some(requester.clone()))
8853 .await
8854 .map_err(|err| {
8855 format!("failed to retire module_id '{module_id}' during rescan: {err}")
8856 })?;
8857 self.supervisor.record_rescan_removal(module_id);
8891 self.supervisor.retire(module_id);
8892 self.route_outages.forget(module_id);
8893 }
8894
8895 for module_id in configured.keys() {
8896 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
8897 continue;
8898 };
8899 let configured_module = configured
8900 .get(module_id)
8901 .expect("configured module id came from configured map");
8902 if configuration_changes.contains(module_id) {
8903 module
8904 .update_configuration(
8905 configured_module.module_spec(),
8906 configured_module.health.clone(),
8907 configured_module.drain_timeout_ms,
8908 )
8909 .await
8910 .map_err(|err| {
8911 format!(
8912 "failed to update module_id '{module_id}' configuration during rescan: {err}"
8913 )
8914 })?;
8915 }
8916 if *current_enabled != configured_module.enabled {
8919 self.route_outages.mark_operator_action(module_id);
8922 module
8923 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
8924 .await
8925 .map_err(|err| {
8926 self.route_outages.operator_action_ended_unrefused(module_id);
8927 format!(
8928 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
8929 configured_module.enabled
8930 )
8931 })?;
8932 }
8933 }
8934
8935 for module_id in &added {
8936 let configured_module = configured
8937 .get(module_id)
8938 .expect("added module id came from configured map");
8939 supervisor
8940 .supervise_configured_requested_async(
8941 configured_module.module_spec(),
8942 configured_module.enabled,
8943 configured_module.health.clone(),
8944 configured_module.drain_timeout_ms,
8945 configured_module.restart,
8946 configured_module.register_timeout,
8947 Some(requester.clone()),
8948 )
8949 .await
8950 .map_err(|err| {
8951 format!("failed to add module_id '{module_id}' during rescan: {err}")
8952 })?;
8953 }
8954
8955 Ok((
8956 SupervisorRescanResult {
8957 added,
8958 removed,
8959 changed_pending_reload,
8960 enabled_changes: enabled_changes.iter().cloned().collect(),
8961 unchanged,
8962 preview: false,
8963 restart_required: Vec::new(),
8966 capability_warnings: Vec::new(),
8967 module_actions: Vec::new(),
8968 earlier_pending: Vec::new(),
8969 expect: None,
8970 },
8971 changed,
8972 ))
8973 }
8974
8975 async fn handle_supervisor_set_enabled(
8976 &self,
8977 frame: Frame,
8978 module_id: String,
8979 enabled: bool,
8980 requester: Principal,
8981 ) -> Result<Vec<Frame>, RouterError> {
8982 let operation_lock = self.supervisor.operation_lock();
8983 let _operation_guard = operation_lock.lock().await;
8984 let Some(module) = self.supervisor.get(&module_id) else {
8985 return Ok(vec![control_error_frame(
8986 &frame,
8987 "unknown_module",
8988 format!("module_id '{module_id}' is not supervised"),
8989 )?]);
8990 };
8991
8992 self.route_outages.mark_operator_action(&module_id);
8995 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
8996 Ok(applied) => applied,
8997 Err(err) => {
8998 self.route_outages
8999 .operator_action_ended_unrefused(&module_id);
9000 return Ok(vec![control_error_frame(
9001 &frame,
9002 if err.is_config_gate_error() {
9003 "config_gated"
9004 } else {
9005 "target_unavailable"
9006 },
9007 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
9008 )?]);
9009 }
9010 };
9011 if !applied {
9012 self.route_outages
9015 .operator_action_ended_unrefused(&module_id);
9016 }
9017
9018 self.capability_evaluator.wake_deadline_loop();
9019 self.refresh_capability_requirements();
9020 let response = ClientControlResponse::SupervisorAck { module_id, applied };
9021 Ok(vec![control_response_body_frame(
9022 &frame,
9023 &response,
9024 "ClientControlResponse::SupervisorAck",
9025 )?])
9026 }
9027
9028 async fn handle_supervisor_health_probe(
9029 &self,
9030 ctx: &RouteCtx,
9031 frame: Frame,
9032 module_id: String,
9033 ) -> Result<Vec<Frame>, RouterError> {
9034 self.refresh_capability_requirements();
9035 let Some(registration) = self
9036 .registry
9037 .get_module(&module_id)
9038 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
9039 else {
9040 return Ok(vec![control_error_frame(
9041 &frame,
9042 "unknown_module",
9043 format!("module_id '{module_id}' is not registered"),
9044 )?]);
9045 };
9046
9047 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
9060 {
9061 return Ok(vec![control_error_frame(
9062 &frame,
9063 "health_not_advertised",
9064 format!("module_id '{module_id}' did not advertise health.check"),
9065 )?]);
9066 }
9067
9068 self.observe_control_wait(
9069 ctx,
9070 Some(®istration.manifest.module_id),
9071 ops::SUPERVISOR_HEALTH_PROBE,
9072 async {
9073 let deadline = Instant::now() + self.health_probe_timeout;
9074 let pending = match self.forwarding.begin_module_control_rpc_for(
9075 &module_id,
9076 MODULE_CONTROL_OP_HEALTH_CHECK,
9077 deadline,
9078 ) {
9079 Ok(pending) => pending,
9080 Err(err) => {
9081 return Ok(vec![control_error_frame(
9082 &frame,
9083 forwarding_error_code(&err),
9084 err.to_string(),
9085 )?])
9086 }
9087 };
9088
9089 let PendingModuleControlRpc {
9090 endpoint,
9091 module_sink,
9092 negotiated_ver,
9093 corr: probe_corr,
9094 receiver,
9095 } = pending;
9096 let mut guard =
9097 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
9098 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
9099 .map_err(|err| {
9100 RouterError::backend(
9101 0,
9102 frame.header.corr,
9103 format!("failed to encode health.check request: {err}"),
9104 )
9105 })?;
9106 let probe_frame = Frame::build_with_version(
9107 negotiated_ver,
9108 FrameType::Request,
9109 control_flags(),
9110 0,
9111 0,
9112 probe_corr,
9113 probe_body,
9114 )
9115 .map_err(RouterError::FrameBuild)?;
9116
9117 match timeout_at(deadline, module_sink.send(probe_frame)).await {
9120 Ok(Ok(())) => {}
9121 Ok(Err(err)) => {
9122 return Ok(vec![control_error_frame(
9123 &frame,
9124 "target_unavailable",
9125 err.to_string(),
9126 )?]);
9127 }
9128 Err(_) => {
9129 return Ok(vec![control_error_frame(
9130 &frame,
9131 "module_timeout",
9132 format!(
9133 "module_id '{module_id}' did not answer health.check within {:?}",
9134 self.health_probe_timeout
9135 ),
9136 )?]);
9137 }
9138 }
9139
9140 match timeout_at(deadline, receiver).await {
9141 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => {
9142 Ok(vec![control_error_frame(
9143 &frame,
9144 "invalid_control_body",
9145 "health.check returned an unexpected config response",
9146 )?])
9147 }
9148 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
9149 guard.disarm();
9150 let Some(report) = response.health_report() else {
9151 return Ok(vec![control_error_frame(
9152 &frame,
9153 "invalid_control_body",
9154 "health.check RPC returned a non-health response",
9155 )?]);
9156 };
9157 let HealthReport {
9162 status,
9163 detail,
9164 metrics,
9165 } = report;
9166 let capability_detail = self
9167 .capability_evaluator
9168 .required_problem_detail(&module_id);
9169 let response = ClientControlResponse::SupervisorHealthProbe {
9170 module_id,
9171 status,
9172 detail: append_capability_problem_detail(detail, capability_detail),
9173 metrics,
9174 };
9175 Ok(vec![control_response_body_frame(
9176 &frame,
9177 &response,
9178 "ClientControlResponse::SupervisorHealthProbe",
9179 )?])
9180 }
9181 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
9182 guard.disarm();
9183 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
9188 }
9189 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
9190 guard.disarm();
9191 Ok(vec![control_error_frame(
9192 &frame,
9193 "target_unavailable",
9194 message,
9195 )?])
9196 }
9197 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
9198 guard.disarm();
9199 Ok(vec![control_error_frame(
9200 &frame,
9201 "invalid_control_body",
9202 message,
9203 )?])
9204 }
9205 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
9206 guard.disarm();
9207 Ok(vec![control_error_frame(
9208 &frame,
9209 "invalid_control_body",
9210 format!("expected module-control op '{expected}', got '{actual}'"),
9211 )?])
9212 }
9213 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
9214 guard.disarm();
9215 Ok(vec![control_error_frame(
9216 &frame,
9217 "module_timeout",
9218 format!(
9219 "module_id '{module_id}' answered health.check after {:?}",
9220 self.health_probe_timeout
9221 ),
9222 )?])
9223 }
9224 Ok(Err(_)) => Ok(vec![control_error_frame(
9225 &frame,
9226 "target_unavailable",
9227 "health.check waiter was canceled before the module responded",
9228 )?]),
9229 Err(_) => Ok(vec![control_error_frame(
9230 &frame,
9231 "module_timeout",
9232 format!(
9233 "module_id '{module_id}' did not answer health.check within {:?}",
9234 self.health_probe_timeout
9235 ),
9236 )?]),
9237 }
9238 },
9239 )
9240 .await
9241 }
9242
9243 fn supervisor_status(
9244 &self,
9245 module_id: &str,
9246 corr: u64,
9247 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
9248 self.supervisor
9249 .get(module_id)
9250 .map(|module| {
9251 let warming = module.is_warming_for_control("status").map_err(|err| {
9252 RouterError::backend(
9253 0,
9254 corr,
9255 format!(
9256 "failed to read supervisor warming state for module_id '{module_id}': {err}"
9257 ),
9258 )
9259 })?;
9260 module.status_for_control("status").map_err(|err| {
9261 RouterError::backend(
9262 0,
9263 corr,
9264 format!(
9265 "failed to read supervisor status for module_id '{module_id}': {err}"
9266 ),
9267 )
9268 }).map(|status| (status, warming))
9269 })
9270 .transpose()
9271 }
9272
9273 fn guard_module_control_op(
9274 &self,
9275 frame: &Frame,
9276 module_id: &str,
9277 op: &str,
9278 ) -> Result<Option<Frame>, RouterError> {
9279 if self.module_grants_op(module_id, op, frame.header.corr)? {
9280 return Ok(None);
9281 }
9282
9283 Ok(Some(control_error_frame(
9284 frame,
9285 "op_not_allowed",
9286 format!("module_id '{module_id}' did not grant control op '{op}'"),
9287 )?))
9288 }
9289
9290 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
9291 let Some(registration) = self
9292 .registry
9293 .get_module(module_id)
9294 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
9295 else {
9296 return Ok(false);
9297 };
9298 Ok(module_registration_grants_op(®istration.control_ops, op))
9299 }
9300
9301 fn handle_status_update(
9302 &self,
9303 endpoint: ModuleEndpointId,
9304 frame: Frame,
9305 ) -> Result<Vec<Frame>, RouterError> {
9306 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
9307 Ok(update) => update,
9308 Err(err) => {
9309 if is_known_module_push_op(&frame.body) {
9314 return Ok(vec![control_error_frame(
9315 &frame,
9316 "invalid_control_body",
9317 format!("malformed module control push body: {err}"),
9318 )?]);
9319 }
9320 return Ok(Vec::new());
9321 }
9322 };
9323
9324 match update {
9325 ModuleControlPush::RouteStatus {
9326 route_channel,
9327 route_epoch,
9328 status,
9329 } => {
9330 self.forwarding
9331 .cache_status(endpoint, route_channel, route_epoch, status)
9332 .map_err(RouterError::Forwarding)?;
9333 }
9334 }
9335 Ok(Vec::new())
9336 }
9337
9338 fn handle_route_poll(
9339 &self,
9340 ctx: &RouteCtx,
9341 frame: Frame,
9342 route_channel: u16,
9343 route_epoch: u32,
9344 kind: PollKind,
9345 ) -> Result<Vec<Frame>, RouterError> {
9346 let snapshot = self
9347 .forwarding
9348 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
9349 .map_err(RouterError::Forwarding)?;
9350 let response = match (kind, snapshot) {
9351 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
9352 ClientControlResponse::RoutePoll {
9353 route_channel,
9354 route_epoch,
9355 status,
9356 live: None,
9357 }
9358 }
9359 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9360 route_channel,
9361 route_epoch,
9362 status: None,
9363 live: None,
9364 },
9365 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
9366 let live = self
9392 .process_liveness
9393 .as_ref()
9394 .and_then(|source| source.process_live(&module_id))
9395 .unwrap_or(true);
9396 ClientControlResponse::RoutePoll {
9397 route_channel,
9398 route_epoch,
9399 status: None,
9400 live: Some(live),
9401 }
9402 }
9403 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9404 route_channel,
9405 route_epoch,
9406 status: None,
9407 live: Some(false),
9408 },
9409 };
9410
9411 Ok(vec![control_response_body_frame(
9412 &frame,
9413 &response,
9414 "ClientControlResponse::RoutePoll",
9415 )?])
9416 }
9417
9418 pub(crate) fn observe_module_control_completion(
9419 &self,
9420 completion: ModuleControlRpcCompletion,
9421 ) -> bool {
9422 match completion {
9423 ModuleControlRpcCompletion::Unknown => false,
9424 ModuleControlRpcCompletion::Settled => true,
9425 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
9426 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
9427 info!(
9428 module_id = %module_id,
9429 latency_ms,
9430 "late health.check answer proves the module is alive"
9431 );
9432 match self
9433 .supervisor
9434 .record_late_health_answer(&module_id, latency_ms)
9435 {
9436 Ok(true) => {}
9437 Ok(false) => debug!(
9438 module_id = %module_id,
9439 latency_ms,
9440 "late health.check answer has no active supervisor snapshot"
9441 ),
9442 Err(err) => warn!(
9443 module_id = %module_id,
9444 latency_ms,
9445 error = %err,
9446 "failed to record late health.check answer"
9447 ),
9448 }
9449 true
9450 }
9451 }
9452 }
9453
9454 fn refuse_to_end_module_connection_for_a_client(
9475 &self,
9476 module_connection_id: ConnectionId,
9477 corr: u64,
9478 err: ForwardingError,
9479 ) -> Result<(), RouterError> {
9480 if let ForwardingError::ConnectionClosing { connection_id } = err {
9481 if connection_id != module_connection_id {
9482 warn!(
9483 module_connection_id = module_connection_id.get(),
9484 client_connection_id = connection_id.get(),
9485 corr,
9486 "dropping a route.bind response for a closing client; the module connection keeps serving"
9487 );
9488 return Ok(());
9489 }
9490 }
9491 Err(RouterError::Forwarding(err))
9492 }
9493
9494 fn handle_module_relay_response(
9495 &self,
9496 connection_id: ConnectionId,
9497 frame: Frame,
9498 ) -> Result<Vec<Frame>, RouterError> {
9499 let mut secondary_error = None;
9500 let outcome = match frame.header.ty {
9501 FrameType::Response => match module_control_response_probe(&frame.body) {
9502 Ok(probe) if probe.op == "route.bind" => {
9503 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9504 Ok(ModuleControlResponse::RouteBindAck {}) => {
9505 RouteBindRelayOutcome::Accepted
9506 }
9507 Ok(other) => {
9508 let message =
9509 format!("route.bind response carried unexpected body: {other:?}");
9510 secondary_error = Some(control_error_frame(
9511 &frame,
9512 "invalid_control_body",
9513 message.clone(),
9514 )?);
9515 RouteBindRelayOutcome::ModuleGone(message)
9516 }
9517 Err(err) => {
9518 let message = format!("malformed route.bind response body: {err}");
9519 secondary_error = Some(control_error_frame(
9520 &frame,
9521 "invalid_control_body",
9522 message.clone(),
9523 )?);
9524 RouteBindRelayOutcome::ModuleGone(message)
9525 }
9526 }
9527 }
9528 Ok(probe) => {
9529 let outcome = if matches!(probe.op.as_str(), "config.review" | "consent/v1") {
9530 ModuleControlRpcOutcome::RawResponse(frame.body.clone())
9531 } else {
9532 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9533 Ok(response) => ModuleControlRpcOutcome::Response(response),
9534 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9535 "malformed {} response body: {err}",
9536 probe.op
9537 )),
9538 }
9539 };
9540 let completion = self
9541 .forwarding
9542 .complete_module_control_rpc(
9543 connection_id,
9544 frame.header.corr,
9545 Some(&probe.op),
9546 outcome,
9547 )
9548 .map_err(RouterError::Forwarding)?;
9549 if !self.observe_module_control_completion(completion) {
9550 debug!(
9551 connection_id = connection_id.get(),
9552 corr = frame.header.corr,
9553 op = %probe.op,
9554 "dropping late or unknown module-control RPC response"
9555 );
9556 }
9557 return Ok(Vec::new());
9558 }
9559 Err(err) => {
9560 if let Some(expected_op) = self
9561 .forwarding
9562 .pending_module_control_op(connection_id, frame.header.corr)
9563 .map_err(RouterError::Forwarding)?
9564 {
9565 let completion = self
9566 .forwarding
9567 .complete_module_control_rpc(
9568 connection_id,
9569 frame.header.corr,
9570 None,
9571 ModuleControlRpcOutcome::MalformedResponse(format!(
9572 "malformed {expected_op} response body: {err}"
9573 )),
9574 )
9575 .map_err(RouterError::Forwarding)?;
9576 if !self.observe_module_control_completion(completion) {
9577 debug!(
9578 connection_id = connection_id.get(),
9579 corr = frame.header.corr,
9580 "dropping late malformed module-control RPC response"
9581 );
9582 }
9583 return Ok(Vec::new());
9584 }
9585 let message = format!("malformed route.bind response body: {err}");
9586 secondary_error = Some(control_error_frame(
9587 &frame,
9588 "invalid_control_body",
9589 message.clone(),
9590 )?);
9591 RouteBindRelayOutcome::ModuleGone(message)
9592 }
9593 },
9594 FrameType::Error => {
9595 if self
9596 .forwarding
9597 .pending_module_control_op(connection_id, frame.header.corr)
9598 .map_err(RouterError::Forwarding)?
9599 .is_some()
9600 {
9601 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
9602 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
9603 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9604 "malformed module-control ERROR body: {err}"
9605 )),
9606 };
9607 let completion = self
9608 .forwarding
9609 .complete_module_control_rpc(
9610 connection_id,
9611 frame.header.corr,
9612 None,
9613 outcome,
9614 )
9615 .map_err(RouterError::Forwarding)?;
9616 if !self.observe_module_control_completion(completion) {
9617 debug!(
9618 connection_id = connection_id.get(),
9619 corr = frame.header.corr,
9620 "dropping late or unknown module-control RPC error"
9621 );
9622 }
9623 return Ok(Vec::new());
9624 }
9625 match serde_json::from_slice::<ErrorBody>(&frame.body) {
9626 Ok(body) => RouteBindRelayOutcome::Rejected(body),
9627 Err(err) => {
9628 let message = format!("malformed route.bind ERROR body: {err}");
9629 secondary_error = Some(control_error_frame(
9630 &frame,
9631 "invalid_control_body",
9632 message.clone(),
9633 )?);
9634 RouteBindRelayOutcome::ModuleGone(message)
9635 }
9636 }
9637 }
9638 ty => {
9639 return Ok(vec![control_error_frame(
9640 &frame,
9641 "unsupported_control_frame",
9642 format!("unsupported module channel-0 frame {ty:?}"),
9643 )?])
9644 }
9645 };
9646
9647 let settled =
9648 self.forwarding
9649 .complete_pending_relay(connection_id, frame.header.corr, outcome);
9650 let completion = match settled {
9651 Ok(completion) => completion,
9652 Err(err) => {
9653 self.refuse_to_end_module_connection_for_a_client(
9654 connection_id,
9655 frame.header.corr,
9656 err,
9657 )?;
9658 return Ok(secondary_error.into_iter().collect());
9659 }
9660 };
9661 if let Some(target) = completion.abandoned.as_ref() {
9662 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
9663 }
9664 if !completion.settled {
9665 debug!(
9666 connection_id = connection_id.get(),
9667 corr = frame.header.corr,
9668 frame_type = ?frame.header.ty,
9669 "dropping late or unknown route.bind relay response"
9670 );
9671 }
9672 Ok(secondary_error.into_iter().collect())
9673 }
9674
9675 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
9676 debug!(connection_id = connection_id.get(), "handling GOODBYE");
9677 self.cleanup_connection_with_end_reason(
9681 connection_id,
9682 RegistrationEndReason::ExplicitGoodbye,
9683 )
9684 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
9685 Ok(Vec::new())
9686 }
9687}
9688
9689impl Default for ControlHandler {
9690 fn default() -> Self {
9691 Self::new(Arc::new(Registry::default()))
9692 }
9693}
9694
9695impl crate::supervise::SwapPromotionObserver for ControlHandler {
9696 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
9697 self.apply_registration_capabilities(registration);
9698 }
9699}
9700
9701fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
9702 CapabilityRequirementStatus {
9703 consumer: status.consumer,
9704 capability: status.capability,
9705 need: match status.need {
9706 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
9707 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
9708 },
9709 verdict: status.verdict.as_str().to_string(),
9710 episode_seq: status.episode_seq,
9711 config_satisfiable: status.config_satisfiable,
9712 runtime_available: status.runtime_available,
9713 detail: status.detail,
9714 }
9715}
9716
9717fn append_capability_problem_detail(
9718 detail: Option<String>,
9719 capability_detail: Option<String>,
9720) -> Option<String> {
9721 match (detail, capability_detail) {
9722 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
9723 (Some(detail), None) => Some(detail),
9724 (None, Some(capability_detail)) => Some(capability_detail),
9725 (None, None) => None,
9726 }
9727}
9728
9729fn subc_ops() -> Vec<String> {
9730 SUBC_CONTROL_OPS
9731 .iter()
9732 .map(|op| (*op).to_string())
9733 .collect()
9734}
9735
9736fn module_subc_ops() -> Vec<String> {
9737 SUBC_CONTROL_OPS
9738 .iter()
9739 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
9740 .map(|op| (*op).to_string())
9741 .collect()
9742}
9743
9744#[cfg(test)]
9745fn module_baseline_control_ops() -> Vec<String> {
9746 MODULE_BASELINE_CONTROL_OPS
9747 .iter()
9748 .map(|op| (*op).to_string())
9749 .collect()
9750}
9751
9752fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
9753 let mut seen = HashSet::new();
9754 let mut effective = Vec::new();
9755 for op in MODULE_BASELINE_CONTROL_OPS {
9756 if seen.insert((*op).to_string()) {
9757 effective.push((*op).to_string());
9758 }
9759 }
9760 for op in declared.unwrap_or_default() {
9761 if seen.insert(op.clone()) {
9762 effective.push(op);
9763 }
9764 }
9765 effective
9766}
9767
9768fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
9769 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
9770}
9771
9772fn target_module_id(target: &RouteTarget) -> &str {
9773 match target {
9774 RouteTarget::ToolProvider { module_id }
9775 | RouteTarget::ManagementSurface { module_id }
9776 | RouteTarget::InternalService { module_id, .. } => module_id,
9777 }
9778}
9779
9780fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
9781 roles.iter().any(|role| match (target, role) {
9782 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
9783 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
9784 (
9785 RouteTarget::InternalService { service_id, .. },
9786 ProviderRole::InternalService {
9787 service_id: provided,
9788 ..
9789 },
9790 ) => service_id == provided,
9791 _ => false,
9792 })
9793}
9794
9795fn is_routable_role(role: &ProviderRole) -> bool {
9796 matches!(
9797 role,
9798 ProviderRole::ToolProvider { .. }
9799 | ProviderRole::ManagementSurface { .. }
9800 | ProviderRole::InternalService { .. }
9801 )
9802}
9803
9804#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9805enum ControlRequestBodyError {
9806 UnknownOp,
9807 InvalidBody,
9808}
9809
9810#[derive(Debug, Deserialize)]
9811struct ControlOpProbe {
9812 op: String,
9813}
9814
9815fn module_control_response_probe(body: &[u8]) -> Result<ControlOpProbe, serde_json::Error> {
9816 serde_json::from_slice(body).or_else(|original| {
9817 #[derive(Deserialize)]
9818 struct ConsentProbe {
9819 protocol: String,
9820 }
9821 match serde_json::from_slice::<ConsentProbe>(body) {
9822 Ok(probe) if probe.protocol == subc_protocol::consent::CONSENT_V1 => {
9823 Ok(ControlOpProbe { op: probe.protocol })
9824 }
9825 _ => Err(original),
9826 }
9827 })
9828}
9829
9830const MODULE_PUSH_OPS: &[&str] = &["route.status"];
9833
9834fn is_known_module_push_op(body: &[u8]) -> bool {
9835 serde_json::from_slice::<ControlOpProbe>(body)
9836 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
9837 .unwrap_or(false)
9838}
9839
9840fn is_known_module_request_op(body: &[u8]) -> bool {
9841 serde_json::from_slice::<ControlOpProbe>(body)
9842 .map(|probe| is_module_to_subc_op(&probe.op))
9843 .unwrap_or(false)
9844}
9845
9846fn is_module_to_subc_op(op: &str) -> bool {
9847 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
9848}
9849
9850fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
9851 debug!(
9852 op = %op,
9853 connection_id = connection_id.get(),
9854 corr,
9855 "control dispatch"
9856 );
9857}
9858
9859fn log_slow_control_dispatch(
9860 dispatch_started_at: Option<StdInstant>,
9861 op: &'static str,
9862 connection_id: ConnectionId,
9863 corr: u64,
9864 route_open_timings: Option<&RouteOpenTimings>,
9865) {
9866 let Some(dispatch_started_at) = dispatch_started_at else {
9867 return;
9868 };
9869 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
9871 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
9872 if let Some(timings) = route_open_timings {
9873 warn!(
9874 op = %op,
9875 connection_id = connection_id.get(),
9876 corr,
9877 elapsed_ms = elapsed.as_millis() as u64,
9878 module_id = ?timings.module_id,
9879 principal = %timings.principal,
9880 admission_ms = timings.admission.as_millis() as u64,
9881 project_root_ms = timings.project_root.as_millis() as u64,
9882 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
9883 bind_send_ms = timings.bind_send.as_millis() as u64,
9884 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
9885 "slow control dispatch"
9886 );
9887 return;
9888 }
9889 warn!(
9890 op = %op,
9891 connection_id = connection_id.get(),
9892 corr,
9893 elapsed_ms = elapsed.as_millis() as u64,
9894 "slow control dispatch"
9895 );
9896 }
9897}
9898
9899#[cfg(test)]
9901type TestCallerLookup =
9902 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
9903
9904fn privileged_operation(request: &impl PrivilegedRequest) -> Option<PrivilegedOperation<'_>> {
9906 request.privileged_operation()
9907}
9908
9909trait PrivilegedRequest {
9910 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>>;
9911}
9912
9913impl PrivilegedRequest for ClientControlRequest {
9914 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>> {
9915 let request = self;
9916 match request {
9917 ClientControlRequest::SupervisorRestart { module_id, .. } => {
9918 Some(PrivilegedOperation::Restart { module_id })
9919 }
9920 ClientControlRequest::SupervisorSwap { module_id, .. } => {
9921 Some(PrivilegedOperation::Swap { module_id })
9922 }
9923 ClientControlRequest::SupervisorReload { module_id } => {
9924 Some(PrivilegedOperation::Reload { module_id })
9925 }
9926 ClientControlRequest::SupervisorRescan { preview } => {
9927 if *preview {
9928 None
9929 } else {
9930 Some(PrivilegedOperation::Rescan)
9931 }
9932 }
9933 ClientControlRequest::SupervisorRescanChecked {
9938 preview,
9939 candidate,
9940 expect,
9941 } => (!*preview && candidate.is_none() && expect.is_some())
9942 .then_some(PrivilegedOperation::Rescan),
9943 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
9944 Some(PrivilegedOperation::SetEnabled {
9945 module_id,
9946 enabled: *enabled,
9947 })
9948 }
9949 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
9950 Some(PrivilegedOperation::ReleaseReserved { module_id })
9951 }
9952 ClientControlRequest::ServerDescribe {}
9953 | ClientControlRequest::DaemonShutdownIntent { .. }
9956 | ClientControlRequest::CatalogList { .. }
9957 | ClientControlRequest::RouteOpen { .. }
9958 | ClientControlRequest::RoutePoll { .. }
9959 | ClientControlRequest::SupervisorList {}
9960 | ClientControlRequest::SupervisorHealth {}
9961 | ClientControlRequest::SupervisorHealthProbe { .. }
9962 | ClientControlRequest::SupervisorRoutes { .. }
9963 | ClientControlRequest::SupervisorProvenance { .. }
9964 | ClientControlRequest::SupervisorStderrTail { .. }
9965 | ClientControlRequest::SupervisorTerminals { .. }
9966 | ClientControlRequest::SupervisorSpawnSnapshot {}
9967 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
9968 }
9969 }
9970}
9971
9972fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
9973 match request {
9974 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
9975 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
9976 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
9977 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
9978 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
9979 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
9980 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
9981 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
9982 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
9983 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
9984 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
9985 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
9986 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
9987 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
9988 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
9989 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
9990 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
9991 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
9992 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
9993 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
9994 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
9995 }
9996}
9997
9998struct RouteOpenCaller {
10006 principal: String,
10010 scope: String,
10013 harness: String,
10016}
10017
10018impl RouteOpenCaller {
10019 fn new(
10020 principal: Option<&Principal>,
10021 scope: Option<&ScopeSelector>,
10022 identity: &BindIdentity,
10023 ) -> Self {
10024 Self {
10025 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
10026 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
10027 harness: bounded_log_text(&identity.harness),
10028 }
10029 }
10030
10031 fn unknown() -> Self {
10034 Self {
10035 principal: "unknown".to_string(),
10036 scope: "unknown".to_string(),
10037 harness: "unknown".to_string(),
10038 }
10039 }
10040}
10041
10042const MAX_LOGGED_TEXT_CHARS: usize = 64;
10045
10046fn bounded_log_text(value: &str) -> String {
10050 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
10051 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
10052 bounded.push('…');
10053 }
10054 format!("{bounded:?}")
10055}
10056
10057fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
10063 use subc_protocol::scope::ScopeKind;
10064 let kind = match stamp.kind {
10065 ScopeKind::Head => "head",
10066 ScopeKind::Worker => "worker",
10067 ScopeKind::Ephemeral => "ephemeral",
10068 ScopeKind::Federated => "federated",
10069 _ => "unknown",
10070 };
10071 format!(
10072 "{kind}/{}/{}@{}",
10073 principal_label(&stamp.owner),
10074 bounded_log_text(&stamp.scope_ref),
10075 stamp.scope_epoch
10076 )
10077}
10078
10079fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
10086 let epoch = selector
10087 .scope_epoch
10088 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
10089 format!(
10090 "requested/{}/{}@{epoch}",
10091 bounded_log_text(&principal_label(&selector.owner)),
10092 bounded_log_text(&selector.scope_ref),
10093 )
10094}
10095
10096fn principal_label(principal: &Principal) -> String {
10097 match principal {
10098 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
10099 Principal::Direct => "direct".to_string(),
10100 other => format!("{other:?}"),
10101 }
10102}
10103
10104fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
10105 match request {
10106 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
10107 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
10108 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
10109 }
10110 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
10111 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
10112 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
10113 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
10114 ModuleControlRequestFromModule::RouteAuthorize { .. } => {
10115 subc_protocol::scope::ROUTE_AUTHORIZE_OP
10116 }
10117 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
10118 _ => "unsupported",
10119 }
10120}
10121
10122fn parse_client_control_request(
10123 body: &[u8],
10124) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
10125 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
10126 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10127 Ok(probe)
10128 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
10129 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
10130 {
10131 ControlRequestBodyError::InvalidBody
10132 }
10133 Ok(_) => ControlRequestBodyError::UnknownOp,
10134 Err(_) => ControlRequestBodyError::InvalidBody,
10135 };
10136 (err, classification)
10137 })
10138}
10139
10140fn parse_module_control_request_from_module(
10141 body: &[u8],
10142) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
10143 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
10144 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10145 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
10146 Ok(_) => ControlRequestBodyError::UnknownOp,
10147 Err(_) => ControlRequestBodyError::InvalidBody,
10148 };
10149 (err, classification)
10150 })
10151}
10152
10153#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
10154enum ProviderRoleKind {
10155 ToolProvider,
10156 PipelineStage,
10157 ManagementSurface,
10158 InternalService,
10159}
10160
10161fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
10162 match role {
10163 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
10164 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
10165 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
10166 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
10167 }
10168}
10169
10170fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
10171 roles.iter().map(provider_role_kind).collect()
10172}
10173
10174const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
10177
10178#[derive(Debug, Default, PartialEq, Eq)]
10180struct ScopeOutcomeCounts {
10181 created: usize,
10182 replaced: usize,
10183 updated: usize,
10184 unchanged: usize,
10185 refused: usize,
10186}
10187
10188impl ScopeOutcomeCounts {
10189 fn of(results: &[ScopeRecordResult]) -> Self {
10190 let mut counts = Self::default();
10191 for result in results {
10192 let slot = match result.outcome {
10193 ScopeRecordOutcome::Created => &mut counts.created,
10194 ScopeRecordOutcome::Replaced => &mut counts.replaced,
10195 ScopeRecordOutcome::Updated => &mut counts.updated,
10196 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
10197 ScopeRecordOutcome::Refused => &mut counts.refused,
10198 };
10199 *slot += 1;
10200 }
10201 counts
10202 }
10203}
10204
10205#[cfg(test)]
10206mod scope_outcome_count_tests {
10207 use super::*;
10208
10209 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
10210 ScopeRecordResult {
10211 scope_ref: "r".to_string(),
10212 scope_epoch: 1,
10213 outcome,
10214 code: None,
10215 message: None,
10216 version: None,
10217 parent_state: None,
10218 }
10219 }
10220
10221 #[test]
10224 fn every_outcome_is_counted_in_its_own_field() {
10225 let results = [
10226 result(ScopeRecordOutcome::Created),
10227 result(ScopeRecordOutcome::Created),
10228 result(ScopeRecordOutcome::Replaced),
10229 result(ScopeRecordOutcome::Updated),
10230 result(ScopeRecordOutcome::Unchanged),
10231 result(ScopeRecordOutcome::Refused),
10232 result(ScopeRecordOutcome::Refused),
10233 result(ScopeRecordOutcome::Refused),
10234 ];
10235 assert_eq!(
10236 ScopeOutcomeCounts::of(&results),
10237 ScopeOutcomeCounts {
10238 created: 2,
10239 replaced: 1,
10240 updated: 1,
10241 unchanged: 1,
10242 refused: 3,
10243 }
10244 );
10245 }
10246}
10247
10248fn capability_census_trigger(
10252 old: Option<&CapabilityDeclarations>,
10253 new: Option<&CapabilityDeclarations>,
10254) -> bool {
10255 let old_provides = old
10256 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
10257 .unwrap_or_default();
10258 let old_denies = old
10259 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
10260 .unwrap_or_default();
10261 let new = new.cloned().unwrap_or(CapabilityDeclarations {
10262 provides: Vec::new(),
10263 requires: Vec::new(),
10264 must_never_reach: Vec::new(),
10265 });
10266
10267 new.provides
10268 .iter()
10269 .any(|capability| !old_provides.contains(capability))
10270 || new
10271 .must_never_reach
10272 .iter()
10273 .any(|capability| !old_denies.contains(capability))
10274}
10275
10276fn catalog_update_frozen_field_message(
10277 registered: &ModuleManifest,
10278 provides: &[ProviderRole],
10279) -> Option<String> {
10280 let old_has_provides = !registered.provides.is_empty();
10281 let new_has_provides = !provides.is_empty();
10282 if old_has_provides != new_has_provides {
10283 return Some(format!(
10284 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
10285 registered.module_id
10286 ));
10287 }
10288
10289 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
10290 return Some(format!(
10291 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
10292 registered.module_id
10293 ));
10294 }
10295
10296 let registered_concurrency = manifest_concurrency(registered);
10297 let mut candidate = registered.clone();
10298 candidate.provides = provides.to_vec();
10299 let candidate_concurrency = manifest_concurrency(&candidate);
10300 if candidate_concurrency != registered_concurrency {
10301 return Some(format!(
10302 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
10303 registered.module_id, registered_concurrency, candidate_concurrency
10304 ));
10305 }
10306
10307 None
10310}
10311
10312fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
10313 manifest.provides.iter().any(is_routable_role)
10314}
10315
10316fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
10322 manifest
10323 .provides
10324 .iter()
10325 .find_map(|provider| match provider {
10326 ProviderRole::ToolProvider { concurrency, .. }
10327 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
10328 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
10329 })
10330 .unwrap_or(Concurrency::ModuleManaged)
10331}
10332
10333fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
10340 let has_management_surface = manifest
10341 .provides
10342 .iter()
10343 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
10344 if !has_management_surface {
10345 return false;
10346 }
10347 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
10348 return false;
10349 };
10350 let Some(provides) = raw
10351 .get("manifest")
10352 .and_then(|manifest| manifest.get("provides"))
10353 .and_then(serde_json::Value::as_array)
10354 else {
10355 return false;
10356 };
10357 provides.iter().any(|role| {
10362 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
10363 && role.get("concurrency").is_none()
10364 })
10365}
10366
10367fn negotiate_version(peer_version: u8) -> Result<u8, String> {
10368 if peer_version != PROTOCOL_VERSION {
10369 return Err(format!(
10370 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
10371 ));
10372 }
10373 Ok(PROTOCOL_VERSION)
10374}
10375
10376fn pong(frame: &Frame) -> Result<Frame, RouterError> {
10377 Frame::build_with_version(
10378 response_version(frame),
10379 FrameType::Pong,
10380 frame.header.flags,
10381 0,
10382 0,
10383 frame.header.corr,
10384 Vec::new(),
10385 )
10386 .map_err(RouterError::FrameBuild)
10387}
10388
10389pub(crate) fn control_error_frame(
10390 frame: &Frame,
10391 code: &'static str,
10392 message: impl Into<String>,
10393) -> Result<Frame, RouterError> {
10394 control_error_body_frame(
10395 frame,
10396 ErrorBody {
10397 code: code.to_string(),
10398 message: message.into(),
10399 detail: None,
10400 },
10401 )
10402}
10403
10404pub(crate) fn control_error_body_frame(
10412 frame: &Frame,
10413 error: ErrorBody,
10414) -> Result<Frame, RouterError> {
10415 control_error_frame_with_flags(frame, error, daemon_error_flags())
10416}
10417
10418fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
10424 control_error_frame_with_flags(frame, error, control_flags())
10425}
10426
10427fn control_error_frame_with_flags(
10428 frame: &Frame,
10429 error: ErrorBody,
10430 flags: Flags,
10431) -> Result<Frame, RouterError> {
10432 let body = serde_json::to_vec(&error).map_err(|err| {
10433 RouterError::backend(
10434 0,
10435 frame.header.corr,
10436 format!("failed to encode control ERROR: {err}"),
10437 )
10438 })?;
10439
10440 Frame::build_with_version(
10441 response_version(frame),
10442 FrameType::Error,
10443 flags,
10444 0,
10445 0,
10446 frame.header.corr,
10447 body,
10448 )
10449 .map_err(RouterError::FrameBuild)
10450}
10451
10452pub(crate) fn control_response_body_frame<T: Serialize>(
10453 frame: &Frame,
10454 reply: &T,
10455 label: &'static str,
10456) -> Result<Frame, RouterError> {
10457 let body = serde_json::to_vec(reply).map_err(|err| {
10458 RouterError::backend(
10459 0,
10460 frame.header.corr,
10461 format!("failed to encode {label}: {err}"),
10462 )
10463 })?;
10464
10465 Frame::build_with_version(
10466 response_version(frame),
10467 FrameType::Response,
10468 control_flags(),
10469 0,
10470 0,
10471 frame.header.corr,
10472 body,
10473 )
10474 .map_err(RouterError::FrameBuild)
10475}
10476
10477struct HelloAuthority {
10510 record: Arc<crate::spawn_authority::SpawnAuthorityRecord>,
10511 requested: ProviderDeclarations,
10512 froze: std::sync::atomic::AtomicBool,
10515}
10516
10517fn forwarding_error_code(err: &ForwardingError) -> &'static str {
10518 match err {
10519 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
10520 ForwardingError::NoModuleConnection => "target_unavailable",
10521 ForwardingError::ModuleReloading { .. } => "module_reloading",
10522 ForwardingError::ClientRouteChannelExhausted { .. }
10523 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
10524 ForwardingError::StaleModuleEndpoint
10525 | ForwardingError::UnknownReservation { .. }
10526 | ForwardingError::ConnectionClosing { .. }
10527 | ForwardingError::ClientEgressClosed { .. }
10528 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
10529 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
10532 ForwardingError::RelayCorrelationExhausted
10533 | ForwardingError::RouteOpenBuild(_)
10534 | ForwardingError::Poisoned => "forwarding_error",
10535 }
10536}
10537
10538fn response_version(frame: &Frame) -> u8 {
10539 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
10540 frame.header.ver
10541 } else {
10542 PROTOCOL_VERSION
10543 }
10544}
10545
10546fn control_flags() -> Flags {
10547 Flags::new(false, Priority::Passive, false)
10548}
10549
10550fn send_goodbye_target_best_effort(
10555 counters: &DaemonCounters,
10556 target: &GoodbyeTarget,
10557 context: &'static str,
10558) {
10559 let Ok(frame) = Frame::build_with_version(
10560 target.negotiated_ver,
10561 FrameType::Goodbye,
10562 control_flags(),
10563 target.channel,
10564 target.epoch,
10565 0,
10566 Vec::new(),
10567 ) else {
10568 return;
10569 };
10570 crate::forwarding::send_module_route_goodbye(
10571 counters,
10572 &target.sink,
10573 frame,
10574 target.module_id.as_deref(),
10575 context,
10576 );
10577}
10578
10579fn route_close_reason_for_opener(
10580 reason: RouteCloseReason,
10581 accepts_close_reasons: Option<&[String]>,
10582) -> RouteCloseReason {
10583 if reason == RouteCloseReason::AuthorityChanged
10584 && !accepts_close_reasons
10585 .is_some_and(|reasons| reasons.iter().any(|reason| reason == "authority_changed"))
10586 {
10587 RouteCloseReason::Reload
10590 } else {
10591 reason
10592 }
10593}
10594
10595pub(crate) fn send_route_control_pushes(
10596 forwarding: &ForwardingTable,
10597 routes: Vec<EndpointRoute>,
10598 push: ClientControlPush,
10599) {
10600 let reason = match &push {
10601 ClientControlPush::RouteClosing { reason, .. }
10602 | ClientControlPush::RouteClosed { reason, .. } => *reason,
10603 };
10604 let mut targets: Vec<(GoodbyeTarget, RouteCloseReason, Vec<u16>)> = Vec::new();
10605 for route in routes {
10606 let recipient_reason =
10607 route_close_reason_for_opener(reason, route.accepts_close_reasons.as_deref());
10608 let target = route.goodbye_target;
10609 if let Some((existing, _, channels)) = targets.iter_mut().find(|(existing, reason, _)| {
10610 existing.connection_id == target.connection_id && *reason == recipient_reason
10611 }) {
10612 debug_assert_eq!(
10613 existing.negotiated_ver, target.negotiated_ver,
10614 "one connection cannot negotiate multiple frame versions"
10615 );
10616 if !channels.contains(&target.channel) {
10617 channels.push(target.channel);
10618 }
10619 continue;
10620 }
10621 let channel = target.channel;
10622 targets.push((target, recipient_reason, vec![channel]));
10623 }
10624 for (target, recipient_reason, mut channels) in targets {
10625 channels.sort_unstable();
10626 let mut push = push.clone();
10627 match &mut push {
10628 ClientControlPush::RouteClosing {
10629 channels: covered,
10630 reason,
10631 ..
10632 }
10633 | ClientControlPush::RouteClosed {
10634 channels: covered,
10635 reason,
10636 ..
10637 } => {
10638 *covered = channels;
10639 *reason = recipient_reason;
10640 }
10641 }
10642 let body = match serde_json::to_vec(&push) {
10643 Ok(body) => body,
10644 Err(err) => {
10645 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
10646 continue;
10647 }
10648 };
10649 let frame = match Frame::build_with_version(
10650 target.negotiated_ver,
10651 FrameType::Push,
10652 control_flags(),
10653 0,
10654 0,
10655 0,
10656 body.clone(),
10657 ) {
10658 Ok(frame) => frame,
10659 Err(err) => {
10660 warn!(
10661 route_channel = target.channel,
10662 error = %err,
10663 "failed to build route lifecycle control PUSH frame"
10664 );
10665 continue;
10666 }
10667 };
10668 if let Err(err) = target.sink.try_send(frame) {
10669 if target.close_on_delivery_failure() {
10670 warn!(
10671 target_connection_id = target.connection_id.get(),
10672 route_channel = target.channel,
10673 error = %err,
10674 "route lifecycle control PUSH was not delivered to client; closing target connection"
10675 );
10676 let _ = forwarding.escalate_client_delivery_failure(
10677 target.connection_id,
10678 target.channel,
10679 target.epoch,
10680 CloseReason::new(
10681 "route_lifecycle_push_delivery_failed",
10682 format!(
10683 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
10684 target.channel
10685 ),
10686 ),
10687 crate::forwarding::UndeliveredFrame {
10688 module_id: target.module_id.as_deref(),
10689 sink: &target.sink,
10690 },
10691 );
10692 }
10693 }
10694 }
10695}
10696
10697#[cfg(test)]
10698mod tests {
10699 include!("control/agent_commands_tests.rs");
10700 include!("control/resource_revision_tests.rs");
10701 include!("control/route_open_recovery_tests.rs");
10702 #[tokio::test]
10703 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
10704 let registry = Arc::new(Registry::default());
10705 let handler = ControlHandler::new(registry);
10706 assert_eq!(
10707 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10708 None
10709 );
10710 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
10711 hello_via_sink(
10712 &handler,
10713 &ctx,
10714 &mut rx,
10715 hello_frame("daemon", PROTOCOL_VERSION, 1),
10716 )
10717 .await;
10718 assert_eq!(
10719 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10720 Some("daemon".into())
10721 );
10722 assert_eq!(
10723 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
10724 None
10725 );
10726 assert_eq!(
10727 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
10728 None,
10729 "a registered module is not necessarily supervised"
10730 );
10731 assert_eq!(
10732 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
10733 None,
10734 "aggregate provenance has no single target"
10735 );
10736 assert_eq!(
10737 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
10738 None
10739 );
10740 }
10741
10742 mod peer_manifest {
10743 include!("peer_manifest_tests.rs");
10744 }
10745 mod catalog_declarations {
10746 include!("control/catalog_tests.rs");
10747 }
10748 mod activity_read {
10749 include!("activity_read_tests.rs");
10750 }
10751
10752 mod lifecycle_activity {
10753 include!("control_lifecycle_tests.rs");
10754 }
10755 #[cfg(unix)]
10756 #[tokio::test]
10757 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
10758 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
10759 let path = dir.join("subc.jsonc");
10760 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
10761 "program":"/bin/sleep","args":["60"],"protocol":"none",
10762 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
10763 }}}).to_string()).unwrap();
10764 let mut configured = crate::daemon_config::load(&path)
10765 .unwrap()
10766 .unwrap()
10767 .modules
10768 .pop()
10769 .unwrap();
10770 let registry = std::sync::Arc::new(crate::Registry::default());
10771 let handle = crate::SupervisorHandle::new();
10772 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
10773 .with_handle(handle.clone());
10774 let module = supervisor
10775 .supervise_configured_with_health(
10776 configured.module_spec(),
10777 true,
10778 configured.health.clone(),
10779 None,
10780 configured.restart,
10781 )
10782 .unwrap();
10783 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
10784 let before = module.status().unwrap().pid;
10785 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
10786 configured.health.cadence = std::time::Duration::from_secs(3600);
10787 let health_only = handler
10788 .reconcile_supervised_modules(
10789 &supervisor,
10790 vec![configured.clone()],
10791 false,
10792 Principal::Direct,
10793 )
10794 .await
10795 .unwrap();
10796 assert!(
10797 health_only.0.changed_pending_reload.is_empty(),
10798 "health policy is already applied live"
10799 );
10800 assert_eq!(module.status().unwrap().pid, before);
10801 assert_eq!(
10802 module.configuration().unwrap().1.http,
10803 configured.health.http
10804 );
10805 configured.args = vec!["61".into()];
10806 let launch = handler
10807 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
10808 .await
10809 .unwrap();
10810 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
10811 assert_eq!(
10812 module.status().unwrap().pid,
10813 before,
10814 "a launch edit is stored until reload"
10815 );
10816 module.drain().await.unwrap();
10817 }
10818 use cortexkit_test_support::ScratchDir;
10819 use std::{
10820 collections::BTreeMap,
10821 fmt,
10822 path::PathBuf,
10823 sync::{Arc, Mutex},
10824 time::Duration,
10825 };
10826
10827 use serde_json::{json, Value};
10828 use subc_protocol::{
10829 manifest::{
10830 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
10831 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
10832 },
10833 session::HealthStatus,
10834 FrameType,
10835 };
10836
10837 use super::*;
10838 use crate::{
10839 forwarding::{DataRoute, DataRouteState},
10840 registry::ChannelState,
10841 router::FrameSink,
10842 stderr_tail::DEFAULT_MAX_LINE_BYTES,
10843 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
10844 RouteCtx, Router,
10845 };
10846 use tokio::{
10847 sync::mpsc,
10848 time::{sleep, Instant},
10849 };
10850 use tracing::{
10851 field::{Field, Visit},
10852 Event, Subscriber,
10853 };
10854 use tracing_subscriber::{layer::Context, prelude::*, Layer};
10855
10856 fn fake_aft_stub_path() -> PathBuf {
10875 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
10876 assert!(
10877 path.exists(),
10878 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
10879 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
10880 path.display()
10881 );
10882 path
10883 }
10884
10885 fn client_retries(code: &str) -> bool {
10890 subc_protocol::error_codes::is_retryable_route_open(code)
10891 }
10892
10893 #[test]
10904 fn retryability_of_forwarding_codes_matches_the_failure() {
10905 let transient = [
10908 ForwardingError::NoModuleConnection,
10909 ForwardingError::ModuleReloading {
10910 module_id: "m".into(),
10911 },
10912 ForwardingError::StaleModuleEndpoint,
10913 ForwardingError::UnknownReservation {
10914 client_channel: 1,
10915 module_channel: 1,
10916 },
10917 ForwardingError::ConnectionClosing {
10918 connection_id: ConnectionId::new(1),
10919 },
10920 ForwardingError::ClientEgressClosed {
10921 connection_id: ConnectionId::new(1),
10922 },
10923 ForwardingError::ModuleEgressUnavailable {
10924 connection_id: ConnectionId::new(1),
10925 },
10926 ];
10927 for err in transient {
10928 let code = forwarding_error_code(&err);
10929 assert!(
10930 client_retries(code),
10931 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
10932 );
10933 }
10934
10935 let permanent = [
10940 ForwardingError::ConnectionRoleConflict {
10941 connection_id: ConnectionId::new(1),
10942 },
10943 ForwardingError::ClientRouteChannelExhausted {
10944 connection_id: ConnectionId::new(1),
10945 },
10946 ForwardingError::ModuleRouteChannelExhausted {
10947 endpoint: ModuleEndpointId {
10948 connection_id: ConnectionId::new(1),
10949 generation: 1,
10950 },
10951 },
10952 ForwardingError::RelayCorrelationExhausted,
10953 ForwardingError::RouteOpenBuild("x".into()),
10954 ForwardingError::Poisoned,
10955 ];
10956 for err in permanent {
10957 let code = forwarding_error_code(&err);
10958 assert!(
10959 !client_retries(code),
10960 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
10961 );
10962 }
10963 }
10964
10965 #[tokio::test]
10993 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
10994 let handler = ControlHandler::default();
10995 let frame =
10996 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
10997
10998 let stamped = handler
11002 .route_open_principal(ConnectionId::new(1), &frame, None)
11003 .unwrap()
11004 .unwrap();
11005 assert_eq!(
11006 stamped,
11007 Principal::Direct,
11008 "a caller that proved nothing must not be stamped as a supervised module"
11009 );
11010
11011 let forged = handler
11016 .route_open_principal(
11017 ConnectionId::new(1),
11018 &frame,
11019 Some(ConsumerIdentity {
11020 module_id: "aft".to_string(),
11021 launch_nonce: "not-a-real-nonce".to_string(),
11022 }),
11023 )
11024 .unwrap();
11025 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
11026 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
11027 }
11028
11029 #[test]
11046 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
11047 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
11048 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
11049 let ClientControlRequest::RouteOpen {
11050 consumer_identity, ..
11051 } = parsed
11052 else {
11053 panic!("route.open body must parse as RouteOpen");
11054 };
11055 assert_eq!(
11056 consumer_identity,
11057 Some(ConsumerIdentity {
11058 module_id: "aft".to_string(),
11059 launch_nonce: "n".to_string(),
11060 }),
11061 "the wire field name must reach the value route_open_principal reads"
11062 );
11063 }
11064
11065 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
11066 ModuleManifest::builder(module_id, "0.1.0")
11067 .protocol_ver(protocol_ver)
11068 .provides(vec![ProviderRole::ToolProvider {
11069 tools: vec![Tool {
11070 name: "read".to_string(),
11071 description: None,
11072 execution_mode: ExecutionMode::Pure,
11073 schema: json!({"type": "object"}),
11074 }],
11075 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
11076 concurrency: Concurrency::ModuleManaged,
11077 emits_push: true,
11078 sub_supervises: true,
11079 }])
11080 .build()
11081 }
11082
11083 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
11084 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
11085 }
11086
11087 fn hello_frame_with_control_ops(
11088 module_id: &str,
11089 protocol_ver: u8,
11090 corr: u64,
11091 control_ops: Option<Vec<String>>,
11092 ) -> Frame {
11093 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
11094 }
11095
11096 fn hello_frame_with_nonce(
11097 module_id: &str,
11098 protocol_ver: u8,
11099 corr: u64,
11100 launch_nonce: Option<&str>,
11101 ) -> Frame {
11102 hello_frame_full(
11103 module_id,
11104 protocol_ver,
11105 corr,
11106 None,
11107 launch_nonce.map(ToOwned::to_owned),
11108 )
11109 }
11110
11111 fn hello_frame_full(
11112 module_id: &str,
11113 protocol_ver: u8,
11114 corr: u64,
11115 control_ops: Option<Vec<String>>,
11116 launch_nonce: Option<String>,
11117 ) -> Frame {
11118 let body = serde_json::to_vec(&ModuleHelloBody {
11119 manifest: manifest(module_id, protocol_ver),
11120 protocol_ver,
11121 control_ops,
11122 launch_nonce,
11123 })
11124 .unwrap();
11125 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11126 }
11127
11128 fn hello_frame_with_rich_ui(
11129 module_id: &str,
11130 protocol_ver: u8,
11131 corr: u64,
11132 launch_nonce: Option<&str>,
11133 rich_ui: Value,
11134 ) -> Frame {
11135 let mut body = serde_json::to_value(ModuleHelloBody {
11136 manifest: manifest(module_id, protocol_ver),
11137 protocol_ver,
11138 control_ops: None,
11139 launch_nonce: launch_nonce.map(str::to_string),
11140 })
11141 .unwrap();
11142 body["manifest"]["rich_ui"] = rich_ui;
11143 Frame::build(
11144 FrameType::Hello,
11145 control_flags(),
11146 0,
11147 0,
11148 corr,
11149 serde_json::to_vec(&body).unwrap(),
11150 )
11151 .unwrap()
11152 }
11153
11154 fn non_routable_hello_frame_with_control_ops(
11155 module_id: &str,
11156 corr: u64,
11157 control_ops: Option<Vec<String>>,
11158 ) -> Frame {
11159 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11160 manifest.provides.clear();
11161 let body = serde_json::to_vec(&ModuleHelloBody {
11162 manifest,
11163 protocol_ver: PROTOCOL_VERSION,
11164 control_ops,
11165 launch_nonce: None,
11166 })
11167 .unwrap();
11168 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11169 }
11170
11171 fn capability_grammar_hello_frame(
11172 capabilities: Value,
11173 runtime_computed: Option<Value>,
11174 corr: u64,
11175 ) -> Frame {
11176 let mut body = serde_json::to_value(ModuleHelloBody {
11177 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
11178 protocol_ver: PROTOCOL_VERSION,
11179 control_ops: None,
11180 launch_nonce: None,
11181 })
11182 .expect("HELLO body serializes");
11183 body["manifest"]["capabilities"] = capabilities;
11184 if let Some(runtime_computed) = runtime_computed {
11185 body["runtime_computed"] = runtime_computed;
11186 }
11187 Frame::build(
11188 FrameType::Hello,
11189 control_flags(),
11190 0,
11191 0,
11192 corr,
11193 serde_json::to_vec(&body).expect("HELLO body reserializes"),
11194 )
11195 .expect("HELLO frame builds")
11196 }
11197
11198 fn channel_request(channel: u16, corr: u64) -> Frame {
11199 Frame::build(
11200 FrameType::Request,
11201 Flags::new(true, Priority::Interactive, false),
11202 channel,
11203 0,
11204 corr,
11205 b"opaque".to_vec(),
11206 )
11207 .unwrap()
11208 }
11209
11210 fn route_ctx(
11211 connection_id: ConnectionId,
11212 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11213 let (tx, rx) = mpsc::channel(8);
11214 (
11215 RouteCtx {
11216 connection_id,
11217 egress: FrameSink::new(tx),
11218 },
11219 rx,
11220 )
11221 }
11222
11223 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
11224 serde_json::from_slice(&frame.body).unwrap()
11225 }
11226
11227 async fn hello_via_sink(
11232 handler: &ControlHandler,
11233 ctx: &RouteCtx,
11234 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11235 hello: Frame,
11236 ) -> Frame {
11237 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
11238 assert!(
11239 replies.is_empty(),
11240 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
11241 );
11242 let ack = rx
11243 .try_recv()
11244 .expect("HELLO_ACK is queued on the module sink")
11245 .frame;
11246 assert_eq!(ack.header.ty, FrameType::HelloAck);
11247 ack
11248 }
11249
11250 fn parse_error(frame: &Frame) -> Value {
11251 serde_json::from_slice(&frame.body).unwrap()
11252 }
11253
11254 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
11255 serde_json::from_slice(&frame.body).unwrap()
11256 }
11257
11258 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
11259 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
11260 route_channel,
11261 route_epoch: 0,
11262 kind,
11263 })
11264 .unwrap();
11265 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11266 }
11267
11268 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
11269 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
11270 module_id: module_id.to_string(),
11271 })
11272 .unwrap();
11273 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11274 }
11275
11276 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
11277 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
11278 }
11279
11280 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
11282 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
11283 let mut frame = route_open_frame(corr, module_id, root);
11284 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
11285 body["identity"]["project_root"] = serde_json::json!(path.as_path());
11286 frame.body = serde_json::to_vec(&body).unwrap();
11287 frame
11288 }
11289
11290 fn route_open_frame_with_consumer_capabilities(
11291 corr: u64,
11292 module_id: &str,
11293 _project_root: ScratchDir,
11294 consumer_capabilities: Option<Vec<String>>,
11295 ) -> Frame {
11296 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11297 busy_refusals: None,
11298 target: RouteTarget::ToolProvider {
11299 module_id: module_id.to_string(),
11300 },
11301 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
11302 consumer_identity: None,
11303 consumer_capabilities,
11304 role_versions: None,
11305 admission_facts: None,
11306 scope: None,
11307 })
11308 .unwrap();
11309 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11310 }
11311
11312 fn route_open_frame_with_role_versions(
11313 corr: u64,
11314 module_id: &str,
11315 _project_root: ScratchDir,
11316 role_versions: Option<BTreeMap<String, String>>,
11317 ) -> Frame {
11318 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11319 busy_refusals: None,
11320 target: RouteTarget::ToolProvider {
11321 module_id: module_id.to_string(),
11322 },
11323 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11324 consumer_identity: None,
11325 consumer_capabilities: None,
11326 role_versions,
11327 admission_facts: None,
11328 scope: None,
11329 })
11330 .unwrap();
11331 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11332 }
11333
11334 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
11335 entries
11336 .iter()
11337 .map(|(role, version)| (role.to_string(), version.to_string()))
11338 .collect()
11339 }
11340
11341 fn route_open_frame_with_admission_facts(
11342 corr: u64,
11343 module_id: &str,
11344 _project_root: ScratchDir,
11345 consumer_identity: Option<subc_control::ConsumerIdentity>,
11346 facts: Option<Value>,
11347 ) -> Frame {
11348 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11349 busy_refusals: None,
11350 target: RouteTarget::ToolProvider {
11351 module_id: module_id.to_string(),
11352 },
11353 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11354 consumer_identity,
11355 consumer_capabilities: None,
11356 role_versions: None,
11357 admission_facts: facts,
11358 scope: None,
11359 })
11360 .unwrap();
11361 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11362 }
11363
11364 #[derive(Clone, Default)]
11365 struct EventCapture {
11366 events: Arc<Mutex<Vec<CapturedEvent>>>,
11367 }
11368
11369 #[derive(Clone, Debug)]
11370 struct CapturedEvent {
11371 target: String,
11372 level: tracing::Level,
11373 fields: BTreeMap<String, String>,
11374 }
11375
11376 impl EventCapture {
11377 fn events(&self) -> Vec<CapturedEvent> {
11378 self.events.lock().unwrap().clone()
11379 }
11380 }
11381
11382 impl<S> Layer<S> for EventCapture
11383 where
11384 S: Subscriber,
11385 {
11386 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
11387 let mut visitor = EventFieldVisitor::default();
11388 event.record(&mut visitor);
11389 self.events.lock().unwrap().push(CapturedEvent {
11390 target: event.metadata().target().to_string(),
11391 level: *event.metadata().level(),
11392 fields: visitor.fields,
11393 });
11394 }
11395 }
11396
11397 #[derive(Default)]
11398 struct EventFieldVisitor {
11399 fields: BTreeMap<String, String>,
11400 }
11401
11402 impl Visit for EventFieldVisitor {
11403 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
11404 self.fields
11405 .insert(field.name().to_string(), format!("{value:?}"));
11406 }
11407 }
11408
11409 fn health_response(corr: u64, status: HealthStatus) -> Frame {
11410 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
11411 status,
11412 detail: Some("warming".to_string()),
11413 metrics: Some(json!({"queue_depth": 3})),
11414 })
11415 .unwrap();
11416 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11417 }
11418
11419 fn route_bind_ack(corr: u64) -> Frame {
11420 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
11421 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11422 }
11423
11424 fn unique_project_root(label: &str) -> ScratchDir {
11425 ScratchDir::new(label)
11426 }
11427
11428 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
11429 match parse_route_poll(frame) {
11430 ClientControlResponse::RoutePoll {
11431 status: None,
11432 live: Some(live),
11433 ..
11434 } => assert_eq!(live, expected_live),
11435 other => panic!("unexpected route.poll response: {other:?}"),
11436 }
11437 }
11438
11439 fn bind_liveness_route(
11440 registry: &Registry,
11441 forwarding: &ForwardingTable,
11442 module_id: &str,
11443 ) -> (RouteCtx, u16, u32) {
11444 let module_connection = ConnectionId::new(101);
11445 let client_connection = ConnectionId::new(202);
11446 let registration = registry
11447 .register_with_control_ops(
11448 manifest(module_id, PROTOCOL_VERSION),
11449 PROTOCOL_VERSION,
11450 module_connection,
11451 module_baseline_control_ops(),
11452 )
11453 .unwrap();
11454 let (module_tx, _module_rx) = mpsc::channel(8);
11455 let endpoint = forwarding
11456 .register_module_connection(
11457 module_connection,
11458 module_id.to_string(),
11459 PROTOCOL_VERSION,
11460 manifest_concurrency(®istration.manifest),
11461 FrameSink::new(module_tx),
11462 )
11463 .unwrap();
11464 let (client_ctx, _client_rx) = route_ctx(client_connection);
11465 let pending = forwarding
11466 .begin_route_bind_relay_for_test(
11467 client_connection,
11468 client_ctx.egress.clone(),
11469 1,
11470 module_id,
11471 )
11472 .unwrap();
11473 assert_eq!(pending.endpoint, endpoint);
11474 let route_channel = pending.client_channel;
11475 let route_epoch = pending.client_epoch;
11476 forwarding
11477 .complete_pending_relay(
11478 module_connection,
11479 pending.corr,
11480 RouteBindRelayOutcome::Accepted,
11481 )
11482 .unwrap();
11483 (client_ctx, route_channel, route_epoch)
11484 }
11485
11486 struct FakeProcessLiveness {
11487 live: Option<bool>,
11488 }
11489
11490 impl ModuleProcessLiveness for FakeProcessLiveness {
11491 fn process_live(&self, _module_id: &str) -> Option<bool> {
11492 self.live
11493 }
11494 }
11495
11496 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11497 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
11498 {
11499 let registry = Arc::new(Registry::default());
11500 let supervisor_handle = SupervisorHandle::new();
11501 let supervisor = Supervisor::new_for_test(
11502 Arc::clone(®istry),
11503 RestartPolicy::new(1, Duration::from_millis(10)),
11504 )
11505 .with_handle(supervisor_handle.clone());
11506 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
11507 let module = supervisor
11508 .spawn(ModuleSpec {
11509 script: None,
11510 carry_containment: false,
11511 module_id: "stderr-tail-wire".to_string(),
11512 program: fake_aft_stub_path(),
11513 args: Vec::new(),
11514 env: vec![
11515 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
11516 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
11517 ],
11518 reserved: false,
11519 reserved_prefixes: Vec::new(),
11520 protocol: ModuleProtocol::Subc,
11521 overlap: Default::default(),
11522 })
11523 .unwrap();
11524
11525 let deadline = Instant::now() + Duration::from_secs(5);
11526 loop {
11527 let tail = module.stderr_tail(None, None);
11528 if tail
11529 .entries
11530 .iter()
11531 .any(|entry| matches!(entry, TailEntry::ProcessStart))
11532 && tail.entries.iter().any(|entry| {
11533 matches!(
11534 entry,
11535 TailEntry::Line {
11536 truncated: true,
11537 ..
11538 }
11539 )
11540 })
11541 {
11542 break;
11543 }
11544 assert!(
11545 Instant::now() < deadline,
11546 "module did not produce a truncated line and restart boundary: {tail:?}"
11547 );
11548 sleep(Duration::from_millis(10)).await;
11549 }
11550
11551 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11552 let request = ClientControlRequest::SupervisorStderrTail {
11553 module_id: "stderr-tail-wire".to_string(),
11554 max_lines: None,
11555 max_bytes: None,
11556 };
11557 let frame = Frame::build(
11558 FrameType::Request,
11559 control_flags(),
11560 0,
11561 0,
11562 1,
11563 serde_json::to_vec(&request).unwrap(),
11564 )
11565 .unwrap();
11566 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11567 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11568 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
11569 serde_json::from_slice(&responses[0].body).unwrap()
11570 else {
11571 panic!("expected supervisor.stderr_tail response");
11572 };
11573
11574 assert!(
11575 tail.entries
11576 .iter()
11577 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
11578 "the control response lost the restart boundary"
11579 );
11580 let Some(StderrTailEntry::Line {
11581 text,
11582 truncated,
11583 at_ms,
11584 }) = tail.entries.iter().find(|entry| {
11585 matches!(
11586 entry,
11587 StderrTailEntry::Line {
11588 truncated: true,
11589 ..
11590 }
11591 )
11592 })
11593 else {
11594 panic!("the control response lost the truncated line");
11595 };
11596 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
11597 assert!(*truncated);
11598 assert!(
11599 at_ms.is_some(),
11600 "the control response lost the line's capture time"
11601 );
11602 }
11603
11604 #[tokio::test(flavor = "current_thread")]
11609 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
11610 let dir = ScratchDir::new("terminals-off-worker");
11611 let journal_path = dir.join("terminals.jsonl");
11612 let registry = Arc::new(Registry::default());
11613 let supervisor_handle = SupervisorHandle::new();
11614 let supervisor =
11615 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11616 .with_handle(supervisor_handle.clone())
11617 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
11618 let module = supervisor
11619 .spawn(ModuleSpec {
11620 script: None,
11621 carry_containment: false,
11622 module_id: "terminal-off-worker".to_string(),
11623 program: fake_aft_stub_path(),
11624 args: Vec::new(),
11625 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11626 reserved: false,
11627 reserved_prefixes: Vec::new(),
11628 protocol: ModuleProtocol::Subc,
11629 overlap: Default::default(),
11630 })
11631 .unwrap();
11632 let deadline = Instant::now() + Duration::from_secs(5);
11633 while module.terminal_history().entries.len() != 2 {
11634 assert!(Instant::now() < deadline, "module did not record two exits");
11635 sleep(Duration::from_millis(10)).await;
11636 }
11637
11638 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
11639 let handler =
11640 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
11641 let frame = Frame::build(
11642 FrameType::Request,
11643 control_flags(),
11644 0,
11645 0,
11646 1,
11647 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
11648 module_id: "terminal-off-worker".to_string(),
11649 })
11650 .unwrap(),
11651 )
11652 .unwrap();
11653 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11654 let spawned_at = std::time::Instant::now();
11655 let read = tokio::spawn({
11656 let handler = Arc::clone(&handler);
11657 async move { handler.handle_control_frame(&ctx, frame).await }
11658 });
11659 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
11662 .await
11663 .unwrap()
11664 .expect("the history read reached its pause");
11665 let elapsed = spawned_at.elapsed();
11666 assert!(
11667 elapsed < Duration::from_secs(2) && !read.is_finished(),
11668 "this task could not run while the history read was paused \
11669 (resumed after {elapsed:?}, read finished: {})",
11670 read.is_finished()
11671 );
11672
11673 drop(release);
11674 let responses = read.await.unwrap().unwrap();
11675 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11676 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
11677 panic!("expected supervisor.terminals response");
11678 };
11679 assert_eq!(terminals.entries.len(), 2);
11680 assert_eq!(terminals.journal_skipped_lines, 0);
11681 assert_eq!(terminals.journal_read_errors, 0);
11682 }
11683
11684 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11685 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
11686 let registry = Arc::new(Registry::default());
11687 let supervisor_handle = SupervisorHandle::new();
11688 let supervisor =
11689 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11690 .with_handle(supervisor_handle.clone());
11691 let module = supervisor
11692 .spawn(ModuleSpec {
11693 script: None,
11694 carry_containment: false,
11695 module_id: "terminal-golden".to_string(),
11696 program: fake_aft_stub_path(),
11697 args: Vec::new(),
11698 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11699 reserved: false,
11700 reserved_prefixes: Vec::new(),
11701 protocol: ModuleProtocol::Subc,
11702 overlap: Default::default(),
11703 })
11704 .unwrap();
11705
11706 let deadline = Instant::now() + Duration::from_secs(5);
11707 while module.terminal_history().entries.len() != 2 {
11708 assert!(
11709 Instant::now() < deadline,
11710 "module did not retain two terminal exits: {:?}",
11711 module.terminal_history()
11712 );
11713 sleep(Duration::from_millis(10)).await;
11714 }
11715
11716 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11717 let request = ClientControlRequest::SupervisorTerminals {
11718 module_id: "terminal-golden".to_string(),
11719 };
11720 let frame = Frame::build(
11721 FrameType::Request,
11722 control_flags(),
11723 0,
11724 0,
11725 1,
11726 serde_json::to_vec(&request).unwrap(),
11727 )
11728 .unwrap();
11729 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11730 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11731 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11732 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
11733 panic!("expected supervisor.terminals response");
11734 };
11735 assert_eq!(terminals.entries.len(), 2);
11736 assert_eq!(terminals.dropped, 0);
11737
11738 let mut rendered = serde_json::to_value(response).unwrap();
11739 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
11742 for (index, entry) in rendered["entries"]
11743 .as_array_mut()
11744 .expect("terminal response entries array")
11745 .iter_mut()
11746 .enumerate()
11747 {
11748 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
11749 }
11750
11751 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11752 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
11753 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
11754 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11755 std::fs::write(&golden_path, &serialized).unwrap();
11756 }
11757 let expected: Value =
11758 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
11759 assert_eq!(rendered, expected);
11760 }
11761
11762 #[test]
11763 fn hello_registers_manifest_and_returns_ack() {
11764 let registry = Arc::new(Registry::default());
11765 let handler = ControlHandler::new(Arc::clone(®istry));
11766 let conn = ConnectionId::new(1);
11767
11768 let responses = handler
11769 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
11770 .unwrap();
11771
11772 assert_eq!(responses.len(), 1);
11773 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11774 assert_eq!(responses[0].header.channel, 0);
11775 assert_eq!(responses[0].header.corr, 7);
11776 let ack = parse_ack(&responses[0]);
11777 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11778 assert!(ack
11779 .subc_capabilities
11780 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
11781 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
11782 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
11783 assert!(ack
11784 .subc_ops
11785 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
11786 assert!(ack
11787 .subc_ops
11788 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
11789 assert!(ack
11790 .subc_ops
11791 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
11792
11793 let registration = registry.get_module("aft").unwrap().unwrap();
11794 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
11795 assert_eq!(registration.state, ChannelState::Active);
11796 assert_eq!(registration.connection_id, conn);
11797 assert_eq!(registration.control_ops, module_baseline_control_ops());
11798 assert!(registration.manifest.display_name.is_none());
11799 }
11800
11801 #[test]
11802 fn hello_display_name_refuses_invalid_manifest_by_field() {
11803 let registry = Arc::new(Registry::default());
11804 let handler = ControlHandler::new(Arc::clone(®istry));
11805 for (index, invalid) in [
11806 json!(""),
11807 json!("a".repeat(41)),
11808 json!(" Routing"),
11809 json!("a\nb"),
11810 json!("a\u{202e}b"),
11811 json!(7),
11812 ]
11813 .into_iter()
11814 .enumerate()
11815 {
11816 let module_id = format!("bad-display-name-{index}");
11817 let mut body = serde_json::to_value(ModuleHelloBody {
11818 manifest: manifest(&module_id, PROTOCOL_VERSION),
11819 protocol_ver: PROTOCOL_VERSION,
11820 control_ops: None,
11821 launch_nonce: None,
11822 })
11823 .unwrap();
11824 body["manifest"]["display_name"] = invalid;
11825 let hello = Frame::build(
11826 FrameType::Hello,
11827 control_flags(),
11828 0,
11829 0,
11830 7,
11831 serde_json::to_vec(&body).unwrap(),
11832 )
11833 .unwrap();
11834 let response = handler
11835 .handle_control(ConnectionId::new(100 + index as u64), hello)
11836 .unwrap();
11837 assert_eq!(response[0].header.ty, FrameType::Error);
11838 let error = parse_error(&response[0]);
11839 assert_eq!(error["code"], "invalid_manifest");
11840 assert!(error["message"].as_str().unwrap().contains("display_name"));
11841 assert!(registry.get_module(&module_id).unwrap().is_none());
11842 }
11843 }
11844
11845 #[test]
11846 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
11847 let invalid_identifiers = [
11848 ("case_change", "credentials-Provider/v1"),
11849 ("leading_zero", "credentials-provider/v01"),
11850 ("trailing_hyphen", "credentials-provider-/v1"),
11851 ("consecutive_hyphens", "credentials--provider/v1"),
11852 ("uppercase", "Credentials-provider/v1"),
11853 ("missing_v", "credentials-provider/1"),
11854 ("whitespace", "credentials provider/v1"),
11855 ("zero_version", "credentials-provider/v0"),
11856 ("out_of_range_version", "credentials-provider/v4294967296"),
11857 (
11858 "overlength_name",
11859 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
11860 ),
11861 ];
11862 let mut cases = invalid_identifiers
11863 .into_iter()
11864 .map(|(name, identifier)| {
11865 (
11866 format!("identifier_{name}"),
11867 "capabilities.provides[0]".to_string(),
11868 identifier.to_string(),
11869 json!({ "provides": [identifier] }),
11870 None,
11871 )
11872 })
11873 .collect::<Vec<_>>();
11874 cases.extend([
11875 (
11876 "unknown_need".to_string(),
11877 "capabilities.requires[0].need".to_string(),
11878 "deferred".to_string(),
11879 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
11880 None,
11881 ),
11882 (
11883 "duplicate_provides".to_string(),
11884 "capabilities.provides[1]".to_string(),
11885 "credentials-provider/v1".to_string(),
11886 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
11887 None,
11888 ),
11889 (
11890 "duplicate_must_never_reach".to_string(),
11891 "capabilities.must_never_reach[1]".to_string(),
11892 "credentials-provider/v1".to_string(),
11893 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
11894 None,
11895 ),
11896 (
11897 "duplicate_requires_same_need".to_string(),
11898 "capabilities.requires[1]".to_string(),
11899 "credentials-provider/v1".to_string(),
11900 json!({ "requires": [
11901 { "capability": "credentials-provider/v1", "need": "required" },
11902 { "capability": "credentials-provider/v1", "need": "required" }
11903 ] }),
11904 None,
11905 ),
11906 (
11907 "duplicate_requires_conflicting_need".to_string(),
11908 "capabilities.requires[1]".to_string(),
11909 "credentials-provider/v1".to_string(),
11910 json!({ "requires": [
11911 { "capability": "credentials-provider/v1", "need": "required" },
11912 { "capability": "credentials-provider/v1", "need": "optional" }
11913 ] }),
11914 None,
11915 ),
11916 (
11917 "capabilities_root_pointer".to_string(),
11918 "runtime_computed[0]".to_string(),
11919 "/capabilities".to_string(),
11920 json!({}),
11921 Some(json!(["/capabilities"])),
11922 ),
11923 (
11924 "capabilities_descendant_pointer".to_string(),
11925 "runtime_computed[0]".to_string(),
11926 "/capabilities/provides".to_string(),
11927 json!({}),
11928 Some(json!(["/capabilities/provides"])),
11929 ),
11930 (
11931 "malformed_pointer_without_leading_slash".to_string(),
11932 "runtime_computed[0]".to_string(),
11933 "capabilities".to_string(),
11934 json!({}),
11935 Some(json!(["capabilities"])),
11936 ),
11937 (
11938 "malformed_pointer_escape".to_string(),
11939 "runtime_computed[0]".to_string(),
11940 "/roles/~2/tools".to_string(),
11941 json!({}),
11942 Some(json!(["/roles/~2/tools"])),
11943 ),
11944 (
11945 "unknown_capabilities_field".to_string(),
11946 "capabilities.future".to_string(),
11947 "<array>".to_string(),
11948 json!({ "future": [] }),
11949 None,
11950 ),
11951 ]);
11952
11953 for (index, (name, field, value, capabilities, runtime_computed)) in
11954 cases.into_iter().enumerate()
11955 {
11956 let registry = Arc::new(Registry::default());
11957 let handler = ControlHandler::new(Arc::clone(®istry));
11958 let response = handler
11959 .handle_control(
11960 ConnectionId::new((index + 1) as u64),
11961 capability_grammar_hello_frame(
11962 capabilities,
11963 runtime_computed,
11964 index as u64 + 1,
11965 ),
11966 )
11967 .expect("invalid HELLO returns a refusal");
11968
11969 assert_eq!(response.len(), 1, "{name} must emit one refusal");
11970 let error = parse_error(&response[0]);
11971 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
11972 let message = error["message"]
11973 .as_str()
11974 .expect("error message is a string");
11975 assert!(
11976 message.contains(&field),
11977 "{name}: field missing from {message}"
11978 );
11979 assert!(
11980 message.contains(&value),
11981 "{name}: value missing from {message}"
11982 );
11983 assert_eq!(
11984 registry
11985 .active_registration_count()
11986 .expect("registry reads"),
11987 0,
11988 "{name}: refused HELLO must not create a catalog entry"
11989 );
11990 }
11991 }
11992
11993 #[test]
11994 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
11995 let registry = Arc::new(Registry::default());
11996 let handler = ControlHandler::new(Arc::clone(®istry));
11997 let capabilities = json!({
11998 "provides": ["credentials-provider/v1"],
11999 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
12000 "must_never_reach": ["federation-transport/v1"]
12001 });
12002 let response = handler
12003 .handle_control(
12004 ConnectionId::new(99),
12005 capability_grammar_hello_frame(
12006 capabilities.clone(),
12007 Some(json!(["/roles/0/tools"])),
12008 99,
12009 ),
12010 )
12011 .expect("valid HELLO registers");
12012 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12013
12014 let request = Frame::build(
12015 FrameType::Request,
12016 control_flags(),
12017 0,
12018 0,
12019 100,
12020 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12021 .expect("catalog request serializes"),
12022 )
12023 .expect("catalog request frame builds");
12024 let response = handler
12025 .handle_catalog_list(request, None)
12026 .expect("catalog list succeeds");
12027 let ClientControlResponse::CatalogList { modules, .. } =
12028 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
12029 else {
12030 panic!("catalog request must return catalog.list");
12031 };
12032 assert_eq!(modules.len(), 1);
12033 assert_eq!(
12034 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
12035 capabilities
12036 );
12037 }
12038
12039 #[test]
12040 fn catalog_list_mirrors_management_operation_description() {
12041 let registry = Arc::new(Registry::default());
12042 let handler = ControlHandler::new(Arc::clone(®istry));
12043 let description = "List managed records and return their identifiers and metadata.";
12044 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
12045 manifest.provides = vec![ProviderRole::ManagementSurface {
12046 operations: vec![ManagementOperation {
12047 name: "records.list".to_string(),
12048 kind: ManagementOperationKind::Query,
12049 description: Some(description.to_string()),
12050 }],
12051 config_schema: json!({"type": "object"}),
12052 observability: vec![ObservabilitySurface {
12053 name: "records.stats".to_string(),
12054 kind: ObservabilityKind::Snapshot,
12055 }],
12056 identity_scope: vec![IdentityScope::Project],
12057 concurrency: Concurrency::ModuleManaged,
12058 }];
12059 registry
12060 .register_with_control_ops(
12061 manifest,
12062 PROTOCOL_VERSION,
12063 ConnectionId::new(99),
12064 Vec::new(),
12065 )
12066 .expect("described management manifest registers");
12067
12068 let request = Frame::build(
12069 FrameType::Request,
12070 control_flags(),
12071 0,
12072 0,
12073 100,
12074 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12075 .expect("catalog request serializes"),
12076 )
12077 .expect("catalog request frame builds");
12078 let response = handler
12079 .handle_catalog_list(request, None)
12080 .expect("catalog list succeeds");
12081 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
12082 assert_eq!(
12083 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
12084 "catalog.list must preserve the declared operation description verbatim"
12085 );
12086 }
12087
12088 #[test]
12089 fn catalog_list_mirrors_registered_operation_delivery() {
12090 let registry = Arc::new(Registry::default());
12091 let handler = ControlHandler::new(Arc::clone(®istry));
12092 let delivery =
12093 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
12094 let mut body = serde_json::to_value(ModuleHelloBody {
12095 manifest: manifest("watch-management", PROTOCOL_VERSION),
12096 protocol_ver: PROTOCOL_VERSION,
12097 control_ops: None,
12098 launch_nonce: None,
12099 })
12100 .unwrap();
12101 body["manifest"]["provides"] = json!([{
12102 "role": "management_surface",
12103 "operations": [
12104 {"name": "status", "kind": "query"},
12105 {"name": "future", "kind": "query"},
12106 {"name": "reset", "kind": "mutate"}
12107 ],
12108 "config_schema": {}, "observability": [], "identity_scope": []
12109 }]);
12110 body["manifest"]["operation_delivery"] = delivery.clone();
12111 let hello = Frame::build(
12112 FrameType::Hello,
12113 control_flags(),
12114 0,
12115 0,
12116 99,
12117 serde_json::to_vec(&body).unwrap(),
12118 )
12119 .unwrap();
12120 let response = handler
12121 .handle_control(ConnectionId::new(99), hello)
12122 .unwrap();
12123 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12124
12125 let (_, registrations) = registry.list_modules().unwrap();
12126 let registered = ®istrations[0].manifest;
12127 assert!(registered.is_watch_operation("status"));
12128 assert!(!registered.is_watch_operation("future"));
12129 assert!(!registered.is_watch_operation("reset"));
12130 assert!(!registered.is_watch_operation("missing"));
12131
12132 let request = Frame::build(
12133 FrameType::Request,
12134 control_flags(),
12135 0,
12136 0,
12137 100,
12138 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12139 )
12140 .unwrap();
12141 let response = handler.handle_catalog_list(request, None).unwrap();
12142 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
12143 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
12144 let ClientControlResponse::CatalogList { modules, .. } =
12145 serde_json::from_slice(&response[0].body).unwrap()
12146 else {
12147 panic!("expected catalog.list");
12148 };
12149 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
12150 }
12151
12152 #[test]
12153 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
12154 let registry = Arc::new(Registry::default());
12155 let supervisor = SupervisorHandle::new();
12156 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12157 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12158 let descriptor = json!({
12159 "schema": "ckios.richui.manifest/1",
12160 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12161 "bytes": 4096,
12162 "chunks": 4
12163 });
12164 let connection = ConnectionId::new(201);
12165 let hello = handler
12166 .handle_control(
12167 connection,
12168 hello_frame_with_rich_ui(
12169 "vault",
12170 PROTOCOL_VERSION,
12171 201,
12172 Some("the-real-nonce"),
12173 descriptor.clone(),
12174 ),
12175 )
12176 .unwrap();
12177 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12178
12179 let request =
12180 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
12181 let provides = registry
12182 .get_module_by_connection(connection)
12183 .unwrap()
12184 .unwrap()
12185 .manifest
12186 .provides;
12187 handler
12188 .handle_catalog_update(connection, request, provides, None, Some(false))
12189 .unwrap();
12190
12191 let request = Frame::build(
12192 FrameType::Request,
12193 control_flags(),
12194 0,
12195 0,
12196 203,
12197 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12198 )
12199 .unwrap();
12200 let response = handler.handle_catalog_list(request, None).unwrap();
12201 let ClientControlResponse::CatalogList { modules, .. } =
12202 serde_json::from_slice(&response[0].body).unwrap()
12203 else {
12204 panic!("expected catalog.list");
12205 };
12206 assert_eq!(
12207 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
12208 descriptor
12209 );
12210 assert!(modules[0].nonce_verified);
12211 }
12212
12213 fn hello_with_role_declarations(declarations: Value) -> Frame {
12214 let mut body = serde_json::to_value(ModuleHelloBody {
12215 manifest: manifest("role-test", PROTOCOL_VERSION),
12216 protocol_ver: PROTOCOL_VERSION,
12217 control_ops: None,
12218 launch_nonce: None,
12219 })
12220 .unwrap();
12221 body["manifest"]["role_declarations"] = declarations;
12222 Frame::build(
12223 FrameType::Hello,
12224 control_flags(),
12225 0,
12226 0,
12227 301,
12228 serde_json::to_vec(&body).unwrap(),
12229 )
12230 .unwrap()
12231 }
12232
12233 fn update_role_declarations_frame(declarations: Value) -> Frame {
12234 Frame::build(
12235 FrameType::Request,
12236 control_flags(),
12237 0,
12238 0,
12239 303,
12240 serde_json::to_vec(&json!({
12241 "op": "catalog.update_role_declarations",
12242 "role_declarations": declarations,
12243 }))
12244 .unwrap(),
12245 )
12246 .unwrap()
12247 }
12248
12249 #[tokio::test]
12250 async fn role_declarations_update_replaces_then_clears_catalog() {
12251 let registry = Arc::new(Registry::default());
12252 let handler = ControlHandler::new(Arc::clone(®istry));
12253 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12254 let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
12255 hello_via_sink(
12256 &handler,
12257 &ctx,
12258 &mut rx,
12259 hello_with_role_declarations(initial.clone()),
12260 )
12261 .await;
12262 assert_eq!(
12263 role_declarations_catalog(&handler).0["role_declarations"],
12264 initial
12265 );
12266 let before = registry.get_module("role-test").unwrap().unwrap();
12267 for declarations in [
12268 json!({"media-provider/v1": {"engines": ["new"]}}),
12269 json!({}),
12270 ] {
12271 let response = handler
12272 .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
12273 .await
12274 .unwrap();
12275 assert_eq!(response[0].header.ty, FrameType::Response);
12276 assert_eq!(
12277 serde_json::from_slice::<Value>(&response[0].body).unwrap(),
12278 json!({"op": "catalog.update_role_declarations"})
12279 );
12280 let (raw, entry) = role_declarations_catalog(&handler);
12281 assert_eq!(
12282 serde_json::to_value(entry.role_declarations).unwrap(),
12283 declarations
12284 );
12285 if declarations.as_object().unwrap().is_empty() {
12286 assert!(raw.get("role_declarations").is_none());
12287 } else {
12288 assert_eq!(raw["role_declarations"], declarations);
12289 }
12290 let after = registry.get_module("role-test").unwrap().unwrap();
12291 let mut expected = before.manifest.clone();
12292 expected.role_declarations = serde_json::from_value(declarations).unwrap();
12293 assert_eq!(after.manifest, expected);
12294 assert_eq!(after.ready, before.ready);
12295 }
12296 }
12297
12298 #[tokio::test]
12299 async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
12300 let registry = Arc::new(Registry::default());
12301 let handler = ControlHandler::new(registry);
12302 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12303 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12304 hello_via_sink(
12305 &handler,
12306 &ctx,
12307 &mut rx,
12308 hello_with_role_declarations(initial.clone()),
12309 )
12310 .await;
12311 for (invalid, reason) in [
12312 (json!(null), "role_declarations must be an object"),
12313 (json!([]), "role_declarations must be an object"),
12314 (
12315 json!({"bad key": {}}),
12316 "role_declarations key must use capability-name grammar",
12317 ),
12318 (
12319 json!({"media-provider/v1": []}),
12320 "role_declarations values must be objects",
12321 ),
12322 (
12323 json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
12324 "role_declarations exceeds 16384 compact JSON bytes",
12325 ),
12326 ] {
12327 let response = handler
12328 .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
12329 .await
12330 .unwrap();
12331 let error = parse_error(&response[0]);
12332 assert_eq!(error["code"], "invalid_role_declarations");
12333 assert_eq!(error["message"], reason);
12334 assert_eq!(
12335 role_declarations_catalog(&handler).0["role_declarations"],
12336 initial
12337 );
12338 }
12339 }
12340
12341 #[tokio::test]
12342 async fn role_declarations_update_is_owned_by_the_registered_connection() {
12343 let registry = Arc::new(Registry::default());
12344 let handler = ControlHandler::new(Arc::clone(®istry));
12345 let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
12346 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12347 hello_via_sink(
12348 &handler,
12349 &owner,
12350 &mut owner_rx,
12351 hello_with_role_declarations(initial.clone()),
12352 )
12353 .await;
12354 let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
12355 let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
12356 let response = handler
12357 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12358 .await
12359 .unwrap();
12360 assert_eq!(parse_error(&response[0])["code"], "not_registered");
12361 hello_via_sink(
12362 &handler,
12363 &other,
12364 &mut other_rx,
12365 hello_frame("other", PROTOCOL_VERSION, 304),
12366 )
12367 .await;
12368 let response = handler
12369 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12370 .await
12371 .unwrap();
12372 assert_eq!(response[0].header.ty, FrameType::Response);
12373 assert_eq!(
12374 serde_json::to_value(
12375 registry
12376 .get_module("role-test")
12377 .unwrap()
12378 .unwrap()
12379 .manifest
12380 .role_declarations
12381 )
12382 .unwrap(),
12383 initial
12384 );
12385 assert_eq!(
12386 serde_json::to_value(
12387 registry
12388 .get_module("other")
12389 .unwrap()
12390 .unwrap()
12391 .manifest
12392 .role_declarations
12393 )
12394 .unwrap(),
12395 replacement
12396 );
12397 }
12398
12399 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
12400 let request = Frame::build(
12401 FrameType::Request,
12402 control_flags(),
12403 0,
12404 0,
12405 302,
12406 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12407 )
12408 .unwrap();
12409 let response = handler.handle_catalog_list(request, None).unwrap();
12410 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
12411 let ClientControlResponse::CatalogList { mut modules, .. } =
12412 serde_json::from_slice(&response[0].body).unwrap()
12413 else {
12414 panic!("expected catalog.list");
12415 };
12416 (raw["modules"][0].clone(), modules.remove(0))
12417 }
12418
12419 #[test]
12420 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
12421 let registry = Arc::new(Registry::default());
12422 let handler = ControlHandler::new(Arc::clone(®istry));
12423 let declarations = json!({
12424 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
12425 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
12426 });
12427 let connection = ConnectionId::new(301);
12428 let response = handler
12429 .handle_control(
12430 connection,
12431 hello_with_role_declarations(declarations.clone()),
12432 )
12433 .unwrap();
12434 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12435 let registered = registry
12436 .get_module_by_connection(connection)
12437 .unwrap()
12438 .unwrap()
12439 .manifest;
12440 assert_eq!(
12444 serde_json::to_value(®istered.role_declarations).unwrap(),
12445 declarations
12446 );
12447 let (raw, entry) = role_declarations_catalog(&handler);
12448 assert_eq!(raw["role_declarations"], declarations);
12449 assert_eq!(entry.role_declarations, registered.role_declarations);
12450
12451 let request =
12452 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
12453 handler
12454 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
12455 .unwrap();
12456 let (raw, entry) = role_declarations_catalog(&handler);
12457 assert_eq!(raw["role_declarations"], declarations);
12458 assert_eq!(entry.role_declarations, registered.role_declarations);
12459 }
12460
12461 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
12462 let registry = Arc::new(Registry::default());
12463 let handler = ControlHandler::new(Arc::clone(®istry));
12464 let capture = EventCapture::default();
12465 let _subscriber =
12466 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12467 let response = handler
12468 .handle_control(
12469 ConnectionId::new(301),
12470 hello_with_role_declarations(declarations),
12471 )
12472 .unwrap();
12473 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12474 assert!(registry
12475 .get_module_by_connection(ConnectionId::new(301))
12476 .unwrap()
12477 .unwrap()
12478 .manifest
12479 .role_declarations
12480 .is_empty());
12481 let (raw, entry) = role_declarations_catalog(&handler);
12482 assert!(raw.get("role_declarations").is_none());
12483 assert!(entry.role_declarations.is_empty());
12484 let warnings: Vec<_> = capture
12485 .events()
12486 .into_iter()
12487 .filter(|event| {
12488 event.fields.get("message").map(String::as_str)
12489 == Some("dropping whole role_declarations map from HELLO")
12490 })
12491 .collect();
12492 assert_eq!(warnings.len(), 1);
12493 assert_eq!(warnings[0].level, tracing::Level::WARN);
12494 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
12495 assert_eq!(warnings[0].fields["reason"], reason);
12496 }
12497
12498 #[test]
12499 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
12500 for value in [
12501 json!(null),
12502 json!(false),
12503 json!(0),
12504 json!("opaque"),
12505 json!([]),
12506 ] {
12507 assert_hello_drops_role_declarations_with_warning(
12508 json!({"routing/v1": {}, "broken/v1": value}),
12509 "role_declarations values must be objects",
12510 );
12511 }
12512 assert_hello_drops_role_declarations_with_warning(
12513 json!([]),
12514 "role_declarations must be an object",
12515 );
12516 }
12517
12518 #[test]
12519 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
12520 assert_hello_drops_role_declarations_with_warning(
12521 json!({"routing/v1": {}, "bad key": {}}),
12522 "role_declarations key must use capability-name grammar",
12523 );
12524 }
12525
12526 #[test]
12527 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
12528 assert_hello_drops_role_declarations_with_warning(
12529 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
12530 "role_declarations exceeds 16384 compact JSON bytes",
12531 );
12532 }
12533
12534 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
12535 let mut body = serde_json::to_value(ModuleHelloBody {
12536 manifest: manifest("flow-ops", PROTOCOL_VERSION),
12537 protocol_ver: PROTOCOL_VERSION,
12538 control_ops: None,
12539 launch_nonce: None,
12540 })
12541 .unwrap();
12542 body["manifest"]["provides"] = json!([{
12543 "role": "management_surface",
12544 "operations": [
12545 {"name": "diagnostics.findings", "kind": "query"},
12546 {"name": "reset", "kind": "mutate"}
12547 ],
12548 "config_schema": {}, "observability": [], "identity_scope": []
12549 }]);
12550 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
12551 if let Some(flow_operations) = flow_operations {
12552 body["manifest"]["flow_operations"] = flow_operations;
12553 }
12554 Frame::build(
12555 FrameType::Hello,
12556 control_flags(),
12557 0,
12558 0,
12559 401,
12560 serde_json::to_vec(&body).unwrap(),
12561 )
12562 .unwrap()
12563 }
12564
12565 #[test]
12566 fn catalog_list_mirrors_flow_operations_unchanged() {
12567 for (capability, flow_operations) in [
12570 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
12571 ("call-modes/v1", json!(["diagnostics.findings"])),
12572 ("flow-scopes/v1", json!([])),
12573 ] {
12574 let registry = Arc::new(Registry::default());
12575 let handler = ControlHandler::new(Arc::clone(®istry));
12576 let connection = ConnectionId::new(401);
12577 let response = handler
12578 .handle_control(
12579 connection,
12580 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
12581 )
12582 .unwrap();
12583 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12584 let registered = registry
12585 .get_module_by_connection(connection)
12586 .unwrap()
12587 .unwrap()
12588 .manifest;
12589 assert_eq!(
12590 serde_json::to_value(®istered.flow_operations).unwrap(),
12591 flow_operations
12592 );
12593 let (raw, entry) = role_declarations_catalog(&handler);
12594 assert_eq!(raw["flow_operations"], flow_operations);
12595 assert_eq!(entry.flow_operations, registered.flow_operations);
12596 }
12597
12598 let registry = Arc::new(Registry::default());
12599 let handler = ControlHandler::new(Arc::clone(®istry));
12600 let response = handler
12601 .handle_control(
12602 ConnectionId::new(401),
12603 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
12604 )
12605 .unwrap();
12606 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12607 let (raw, entry) = role_declarations_catalog(&handler);
12608 assert!(raw.get("flow_operations").is_none());
12609 assert_eq!(entry.flow_operations, None);
12610 }
12611
12612 #[test]
12613 fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
12614 let oversized: serde_json::Map<String, Value> = (0..257)
12615 .map(|n| (format!("tool:t{n}"), json!(true)))
12616 .collect();
12617 for (capabilities, map, valid) in [
12618 (
12619 json!(["scope-federated/v1"]),
12620 json!({"tool:read":true,"management:reset":true}),
12621 true,
12622 ),
12623 (json!(["scope-federated/v1"]), json!({}), true),
12624 (json!([]), json!({"tool:read":true}), false),
12625 (
12626 json!(["scope-federated/v1"]),
12627 json!({"tool:valid":true,"tool:bad":false}),
12628 false,
12629 ),
12630 (
12631 json!(["scope-federated/v1"]),
12632 json!({"tool:valid":true,"management:missing":true}),
12633 false,
12634 ),
12635 (
12636 json!(["scope-federated/v1"]),
12637 Value::Object(oversized),
12638 false,
12639 ),
12640 ] {
12641 let capture = EventCapture::default();
12642 let _subscriber = tracing::subscriber::set_default(
12643 tracing_subscriber::registry().with(capture.clone()),
12644 );
12645 let registry = Arc::new(Registry::default());
12646 let handler = ControlHandler::new(Arc::clone(®istry));
12647 let hello = hello_with_flow_operations(capabilities, None);
12648 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12649 body["manifest"]["operation_deferral"] = map.clone();
12650 let hello = Frame::build(
12651 FrameType::Hello,
12652 control_flags(),
12653 0,
12654 0,
12655 401,
12656 serde_json::to_vec(&body).unwrap(),
12657 )
12658 .unwrap();
12659 let response = handler
12660 .handle_control(ConnectionId::new(401), hello)
12661 .unwrap();
12662 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12663 let (raw, entry) = role_declarations_catalog(&handler);
12664 if valid {
12665 assert_eq!(raw["operation_deferral"], map);
12666 assert_eq!(
12667 serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
12668 map
12669 );
12670 } else {
12671 assert!(entry.operation_deferral.is_none());
12672 assert!(raw.get("operation_deferral").is_none());
12673 assert_eq!(
12674 capture
12675 .events()
12676 .iter()
12677 .filter(|event| event.fields.get("message").map(String::as_str)
12678 == Some("dropping operation_deferral from HELLO"))
12679 .count(),
12680 1
12681 );
12682 }
12683 }
12684 }
12685
12686 fn assert_hello_drops_flow_operations_with_warning(
12687 capabilities: Value,
12688 flow_operations: Value,
12689 reason: &str,
12690 ) {
12691 let registry = Arc::new(Registry::default());
12692 let handler = ControlHandler::new(Arc::clone(®istry));
12693 let capture = EventCapture::default();
12694 let _subscriber =
12695 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12696 let response = handler
12697 .handle_control(
12698 ConnectionId::new(401),
12699 hello_with_flow_operations(capabilities, Some(flow_operations)),
12700 )
12701 .unwrap();
12702 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12703 let registered = registry
12704 .get_module_by_connection(ConnectionId::new(401))
12705 .unwrap()
12706 .expect("the module still registers")
12707 .manifest;
12708 assert_eq!(registered.module_id, "flow-ops");
12709 assert_eq!(registered.flow_operations, None);
12710 let (raw, entry) = role_declarations_catalog(&handler);
12711 assert_eq!(raw["module_id"], "flow-ops");
12712 assert!(raw.get("flow_operations").is_none());
12713 assert_eq!(entry.flow_operations, None);
12714 let warnings: Vec<_> = capture
12715 .events()
12716 .into_iter()
12717 .filter(|event| {
12718 event.fields.get("message").map(String::as_str)
12719 == Some("dropping flow_operations from HELLO")
12720 })
12721 .collect();
12722 assert_eq!(warnings.len(), 1);
12723 assert_eq!(warnings[0].level, tracing::Level::WARN);
12724 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
12725 assert_eq!(warnings[0].fields["reason"], reason);
12726 }
12727
12728 #[test]
12729 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
12730 assert_hello_drops_flow_operations_with_warning(
12731 json!(["flow-scopes/v1"]),
12732 json!(["diagnostics.findings", "missing"]),
12733 "flow_operations names 'missing', which is not a declared management operation",
12734 );
12735 }
12736
12737 #[test]
12738 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
12739 assert_hello_drops_flow_operations_with_warning(
12740 json!([]),
12741 json!(["diagnostics.findings"]),
12742 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12743 );
12744 assert_hello_drops_flow_operations_with_warning(
12746 json!(["agent-run-scopes/v1"]),
12747 json!([]),
12748 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12749 );
12750 }
12751
12752 #[test]
12753 fn hello_malformed_flow_operations_is_dropped_with_warning() {
12754 assert_hello_drops_flow_operations_with_warning(
12755 json!(["flow-scopes/v1"]),
12756 json!(["reset", "reset"]),
12757 "flow_operations names 'reset' more than once",
12758 );
12759 assert_hello_drops_flow_operations_with_warning(
12760 json!(["flow-scopes/v1"]),
12761 json!(vec!["reset"; 257]),
12762 "flow_operations lists 257 names, more than 256",
12763 );
12764 assert_hello_drops_flow_operations_with_warning(
12765 json!(["flow-scopes/v1"]),
12766 json!("reset"),
12767 "flow_operations must be an array of operation names",
12768 );
12769 }
12770
12771 #[test]
12772 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
12773 let registry = Arc::new(Registry::default());
12774 let handler = ControlHandler::new(registry.clone());
12775 let descriptor = json!({
12776 "schema": "ckios.richui.manifest/1",
12777 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12778 "bytes": 128,
12779 "chunks": 2
12780 });
12781 let malformed = [
12782 ("schema", json!("")),
12783 ("schema", json!("a".repeat(65))),
12784 ("schema", json!("Bad schema")),
12785 (
12786 "manifest_digest",
12787 json!(format!("sha256:{}", "A".repeat(64))),
12788 ),
12789 ("manifest_digest", json!("sha256:abc")),
12790 ("bytes", json!(0)),
12791 ("bytes", json!(16_777_217)),
12792 ("bytes", json!("128")),
12793 ("chunks", json!(0)),
12794 ("chunks", json!(1025)),
12795 ("chunks", json!("2")),
12796 ];
12797 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
12798 let mut invalid = descriptor.clone();
12799 invalid[field] = invalid_value;
12800 let response = handler
12801 .handle_control(
12802 ConnectionId::new(210 + index as u64),
12803 hello_frame_with_rich_ui(
12804 &format!("bad-rich-ui-{index}"),
12805 PROTOCOL_VERSION,
12806 210 + index as u64,
12807 None,
12808 invalid,
12809 ),
12810 )
12811 .unwrap();
12812 assert_eq!(response[0].header.ty, FrameType::Error);
12813 let error = parse_error(&response[0]);
12814 assert_eq!(error["code"], "invalid_manifest");
12815 assert!(error["message"]
12816 .as_str()
12817 .unwrap()
12818 .contains(&format!("rich_ui.{field}")));
12819 assert!(registry
12820 .get_module(&format!("bad-rich-ui-{index}"))
12821 .unwrap()
12822 .is_none());
12823 }
12824 }
12825
12826 #[test]
12827 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
12828 let registry = Arc::new(Registry::default());
12829 let handler = ControlHandler::new(Arc::clone(®istry));
12830 let connection = ConnectionId::new(219);
12831 registry
12832 .register_with_control_ops(
12833 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
12834 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
12835 "bad schema",
12836 format!("sha256:{}", "a".repeat(64)),
12837 128,
12838 2,
12839 )))
12840 .build(),
12841 PROTOCOL_VERSION,
12842 connection,
12843 Vec::new(),
12844 )
12845 .unwrap();
12846 let request =
12847 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
12848 let response = handler
12849 .handle_catalog_update(connection, request, Vec::new(), None, None)
12850 .unwrap();
12851 assert_eq!(response[0].header.ty, FrameType::Error);
12852 let error = parse_error(&response[0]);
12853 assert_eq!(error["code"], "invalid_manifest");
12854 assert!(error["message"]
12855 .as_str()
12856 .unwrap()
12857 .contains("rich_ui.schema"));
12858 }
12859
12860 #[test]
12861 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
12862 let registry = Arc::new(Registry::default());
12863 let supervisor = SupervisorHandle::new();
12864 supervisor.open_swap("provider", "candidate-nonce".to_string());
12865 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12866 let connection = ConnectionId::new(218);
12867 let hello = handler
12868 .handle_control(
12869 connection,
12870 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
12871 )
12872 .unwrap();
12873 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12874 assert!(registry.list_modules().unwrap().1.is_empty());
12875
12876 registry.promote_candidate("provider").unwrap().unwrap();
12877 let request = Frame::build(
12878 FrameType::Request,
12879 control_flags(),
12880 0,
12881 0,
12882 219,
12883 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12884 )
12885 .unwrap();
12886 let response = handler.handle_catalog_list(request, None).unwrap();
12887 let ClientControlResponse::CatalogList { modules, .. } =
12888 serde_json::from_slice(&response[0].body).unwrap()
12889 else {
12890 panic!("expected catalog.list");
12891 };
12892 assert_eq!(modules.len(), 1);
12893 assert!(modules[0].nonce_verified);
12894 }
12895
12896 #[test]
12897 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
12898 for provenance_json in [
12899 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
12900 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
12901 ] {
12902 let registry = Arc::new(Registry::default());
12903 let handler = ControlHandler::new(registry);
12904 let hello = Frame::build(
12905 FrameType::Hello,
12906 control_flags(),
12907 0,
12908 0,
12909 220,
12910 serde_json::to_vec(&ModuleHelloBody {
12911 manifest: ModuleManifest::builder("built", "0.1.0")
12912 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
12913 .build(),
12914 protocol_ver: PROTOCOL_VERSION,
12915 control_ops: None,
12916 launch_nonce: None,
12917 })
12918 .unwrap(),
12919 )
12920 .unwrap();
12921 assert_eq!(
12922 handler
12923 .handle_control(ConnectionId::new(220), hello)
12924 .unwrap()[0]
12925 .header
12926 .ty,
12927 FrameType::HelloAck
12928 );
12929 let request = Frame::build(
12930 FrameType::Request,
12931 control_flags(),
12932 0,
12933 0,
12934 221,
12935 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12936 )
12937 .unwrap();
12938 let response = handler.handle_catalog_list(request, None).unwrap();
12939 let ClientControlResponse::CatalogList { modules, .. } =
12940 serde_json::from_slice(&response[0].body).unwrap()
12941 else {
12942 panic!("expected catalog.list");
12943 };
12944 assert_eq!(modules.len(), 1);
12945 assert_eq!(
12946 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
12947 format!(
12948 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
12949 )
12950 .as_bytes()
12951 );
12952 }
12953 }
12954
12955 #[test]
12956 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
12957 let registry = Arc::new(Registry::default());
12958 let handler = ControlHandler::new(Arc::clone(®istry));
12959 let hello = Frame::build(
12960 FrameType::Hello,
12961 control_flags(),
12962 0,
12963 0,
12964 220,
12965 serde_json::to_vec(&ModuleHelloBody {
12966 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
12967 protocol_ver: PROTOCOL_VERSION,
12968 control_ops: None,
12969 launch_nonce: Some("unverified".to_string()),
12970 })
12971 .unwrap(),
12972 )
12973 .unwrap();
12974 assert_eq!(
12975 handler
12976 .handle_control(ConnectionId::new(220), hello)
12977 .unwrap()[0]
12978 .header
12979 .ty,
12980 FrameType::HelloAck
12981 );
12982 let request = Frame::build(
12983 FrameType::Request,
12984 control_flags(),
12985 0,
12986 0,
12987 221,
12988 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12989 )
12990 .unwrap();
12991 let response = handler.handle_catalog_list(request, None).unwrap();
12992 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
12993 assert!(wire["modules"][0].get("provenance").is_none());
12994 let ClientControlResponse::CatalogList { modules, .. } =
12995 serde_json::from_slice(&response[0].body).unwrap()
12996 else {
12997 panic!("expected catalog.list");
12998 };
12999 assert_eq!(
13000 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13001 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
13002 );
13003 assert!(!modules[0].nonce_verified);
13004 assert!(modules[0].provenance.is_none());
13005 }
13006
13007 fn catalog_legacy_projection(mut entry: CatalogEntry) -> CatalogEntry {
13009 assert_eq!(entry.first_party, Some(false));
13010 entry.incarnation = None;
13011 entry.spawn = None;
13012 entry.first_party = None;
13013 entry.origin = None;
13014 entry.requested_declarations_digest = None;
13015 entry.approved_declarations_digest = None;
13016 entry
13017 }
13018
13019 #[test]
13020 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
13021 let registry = Arc::new(Registry::default());
13022 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13023 [("vault".to_string(), true), ("squatter".to_string(), true)],
13024 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13025 );
13026 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
13027 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13028 provides: vec!["credentials-provider/v1".to_string()],
13029 requires: Vec::new(),
13030 must_never_reach: Vec::new(),
13031 });
13032 let frame = Frame::build(
13033 FrameType::Hello,
13034 control_flags(),
13035 0,
13036 0,
13037 77,
13038 serde_json::to_vec(&ModuleHelloBody {
13039 manifest: squatter,
13040 protocol_ver: PROTOCOL_VERSION,
13041 control_ops: None,
13042 launch_nonce: None,
13043 })
13044 .expect("HELLO serializes"),
13045 )
13046 .expect("HELLO frame builds");
13047 let response = handler
13048 .handle_control(ConnectionId::new(77), frame)
13049 .expect("reserved claim receives a typed refusal");
13050 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13051 assert_eq!(
13052 registry
13053 .active_registration_count()
13054 .expect("registry reads"),
13055 0,
13056 "a reserved capability refusal must not leave a catalog entry"
13057 );
13058 }
13059
13060 #[test]
13061 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
13062 for settlement in ["timeout", "inconclusive", "drop"] {
13063 let breakers = RouteBindBreakers::default();
13064 let RouteBindAdmission::Admitted {
13065 guard: mut old,
13066 probe: false,
13067 } = breakers.admit("prov")
13068 else {
13069 panic!("ordinary relay admitted")
13070 };
13071 let RouteBindAdmission::Admitted {
13072 guard: mut opener, ..
13073 } = breakers.admit("prov")
13074 else {
13075 panic!("second relay admitted")
13076 };
13077 assert!(
13078 !opener
13079 .record_timeout(1, Duration::ZERO)
13080 .unwrap()
13081 .reopened_after_probe
13082 );
13083 let RouteBindAdmission::Admitted {
13084 guard: mut probe,
13085 probe: true,
13086 } = breakers.admit("prov")
13087 else {
13088 panic!("one cooldown probe admitted")
13089 };
13090 match settlement {
13091 "timeout" => assert!(
13092 !old.record_timeout(1, Duration::ZERO)
13093 .unwrap()
13094 .reopened_after_probe
13095 ),
13096 "inconclusive" => old.record_inconclusive(),
13097 "drop" => drop(old),
13098 _ => unreachable!(),
13099 }
13100 assert!(
13101 matches!(
13102 breakers.admit("prov"),
13103 RouteBindAdmission::Refused {
13104 probe_in_flight: true,
13105 ..
13106 }
13107 ),
13108 "{settlement} of a pre-open relay cannot release the real probe"
13109 );
13110 assert!(
13111 probe
13112 .record_timeout(1, Duration::ZERO)
13113 .unwrap()
13114 .reopened_after_probe
13115 );
13116 assert!(matches!(
13117 breakers.admit("prov"),
13118 RouteBindAdmission::Admitted { probe: true, .. }
13119 ));
13120 }
13121 let breakers = RouteBindBreakers::default();
13122 let admit = || match breakers.admit("prov") {
13123 RouteBindAdmission::Admitted { guard, .. } => guard,
13124 _ => panic!("relay admitted"),
13125 };
13126 admit().record_timeout(1, Duration::ZERO);
13127 let mut old_probe = admit();
13128 breakers.reset_for_new_module_connection("prov");
13129 admit().record_timeout(1, Duration::ZERO);
13130 let _new_probe = admit();
13131 old_probe.record_inconclusive();
13132 assert!(matches!(
13133 breakers.admit("prov"),
13134 RouteBindAdmission::Refused {
13135 probe_in_flight: true,
13136 ..
13137 }
13138 ));
13139 }
13140
13141 #[tokio::test]
13142 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
13143 for candidate in [false, true] {
13144 let registry = Arc::new(Registry::default());
13145 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13146 [("vault".to_string(), true), ("squatter".to_string(), true)],
13147 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13148 );
13149 let conn = ConnectionId::new(77);
13150 let (ctx, mut rx) = route_ctx(conn);
13151 let initial = capability_manifest("squatter", &[], &[]);
13152 if candidate {
13153 registry
13154 .register_candidate_with_control_ops(
13155 initial.clone(),
13156 PROTOCOL_VERSION,
13157 conn,
13158 module_baseline_control_ops(),
13159 )
13160 .unwrap();
13161 handler
13162 .forwarding
13163 .register_candidate_module_connection(
13164 conn,
13165 "squatter".to_string(),
13166 PROTOCOL_VERSION,
13167 manifest_concurrency(&initial),
13168 ctx.egress.clone(),
13169 )
13170 .unwrap();
13171 } else {
13172 hello_via_sink(
13173 &handler,
13174 &ctx,
13175 &mut rx,
13176 hello_frame_with_manifest(initial.clone(), 1),
13177 )
13178 .await;
13179 }
13180 let response = handler
13181 .handle_control_frame(
13182 &ctx,
13183 catalog_update_with_capabilities_frame(
13184 2,
13185 capability_manifest("squatter", &["credentials-provider/v1"], &[])
13186 .capabilities
13187 .unwrap(),
13188 ),
13189 )
13190 .await
13191 .unwrap();
13192 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13193 assert_eq!(
13194 registry
13195 .get_module_by_connection(conn)
13196 .unwrap()
13197 .unwrap()
13198 .manifest,
13199 initial
13200 );
13201 }
13202 }
13203
13204 #[test]
13205 #[cfg(not(feature = "dev-desktop-approval"))]
13206 fn server_describe_default_build_advertises_no_config_capability() {
13207 let assert_default = |handler: &ControlHandler| {
13208 let request = Frame::build(
13209 FrameType::Request,
13210 control_flags(),
13211 0,
13212 0,
13213 79,
13214 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13215 )
13216 .unwrap();
13217 let responses = handler.handle_server_describe(request).unwrap();
13218 assert_eq!(responses.len(), 1);
13219 let response: Value = serde_json::from_slice(&responses[0].body).unwrap();
13220 assert_eq!(response["op"], "server.describe");
13221 let capabilities = response["capabilities"]
13222 .as_array()
13223 .expect("capability array");
13224 assert!(
13225 !capabilities.is_empty(),
13226 "exercise an actual describe response, not an empty fixture"
13227 );
13228 assert!(
13229 capabilities
13230 .iter()
13231 .all(|capability| !capability.as_str().unwrap().starts_with("config.")),
13232 "a default build must not announce configuration authority: {capabilities:?}"
13233 );
13234 };
13235 assert_default(&ControlHandler::new(Arc::new(Registry::default())));
13236 #[cfg(unix)]
13237 {
13238 let (_rig, _owner, _supervisor, handler, _spec) =
13242 crate::config_gate::runtime::RuntimeOwner::unactivated_supervisor_fixture_for_test(
13243 );
13244 assert_default(&handler);
13245 }
13246 }
13247
13248 #[test]
13249 fn server_describe_surfaces_required_capability_verdict_fields() {
13250 let registry = Arc::new(Registry::default());
13251 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13252 [
13253 ("consumer".to_string(), true),
13254 ("provider".to_string(), false),
13255 ],
13256 BTreeMap::new(),
13257 );
13258 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13259 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13260 provides: Vec::new(),
13261 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13262 capability: "credentials-provider/v1".to_string(),
13263 need: subc_protocol::manifest::CapabilityNeed::Required,
13264 }],
13265 must_never_reach: Vec::new(),
13266 });
13267 let hello = Frame::build(
13268 FrameType::Hello,
13269 control_flags(),
13270 0,
13271 0,
13272 78,
13273 serde_json::to_vec(&ModuleHelloBody {
13274 manifest: consumer,
13275 protocol_ver: PROTOCOL_VERSION,
13276 control_ops: None,
13277 launch_nonce: None,
13278 })
13279 .expect("HELLO serializes"),
13280 )
13281 .expect("HELLO frame builds");
13282 handler
13283 .handle_control(ConnectionId::new(78), hello)
13284 .expect("consumer registers");
13285 let describe = Frame::build(
13286 FrameType::Request,
13287 control_flags(),
13288 0,
13289 0,
13290 79,
13291 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
13292 .expect("request serializes"),
13293 )
13294 .expect("describe frame builds");
13295 let response = handler
13296 .handle_server_describe(describe)
13297 .expect("server.describe succeeds");
13298 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
13299 let requirement = &rendered["capability_requirements"][0];
13300 assert_eq!(requirement["consumer"], "consumer");
13301 assert_eq!(requirement["verdict"], "never_provided");
13302 assert_eq!(requirement["episode_seq"], 1);
13303 assert_eq!(requirement["config_satisfiable"], false);
13304 assert_eq!(requirement["runtime_available"], false);
13305 assert!(requirement["detail"]
13306 .as_str()
13307 .expect("detail string")
13308 .contains("credentials-provider/v1"));
13309 }
13310
13311 fn deregister_required_provider(
13317 daemon_shutdown_begun: bool,
13318 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
13319 const CAPABILITY: &str = "project-identity/v1";
13320 let registry = Arc::new(Registry::default());
13321 let daemon_shutdown = DaemonShutdownFlag::default();
13322 let handler = ControlHandler::new(Arc::clone(®istry))
13323 .with_capability_config(
13324 [
13325 ("consumer".to_string(), true),
13326 ("provider".to_string(), true),
13327 ],
13328 BTreeMap::new(),
13329 )
13330 .with_daemon_shutdown_flag(daemon_shutdown.clone());
13331 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13332 consumer.capabilities = Some(CapabilityDeclarations {
13333 provides: Vec::new(),
13334 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13335 capability: CAPABILITY.to_string(),
13336 need: subc_protocol::manifest::CapabilityNeed::Required,
13337 }],
13338 must_never_reach: Vec::new(),
13339 });
13340 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
13341 let provider_conn = ConnectionId::new(182);
13342 for (conn, manifest, corr) in [
13343 (ConnectionId::new(181), consumer, 181),
13344 (provider_conn, provider, 182),
13345 ] {
13346 let hello = handler
13347 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
13348 .expect("module registers");
13349 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13350 }
13351 assert_eq!(
13352 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13353 Some(CapabilityVerdict::Provided),
13354 "the registered provider must satisfy the requirement first"
13355 );
13356 if daemon_shutdown_begun {
13357 daemon_shutdown.set_for_test();
13358 }
13359
13360 let capture = EventCapture::default();
13361 let _subscriber =
13362 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13363 handler
13364 .cleanup_connection(provider_conn)
13365 .expect("provider deregisters");
13366 let events = capture
13367 .events()
13368 .into_iter()
13369 .filter(|event| {
13370 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
13371 })
13372 .collect();
13373 (
13374 events,
13375 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13376 )
13377 }
13378
13379 #[test]
13380 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
13381 let (events, verdict) = deregister_required_provider(true);
13382 assert_eq!(
13383 verdict,
13384 Some(CapabilityVerdict::NeverProvided),
13385 "shutdown must not change the verdict route.open refusals read"
13386 );
13387 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13388 let event = &events[0];
13389 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
13390 assert_eq!(
13391 event.fields.get("cause").map(String::as_str),
13392 Some("\"daemon_shutdown\""),
13393 "{event:?}"
13394 );
13395 assert_eq!(
13396 event.fields.get("verdict").map(String::as_str),
13397 Some("\"never_provided\""),
13398 "{event:?}"
13399 );
13400 }
13401
13402 #[test]
13403 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
13404 let (events, verdict) = deregister_required_provider(false);
13405 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
13406 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13407 let event = &events[0];
13408 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
13409 assert_eq!(event.fields.get("cause"), None, "{event:?}");
13410 assert_eq!(
13411 event.fields.get("verdict").map(String::as_str),
13412 Some("\"never_provided\""),
13413 "{event:?}"
13414 );
13415 }
13416
13417 #[test]
13418 fn catalog_list_omits_capabilities_for_legacy_manifest() {
13419 let registry = Arc::new(Registry::default());
13420 let handler = ControlHandler::new(Arc::clone(®istry));
13421 let hello = handler
13422 .handle_control(
13423 ConnectionId::new(101),
13424 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
13425 )
13426 .expect("legacy HELLO registers");
13427 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13428
13429 let request = Frame::build(
13430 FrameType::Request,
13431 control_flags(),
13432 0,
13433 0,
13434 102,
13435 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
13436 .expect("catalog request serializes"),
13437 )
13438 .expect("catalog request frame builds");
13439 let response = handler
13440 .handle_catalog_list(request, None)
13441 .expect("catalog list succeeds");
13442 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
13443 assert!(
13444 body["modules"][0].get("capabilities").is_none(),
13445 "legacy manifest must retain an absent capabilities field on catalog.list"
13446 );
13447 }
13448
13449 #[test]
13450 fn hello_ack_omits_storage_when_no_storage_config() {
13451 let registry = Arc::new(Registry::default());
13452 let handler = ControlHandler::new(Arc::clone(®istry));
13453 let responses = handler
13454 .handle_control(
13455 ConnectionId::new(1),
13456 hello_frame("aft", PROTOCOL_VERSION, 7),
13457 )
13458 .unwrap();
13459 let ack = parse_ack(&responses[0]);
13460 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
13461 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
13462 }
13463
13464 #[tokio::test]
13465 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
13466 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
13467 let registry = Arc::new(Registry::default());
13468 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
13469 let responses = handler
13470 .handle_control(
13471 ConnectionId::new(1),
13472 hello_frame("aft", PROTOCOL_VERSION, 7),
13473 )
13474 .unwrap();
13475 let ack = parse_ack(&responses[0]);
13476 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
13477
13478 let described = handler
13479 .handle_control_frame(
13480 &route_ctx(ConnectionId::new(2)).0,
13481 Frame::build(
13482 FrameType::Request,
13483 control_flags(),
13484 0,
13485 0,
13486 9,
13487 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13488 )
13489 .unwrap(),
13490 )
13491 .await
13492 .unwrap();
13493 let ClientControlResponse::ServerDescribe { machine_id, .. } =
13494 serde_json::from_slice(&described[0].body).unwrap()
13495 else {
13496 panic!("server.describe answered with another shape");
13497 };
13498 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
13499 }
13500
13501 #[test]
13502 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
13503 let registry = Arc::new(Registry::default());
13506 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
13507 crate::daemon_config::StorageConfig::Sqlite {
13508 data_home: std::path::PathBuf::from("/data"),
13509 },
13510 ));
13511
13512 let responses = handler
13513 .handle_control(
13514 ConnectionId::new(1),
13515 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
13516 )
13517 .unwrap();
13518 let ack = parse_ack(&responses[0]);
13519 assert_eq!(
13520 ack.storage,
13521 Some(serde_json::json!({
13522 "module_id": "alfonso-routing",
13523 "storage_namespace": "default",
13524 "isolation": { "kind": "module" },
13525 "backend": {
13526 "backend": "sqlite",
13527 "path": "/data/cortexkit/alfonso-routing/store.db"
13528 }
13529 })),
13530 "the delivered descriptor is the module's own sqlite store path"
13531 );
13532 }
13533
13534 #[test]
13535 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
13536 let registry = Arc::new(Registry::default());
13537 let handler = ControlHandler::new(Arc::clone(®istry));
13538 let conn = ConnectionId::new(1);
13539 let responses = handler
13540 .handle_control(
13541 conn,
13542 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13543 )
13544 .unwrap();
13545 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
13546 let registration = registry.get_module("aft").unwrap().unwrap();
13547 assert_eq!(registration.control_ops, module_baseline_control_ops());
13548
13549 let frame =
13550 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
13551 assert!(handler
13552 .guard_module_control_op(&frame, "aft", "route.bind")
13553 .unwrap()
13554 .is_none());
13555 let error = handler
13556 .guard_module_control_op(&frame, "aft", "test.synthetic")
13557 .unwrap()
13558 .expect("synthetic ungranted op should be rejected");
13559 assert_eq!(error.header.ty, FrameType::Error);
13560 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
13561 }
13562
13563 #[test]
13564 fn hello_control_ops_some_adds_optional_grants() {
13565 let registry = Arc::new(Registry::default());
13566 let handler = ControlHandler::new(Arc::clone(®istry));
13567 handler
13568 .handle_control(
13569 ConnectionId::new(1),
13570 hello_frame_with_control_ops(
13571 "aft",
13572 PROTOCOL_VERSION,
13573 7,
13574 Some(vec![
13575 "future.synthetic".to_string(),
13576 "route.bind".to_string(),
13577 ]),
13578 ),
13579 )
13580 .unwrap();
13581 let registration = registry.get_module("aft").unwrap().unwrap();
13582 assert_eq!(
13583 registration.control_ops,
13584 vec![
13585 "route.bind".to_string(),
13586 "route.status".to_string(),
13587 "future.synthetic".to_string(),
13588 ]
13589 );
13590 let frame =
13591 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
13592 assert!(handler
13593 .guard_module_control_op(&frame, "aft", "future.synthetic")
13594 .unwrap()
13595 .is_none());
13596 }
13597
13598 #[tokio::test]
13599 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
13600 let registry = Arc::new(Registry::default());
13601 let forwarding = Arc::new(ForwardingTable::default());
13602 let handler =
13603 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13604 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
13605 hello_via_sink(
13606 &handler,
13607 &module_ctx,
13608 &mut module_rx,
13609 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13610 )
13611 .await;
13612
13613 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
13614 let responses = handler
13615 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
13616 .await
13617 .unwrap();
13618 assert_eq!(responses.len(), 1);
13619 assert_eq!(responses[0].header.ty, FrameType::Error);
13620 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
13621 assert!(module_rx.try_recv().is_err());
13622 }
13623
13624 #[tokio::test]
13625 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
13626 let registry = Arc::new(Registry::default());
13627 let forwarding = Arc::new(ForwardingTable::default());
13628 let handler =
13629 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13630 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
13631 hello_via_sink(
13632 &handler,
13633 &module_ctx,
13634 &mut module_rx,
13635 hello_frame_with_control_ops(
13636 "aft",
13637 PROTOCOL_VERSION,
13638 7,
13639 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13640 ),
13641 )
13642 .await;
13643
13644 let project_root = unique_project_root("demux");
13645 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
13646 let route_handler = handler.clone();
13647 let route_task = tokio::spawn(async move {
13648 route_handler
13649 .handle_control_frame(
13650 &route_client_ctx,
13651 route_open_frame(100, "aft", project_root),
13652 )
13653 .await
13654 .unwrap()
13655 });
13656 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13657 .await
13658 .unwrap()
13659 .unwrap();
13660 assert!(matches!(
13661 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
13662 ModuleControlRequest::RouteBind { .. }
13663 ));
13664
13665 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
13666 let health_handler = handler.clone();
13667 let health_task = tokio::spawn(async move {
13668 health_handler
13669 .handle_control_frame(
13670 &health_client_ctx,
13671 supervisor_health_probe_frame(101, "aft"),
13672 )
13673 .await
13674 .unwrap()
13675 });
13676 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13677 .await
13678 .unwrap()
13679 .unwrap();
13680 assert_eq!(
13681 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13682 ModuleControlRequest::HealthCheck {}
13683 );
13684
13685 handler
13686 .handle_control_frame(
13687 &module_ctx,
13688 health_response(health_frame.header.corr, HealthStatus::Degraded),
13689 )
13690 .await
13691 .unwrap();
13692 let health_response = health_task.await.unwrap();
13693 assert_eq!(health_response.len(), 1);
13694 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13695 ClientControlResponse::SupervisorHealthProbe {
13696 module_id,
13697 status,
13698 detail,
13699 metrics,
13700 } => {
13701 assert_eq!(module_id, "aft");
13702 assert_eq!(status, HealthStatus::Degraded);
13703 assert_eq!(detail.as_deref(), Some("warming"));
13704 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
13705 }
13706 other => panic!("unexpected health response: {other:?}"),
13707 }
13708
13709 handler
13710 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13711 .await
13712 .unwrap();
13713 let route_response = route_task.await.unwrap();
13714 assert!(route_response.is_empty());
13715 let published = route_client_rx.recv().await.unwrap();
13716 assert!(matches!(
13717 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13718 ClientControlResponse::RouteOpen { .. }
13719 ));
13720 }
13721
13722 async fn relay_route_open(
13727 handler: &ControlHandler,
13728 client_connection: ConnectionId,
13729 client_egress: &FrameSink,
13730 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13731 corr: u64,
13732 module_id: &str,
13733 project_root_label: &str,
13734 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
13735 let ctx = RouteCtx {
13736 connection_id: client_connection,
13737 egress: client_egress.clone(),
13738 };
13739 let handler = handler.clone();
13740 let project_root = unique_project_root(project_root_label);
13741 let module_id = module_id.to_string();
13742 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
13743 let task = tokio::spawn(async move {
13744 let _guard = tracing::dispatcher::set_default(&dispatch);
13745 handler
13746 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
13747 .await
13748 .unwrap()
13749 });
13750 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13751 .await
13752 .expect("module receives the relayed route.bind")
13753 .expect("module egress is open");
13754 (task, bind.frame)
13755 }
13756
13757 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
13758 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
13759 ModuleControlRequest::RouteBind {
13760 route_channel,
13761 epoch,
13762 ..
13763 } => (route_channel, epoch),
13764 other => panic!("expected a route.bind request, got {other:?}"),
13765 }
13766 }
13767
13768 fn published_route(frame: &Frame) -> (u16, u32) {
13769 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
13770 ClientControlResponse::RouteOpen {
13771 route_channel,
13772 route_epoch,
13773 ..
13774 } => (route_channel, route_epoch),
13775 other => panic!("expected a route.open response, got {other:?}"),
13776 }
13777 }
13778
13779 #[tokio::test]
13799 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
13800 let registry = Arc::new(Registry::default());
13801 let forwarding = Arc::new(ForwardingTable::default());
13802 let handler =
13803 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13804
13805 let module_connection = ConnectionId::new(30);
13806 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13807 hello_via_sink(
13808 &handler,
13809 &module_ctx,
13810 &mut module_rx,
13811 hello_frame("aft", PROTOCOL_VERSION, 7),
13812 )
13813 .await;
13814
13815 let dying_client = ConnectionId::new(31);
13816 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
13817
13818 let (first_task, first_bind) = relay_route_open(
13821 &handler,
13822 dying_client,
13823 &dying_ctx.egress,
13824 &mut module_rx,
13825 100,
13826 "aft",
13827 "closing-first",
13828 )
13829 .await;
13830 handler
13831 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
13832 .await
13833 .unwrap();
13834 assert!(first_task.await.unwrap().is_empty());
13835 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
13836
13837 let (second_task, second_bind) = relay_route_open(
13839 &handler,
13840 dying_client,
13841 &dying_ctx.egress,
13842 &mut module_rx,
13843 101,
13844 "aft",
13845 "closing-second",
13846 )
13847 .await;
13848 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
13849
13850 assert!(forwarding
13853 .escalate_client_delivery_failure(
13854 dying_client,
13855 first_channel,
13856 first_epoch,
13857 CloseReason::new(
13858 "module_to_client_delivery_failed",
13859 "client egress refused a module frame",
13860 ),
13861 crate::forwarding::UndeliveredFrame {
13862 module_id: None,
13863 sink: &dying_ctx.egress,
13864 },
13865 )
13866 .unwrap());
13867 assert!(!dying_ctx.egress.is_closed());
13868
13869 let ack = handler
13871 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
13872 .await;
13873 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13874 if module_loop_error.is_some() {
13875 handler.cleanup_connection(module_connection).unwrap();
13879 }
13880 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13884 .await
13885 .ok()
13886 .flatten();
13887
13888 assert!(
13890 registry
13891 .get_module_by_connection(module_connection)
13892 .unwrap()
13893 .is_some(),
13894 "one client's closing connection ended the shared module connection: \
13895 {module_loop_error:?}"
13896 );
13897 let cotenant = ConnectionId::new(32);
13899 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13900 let (cotenant_task, cotenant_bind) = relay_route_open(
13901 &handler,
13902 cotenant,
13903 &cotenant_ctx.egress,
13904 &mut module_rx,
13905 102,
13906 "aft",
13907 "closing-cotenant",
13908 )
13909 .await;
13910 handler
13911 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
13912 .await
13913 .unwrap();
13914 assert!(cotenant_task.await.unwrap().is_empty());
13915 let (cotenant_channel, cotenant_epoch) =
13916 published_route(&cotenant_rx.recv().await.unwrap());
13917 assert!(matches!(
13918 forwarding
13919 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
13920 .unwrap(),
13921 DataRoute::Client(DataRouteState::Bound(_))
13922 ));
13923
13924 let goodbye = post_ack_module_frame
13927 .expect("module receives a GOODBYE for the abandoned route channel");
13928 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13929 assert_eq!(goodbye.header.channel, abandoned_channel);
13930 assert_eq!(goodbye.header.epoch, abandoned_epoch);
13931
13932 assert!(dying_rx.try_recv().is_err());
13936 let second_response = second_task.await.unwrap();
13937 assert_eq!(second_response.len(), 1);
13938 assert_eq!(
13939 parse_error(&second_response[0])["code"],
13940 "target_unavailable"
13941 );
13942 }
13943
13944 async fn catalog_snapshot(
13947 handler: &ControlHandler,
13948 client_ctx: &RouteCtx,
13949 corr: u64,
13950 ) -> (u64, Vec<(String, bool)>) {
13951 let body =
13952 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
13953 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
13954 let reply = handler
13955 .handle_control_frame(client_ctx, request)
13956 .await
13957 .unwrap()
13958 .pop()
13959 .unwrap();
13960 let ClientControlResponse::CatalogList {
13961 generation,
13962 modules,
13963 ..
13964 } = serde_json::from_slice(&reply.body).unwrap()
13965 else {
13966 panic!("not a catalog.list reply");
13967 };
13968 (
13969 generation,
13970 modules
13971 .into_iter()
13972 .map(|entry| (entry.module_id, entry.ready))
13973 .collect(),
13974 )
13975 }
13976
13977 fn hello_held_before_forwarding() -> (
13983 ControlHandler,
13984 Arc<std::sync::Barrier>,
13985 std::thread::JoinHandle<Vec<Frame>>,
13986 RouteCtx,
13987 mpsc::Receiver<crate::router::OutboundFrame>,
13988 ) {
13989 let hold = Arc::new(std::sync::Barrier::new(2));
13990 let handler = ControlHandler::with_forwarding(
13991 Arc::new(Registry::default()),
13992 Arc::new(ForwardingTable::default()),
13993 )
13994 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
13995 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
13996 let hello = {
13997 let handler = handler.clone();
13998 let module_ctx = module_ctx.clone();
13999 std::thread::spawn(move || {
14000 handler
14001 .handle_hello(
14002 module_ctx.connection_id,
14003 Some(module_ctx.egress.clone()),
14004 hello_frame("aft", PROTOCOL_VERSION, 7),
14005 )
14006 .unwrap()
14007 })
14008 };
14009 (handler, hold, hello, module_ctx, module_rx)
14010 }
14011
14012 fn lookup_during_hello_publication<T: Send + 'static>(
14018 registry: &Registry,
14019 hold: &std::sync::Barrier,
14020 lookup: impl FnOnce() -> T + Send + 'static,
14021 ) -> T {
14022 let attempted = registry.observe_next_lock_attempt();
14023 let reader = std::thread::spawn(lookup);
14024 if attempted.recv().unwrap() {
14025 let result = reader.join().unwrap();
14026 hold.wait();
14027 result
14028 } else {
14029 hold.wait();
14030 reader.join().unwrap()
14031 }
14032 }
14033
14034 #[test]
14035 fn catalog_read_after_hello_ack_waits_for_publication() {
14036 let registry = Arc::new(Registry::default());
14037 let hold = Arc::new(std::sync::Barrier::new(2));
14038 let mut handler = ControlHandler::with_forwarding(
14039 Arc::clone(®istry),
14040 Arc::new(ForwardingTable::default()),
14041 );
14042 handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
14043 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14044 let hello = {
14045 let handler = handler.clone();
14046 std::thread::spawn(move || {
14047 handler
14048 .handle_hello(
14049 module_ctx.connection_id,
14050 Some(module_ctx.egress),
14051 hello_frame("aft", PROTOCOL_VERSION, 7),
14052 )
14053 .unwrap()
14054 })
14055 };
14056 hold.wait();
14057 assert_eq!(
14058 module_rx.try_recv().unwrap().frame.header.ty,
14059 FrameType::HelloAck
14060 );
14061 let replies = lookup_during_hello_publication(®istry, &hold, move || {
14062 let request = Frame::build(
14065 FrameType::Request,
14066 control_flags(),
14067 0,
14068 0,
14069 300,
14070 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
14071 )
14072 .unwrap();
14073 handler.handle_catalog_list(request, None).unwrap()
14074 });
14075 assert!(hello.join().unwrap().is_empty());
14076 let ClientControlResponse::CatalogList {
14077 generation,
14078 modules,
14079 ..
14080 } = serde_json::from_slice(&replies[0].body).unwrap()
14081 else {
14082 panic!("not a catalog.list reply");
14083 };
14084 assert_eq!(
14085 modules
14086 .iter()
14087 .map(|entry| entry.module_id.as_str())
14088 .collect::<Vec<_>>(),
14089 vec!["aft"]
14090 );
14091 assert_eq!(generation, 1);
14092 }
14093
14094 #[tokio::test]
14103 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
14104 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
14105 let client = ConnectionId::new(41);
14106 let (client_ctx, mut client_rx) = route_ctx(client);
14107
14108 hold.wait();
14109 assert!(handler.registry.get_module("aft").unwrap().is_some());
14111 assert!(!handler
14112 .forwarding
14113 .has_live_module_connection("aft")
14114 .unwrap());
14115 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
14116 assert!(
14117 listed.is_empty(),
14118 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
14119 );
14120 let refused = handler
14121 .handle_control_frame(
14122 &client_ctx,
14123 route_open_frame(301, "aft", unique_project_root("listed-window")),
14124 )
14125 .await
14126 .unwrap();
14127 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
14128 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
14129 let duplicate = handler
14130 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
14131 .await
14132 .unwrap();
14133 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14134
14135 hold.wait();
14136 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14137 let ack = module_rx.try_recv().unwrap().frame;
14138 assert_eq!(ack.header.ty, FrameType::HelloAck);
14139 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14140 assert_eq!(listed, vec![("aft".to_string(), true)]);
14141
14142 let (open_task, bind) = relay_route_open(
14143 &handler,
14144 client,
14145 &client_ctx.egress,
14146 &mut module_rx,
14147 303,
14148 "aft",
14149 "listed-after",
14150 )
14151 .await;
14152 assert!(handler
14153 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14154 .await
14155 .unwrap()
14156 .is_empty());
14157 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
14158 published_route(&client_rx.try_recv().unwrap().frame);
14159 }
14160
14161 #[tokio::test]
14168 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
14169 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
14170 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14171 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14172
14173 hold.wait();
14174 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
14175 assert!(listed.is_empty());
14176 assert_eq!(
14177 during, before,
14178 "the generation moved while the module was not listed"
14179 );
14180
14181 hold.wait();
14182 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14183 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14184 assert_eq!(listed, vec![("aft".to_string(), true)]);
14185 assert_eq!(
14186 after,
14187 before + 1,
14188 "the module appeared without exactly one generation bump"
14189 );
14190 }
14191
14192 #[tokio::test]
14197 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
14198 let registry = Arc::new(Registry::default());
14199 let handler = ControlHandler::with_forwarding(
14200 Arc::clone(®istry),
14201 Arc::new(ForwardingTable::default()),
14202 );
14203 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14204 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14205 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14206 drop(module_rx);
14207
14208 let replies = handler
14209 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
14210 .await
14211 .unwrap();
14212 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
14213 assert!(registry.get_module("aft").unwrap().is_none());
14214 assert!(registry
14215 .get_module_by_connection(module_ctx.connection_id)
14216 .unwrap()
14217 .is_none());
14218 assert_eq!(
14219 catalog_snapshot(&handler, &client_ctx, 301).await,
14220 (before, Vec::new())
14221 );
14222 }
14223
14224 async fn route_open_held_after_bind_send() -> (
14228 ControlHandler,
14229 Arc<tokio::sync::Barrier>,
14230 RouteCtx,
14231 mpsc::Receiver<crate::router::OutboundFrame>,
14232 ) {
14233 let hold = Arc::new(tokio::sync::Barrier::new(2));
14234 let handler = ControlHandler::with_forwarding(
14235 Arc::new(Registry::default()),
14236 Arc::new(ForwardingTable::default()),
14237 )
14238 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
14239 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14240 hello_via_sink(
14241 &handler,
14242 &module_ctx,
14243 &mut module_rx,
14244 hello_frame("aft", PROTOCOL_VERSION, 7),
14245 )
14246 .await;
14247 (handler, hold, module_ctx, module_rx)
14248 }
14249
14250 #[tokio::test]
14257 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
14258 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14259 let client = ConnectionId::new(41);
14260 let (client_ctx, mut client_rx) = route_ctx(client);
14261
14262 let (open_task, bind) = relay_route_open(
14263 &handler,
14264 client,
14265 &client_ctx.egress,
14266 &mut module_rx,
14267 200,
14268 "aft",
14269 "acked-before-mark",
14270 )
14271 .await;
14272 let (module_channel, module_epoch) = route_bind_channel(&bind);
14273 hold.wait().await;
14275
14276 assert!(handler
14278 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14279 .await
14280 .unwrap()
14281 .is_empty());
14282 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
14283
14284 hold.wait().await;
14285 assert!(
14286 open_task.await.unwrap().is_empty(),
14287 "route.open answers accepted"
14288 );
14289
14290 match module_rx.try_recv() {
14293 Err(mpsc::error::TryRecvError::Empty) => {}
14294 Ok(outbound) => panic!(
14295 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
14296 outbound.frame.header.ty,
14297 outbound.frame.header.channel,
14298 outbound.frame.header.epoch
14299 ),
14300 Err(err) => panic!("module egress closed: {err}"),
14301 }
14302
14303 let router = Router::with_control_handler(Arc::new(handler.clone()));
14306 let request = Frame::build(
14307 FrameType::Request,
14308 Flags::new(true, Priority::Interactive, false),
14309 client_channel,
14310 client_epoch,
14311 201,
14312 b"first request".to_vec(),
14313 )
14314 .unwrap();
14315 router
14316 .route_for_connection(&client_ctx, request)
14317 .await
14318 .unwrap();
14319 let delivered = module_rx.try_recv().unwrap().frame;
14320 assert_eq!(delivered.header.ty, FrameType::Request);
14321 assert_eq!(
14322 (delivered.header.channel, delivered.header.epoch),
14323 (module_channel, module_epoch)
14324 );
14325 assert_eq!(delivered.body, b"first request");
14326 }
14327
14328 #[tokio::test]
14334 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
14335 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14336 let client = ConnectionId::new(42);
14337 let (client_ctx, _client_rx) = route_ctx(client);
14338
14339 let (open_task, bind) = relay_route_open(
14340 &handler,
14341 client,
14342 &client_ctx.egress,
14343 &mut module_rx,
14344 300,
14345 "aft",
14346 "released-before-mark",
14347 )
14348 .await;
14349 let (module_channel, module_epoch) = route_bind_channel(&bind);
14350 hold.wait().await;
14351
14352 handler.cleanup_connection(client).unwrap();
14353 assert!(
14354 module_rx.try_recv().is_err(),
14355 "client teardown must not GOODBYE a bind route.open is still queueing"
14356 );
14357
14358 hold.wait().await;
14359 let response = open_task.await.unwrap();
14360 assert_eq!(response.len(), 1);
14361 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14362
14363 let goodbye = module_rx
14364 .try_recv()
14365 .expect("module is told to drop the binding the daemon abandoned")
14366 .frame;
14367 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14368 assert_eq!(
14369 (goodbye.header.channel, goodbye.header.epoch),
14370 (module_channel, module_epoch)
14371 );
14372 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14373 }
14374
14375 #[tokio::test]
14380 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
14381 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14382 let client = ConnectionId::new(43);
14383 let (client_ctx, client_rx) = route_ctx(client);
14384
14385 let (open_task, bind) = relay_route_open(
14386 &handler,
14387 client,
14388 &client_ctx.egress,
14389 &mut module_rx,
14390 400,
14391 "aft",
14392 "gone-client-before-mark",
14393 )
14394 .await;
14395 let (module_channel, module_epoch) = route_bind_channel(&bind);
14396 hold.wait().await;
14397
14398 drop(client_rx);
14399 assert!(client_ctx.egress.is_closed());
14400 handler
14401 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14402 .await
14403 .unwrap();
14404
14405 hold.wait().await;
14406 let response = open_task.await.unwrap();
14407 assert_eq!(response.len(), 1);
14408 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14409
14410 let goodbye = module_rx
14411 .try_recv()
14412 .expect("module is told to drop the binding it acked")
14413 .frame;
14414 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14415 assert_eq!(
14416 (goodbye.header.channel, goodbye.header.epoch),
14417 (module_channel, module_epoch)
14418 );
14419 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14420 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14421 }
14422
14423 #[test]
14430 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
14431 let handler = ControlHandler::default();
14432 let module_connection = ConnectionId::new(30);
14433 let client_connection = ConnectionId::new(31);
14434
14435 handler
14436 .refuse_to_end_module_connection_for_a_client(
14437 module_connection,
14438 77,
14439 ForwardingError::ConnectionClosing {
14440 connection_id: client_connection,
14441 },
14442 )
14443 .expect("a closing client must never end the module connection");
14444
14445 assert!(matches!(
14446 handler.refuse_to_end_module_connection_for_a_client(
14447 module_connection,
14448 78,
14449 ForwardingError::ConnectionClosing {
14450 connection_id: module_connection,
14451 },
14452 ),
14453 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
14454 connection_id
14455 })) if connection_id == module_connection
14456 ));
14457 assert!(matches!(
14458 handler.refuse_to_end_module_connection_for_a_client(
14459 module_connection,
14460 79,
14461 ForwardingError::Poisoned,
14462 ),
14463 Err(RouterError::Forwarding(ForwardingError::Poisoned))
14464 ));
14465 assert!(matches!(
14466 handler.refuse_to_end_module_connection_for_a_client(
14467 module_connection,
14468 80,
14469 ForwardingError::StaleModuleEndpoint,
14470 ),
14471 Err(RouterError::Forwarding(
14472 ForwardingError::StaleModuleEndpoint
14473 ))
14474 ));
14475 }
14476
14477 #[tokio::test]
14490 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
14491 let registry = Arc::new(Registry::default());
14492 let forwarding = Arc::new(ForwardingTable::default());
14493 let supervisor = SupervisorHandle::new();
14494 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14495 let handler =
14496 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14497 .with_supervisor(supervisor);
14498
14499 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14500 hello_via_sink(
14501 &handler,
14502 &target_ctx,
14503 &mut target_rx,
14504 hello_frame("target", PROTOCOL_VERSION, 1),
14505 )
14506 .await;
14507
14508 let wrong_nonce = handler
14512 .handle_control_frame(
14513 &route_ctx(ConnectionId::new(91)).0,
14514 route_open_frame_with_admission_facts(
14515 20,
14516 "target",
14517 unique_project_root("admission-facts"),
14518 Some(subc_control::ConsumerIdentity {
14519 module_id: "fed".to_string(),
14520 launch_nonce: "not-the-real-nonce".to_string(),
14521 }),
14522 None,
14523 ),
14524 )
14525 .await
14526 .unwrap();
14527 assert_eq!(
14528 parse_error(&wrong_nonce[0])["code"],
14529 "bad_consumer_identity",
14530 "a mismatched launch nonce must be refused, not stamped Reserved"
14531 );
14532
14533 let never_spawned = handler
14537 .handle_control_frame(
14538 &route_ctx(ConnectionId::new(92)).0,
14539 route_open_frame_with_admission_facts(
14540 21,
14541 "target",
14542 unique_project_root("admission-facts"),
14543 Some(subc_control::ConsumerIdentity {
14544 module_id: "never-spawned".to_string(),
14545 launch_nonce: "any-nonce".to_string(),
14546 }),
14547 None,
14548 ),
14549 )
14550 .await
14551 .unwrap();
14552 assert_eq!(
14553 parse_error(&never_spawned[0])["code"],
14554 "bad_consumer_identity",
14555 "an unspawned module_id must be refused rather than accepted for lack of a record"
14556 );
14557 }
14558
14559 #[tokio::test]
14582 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
14583 let registry = Arc::new(Registry::default());
14584 let forwarding = Arc::new(ForwardingTable::default());
14585 let supervisor = SupervisorHandle::new();
14586 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14587 let handler =
14588 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14589 .with_supervisor(supervisor);
14590
14591 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
14592 hello_via_sink(
14593 &handler,
14594 &target_ctx,
14595 &mut target_rx,
14596 hello_frame("target", PROTOCOL_VERSION, 1),
14597 )
14598 .await;
14599
14600 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
14601 let route_handler = handler.clone();
14602 let route_task = tokio::spawn(async move {
14603 route_handler
14604 .handle_control_frame(
14605 &client_ctx,
14606 route_open_frame_with_admission_facts(
14607 30,
14608 "target",
14609 unique_project_root("admission-facts"),
14610 Some(subc_control::ConsumerIdentity {
14611 module_id: "fed".to_string(),
14612 launch_nonce: "fed-nonce".to_string(),
14613 }),
14614 None,
14615 ),
14616 )
14617 .await
14618 .unwrap()
14619 });
14620
14621 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14628 .await
14629 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
14630 .expect("module control channel closed before route.bind");
14631 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14632 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14633 panic!("expected route.bind")
14634 };
14635 assert_eq!(
14636 principal,
14637 Some(Principal::Reserved {
14638 module_id: "fed".to_string()
14639 }),
14640 "a correctly attested consumer must be stamped Reserved for its own id"
14641 );
14642
14643 handler
14644 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14645 .await
14646 .unwrap();
14647 assert!(route_task.await.unwrap().is_empty());
14648 assert!(
14649 matches!(
14650 serde_json::from_slice::<ClientControlResponse>(
14651 &client_rx.recv().await.unwrap().body
14652 )
14653 .unwrap(),
14654 ClientControlResponse::RouteOpen { .. }
14655 ),
14656 "the route must actually open, not merely avoid an error"
14657 );
14658 }
14659
14660 #[tokio::test(start_paused = true)]
14661 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
14662 let registry = Arc::new(Registry::default());
14663 let forwarding = Arc::new(ForwardingTable::default());
14664 let supervisor = SupervisorHandle::new();
14665 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14666 let handler =
14667 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14668 .with_supervisor(supervisor);
14669
14670 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
14671 hello_via_sink(
14672 &handler,
14673 &target_ctx,
14674 &mut target_rx,
14675 hello_frame("target", PROTOCOL_VERSION, 1),
14676 )
14677 .await;
14678
14679 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
14680 let direct_handler = handler.clone();
14681 let direct_open = tokio::spawn(async move {
14682 direct_handler
14683 .handle_control_frame(
14684 &direct_ctx,
14685 route_open_frame(2, "target", unique_project_root("route-census-direct")),
14686 )
14687 .await
14688 .unwrap()
14689 });
14690 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14691 .await
14692 .expect("no direct route.bind within 5s")
14693 .expect("target control channel closed before direct route.bind");
14694 handler
14695 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
14696 .await
14697 .unwrap();
14698 assert!(direct_open.await.unwrap().is_empty());
14699 let _ = direct_rx.recv().await.unwrap();
14700
14701 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
14702 let reserved_handler = handler.clone();
14703 let reserved_open = tokio::spawn(async move {
14704 reserved_handler
14705 .handle_control_frame(
14706 &reserved_ctx,
14707 route_open_frame_with_admission_facts(
14708 3,
14709 "target",
14710 unique_project_root("admission-facts"),
14711 Some(ConsumerIdentity {
14712 module_id: "fed".to_string(),
14713 launch_nonce: "fed-nonce".to_string(),
14714 }),
14715 None,
14716 ),
14717 )
14718 .await
14719 .unwrap()
14720 });
14721 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14722 .await
14723 .expect("no reserved route.bind within 5s")
14724 .expect("target control channel closed before reserved route.bind");
14725 handler
14726 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
14727 .await
14728 .unwrap();
14729 assert!(reserved_open.await.unwrap().is_empty());
14730 let _ = reserved_rx.recv().await.unwrap();
14731
14732 forwarding
14733 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
14734 .unwrap();
14735 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
14736 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
14737 module_id: Some("target".to_string()),
14738 })
14739 .unwrap();
14740 let census_frame =
14741 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
14742 let response = handler
14743 .handle_control_frame(&census_ctx, census_frame)
14744 .await
14745 .unwrap()
14746 .pop()
14747 .unwrap();
14748 let actual: Value = serde_json::from_slice(&response.body).unwrap();
14749 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
14750 assert!(matches!(
14751 decoded,
14752 ClientControlResponse::SupervisorRoutes { .. }
14753 ));
14754 let routes = actual["modules"][0]["routes"].as_array().unwrap();
14755 assert_eq!(routes.len(), 2);
14756 assert!(routes.iter().all(|route| route["draining"] == true));
14757 assert!(
14760 routes.iter().all(|route| route["drain_reason"] == "reload"),
14761 "draining routes must name the drain's reason: {routes:?}"
14762 );
14763 assert!(routes.iter().any(|route| {
14764 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
14765 }));
14766 assert!(routes.iter().any(|route| {
14767 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
14768 }));
14769
14770 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14771 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
14772 if std::env::var_os("UPDATE_GOLDEN").is_some() {
14773 std::fs::write(
14774 &golden_path,
14775 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
14776 )
14777 .unwrap();
14778 }
14779 let expected: Value =
14780 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
14781 assert_eq!(actual, expected);
14782 }
14783
14784 async fn query_live_roots(
14785 handler: &ControlHandler,
14786 module_ctx: &RouteCtx,
14787 ) -> ModuleControlResponseToModule {
14788 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
14789 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
14790 let response = handler
14791 .handle_control_frame(module_ctx, frame)
14792 .await
14793 .unwrap()
14794 .pop()
14795 .unwrap();
14796 serde_json::from_slice(&response.body).unwrap()
14797 }
14798
14799 #[tokio::test(start_paused = true)]
14800 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
14801 let registry = Arc::new(Registry::default());
14802 let forwarding = Arc::new(ForwardingTable::default());
14803 let handler = ControlHandler::with_forwarding(registry, forwarding);
14804 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
14805 hello_via_sink(
14806 &handler,
14807 &target_ctx,
14808 &mut target_rx,
14809 hello_frame("target", PROTOCOL_VERSION, 1),
14810 )
14811 .await;
14812 let root = unique_project_root("live-roots-known");
14813 let path = ProjectRootId::from_path_allowing_missing(root.path())
14814 .unwrap()
14815 .as_path()
14816 .to_path_buf();
14817 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
14818 let open_handler = handler.clone();
14819 let opened = tokio::spawn(async move {
14820 open_handler
14821 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
14822 .await
14823 .unwrap()
14824 });
14825 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14826 .await
14827 .unwrap()
14828 .unwrap();
14829 handler
14830 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
14831 .await
14832 .unwrap();
14833 assert!(opened.await.unwrap().is_empty());
14834 let _ = client_rx.recv().await.unwrap();
14835
14836 let root = unique_project_root("live-roots-pending");
14837 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
14838 .unwrap()
14839 .as_path()
14840 .to_path_buf();
14841 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
14842 let open_handler = handler.clone();
14843 let pending = tokio::spawn(async move {
14844 open_handler
14845 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
14846 .await
14847 .unwrap()
14848 });
14849 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14850 .await
14851 .unwrap()
14852 .unwrap();
14853 let actual = query_live_roots(&handler, &target_ctx).await;
14854 let ModuleControlResponseToModule::LiveRoots {
14855 roots,
14856 unknown_root_bindings,
14857 total_bindings,
14858 } = actual
14859 else {
14860 panic!("expected live roots")
14861 };
14862 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
14863 assert_eq!(unknown_root_bindings, 0);
14864 assert_eq!(
14865 roots.len(),
14866 2,
14867 "root-known arm must retain each canonical root"
14868 );
14869 assert_eq!(
14870 total_bindings,
14871 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14872 );
14873 let counts = roots
14874 .iter()
14875 .map(|root| (root.project_root.clone(), root.bound, root.pending))
14876 .collect::<Vec<_>>();
14877 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
14878 expected.sort_by(|a, b| a.0.cmp(&b.0));
14879 assert_eq!(
14880 counts, expected,
14881 "roots must sort by path and count pending separately"
14882 );
14883 handler
14884 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
14885 .await
14886 .unwrap();
14887 assert!(pending.await.unwrap().is_empty());
14888 }
14889
14890 #[tokio::test(start_paused = true)]
14891 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
14892 let forwarding = Arc::new(ForwardingTable::default());
14893 let handler =
14894 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14895 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
14896 hello_via_sink(
14897 &handler,
14898 &target_ctx,
14899 &mut target_rx,
14900 hello_frame("target", PROTOCOL_VERSION, 1),
14901 )
14902 .await;
14903 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
14904 let pending = forwarding
14905 .begin_route_bind_relay_for_test(
14906 client_ctx.connection_id,
14907 client_ctx.egress.clone(),
14908 2,
14909 "target",
14910 )
14911 .unwrap();
14912 forwarding
14913 .complete_pending_relay(
14914 target_ctx.connection_id,
14915 pending.corr,
14916 RouteBindRelayOutcome::Accepted,
14917 )
14918 .unwrap();
14919 let actual = query_live_roots(&handler, &target_ctx).await;
14920 let ModuleControlResponseToModule::LiveRoots {
14921 roots,
14922 unknown_root_bindings,
14923 total_bindings,
14924 } = actual
14925 else {
14926 panic!("expected live roots")
14927 };
14928 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
14929 assert_eq!(
14930 unknown_root_bindings, 1,
14931 "unknown-root arm must not read as no bindings"
14932 );
14933 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
14934 assert_eq!(
14935 total_bindings,
14936 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14937 );
14938 }
14939
14940 #[tokio::test(start_paused = true)]
14948 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
14949 let forwarding = Arc::new(ForwardingTable::default());
14950 let handler =
14951 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14952 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
14953 let replies = handler
14954 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
14955 .await
14956 .unwrap();
14957 let queued_by_hello = module_rx.len();
14958
14959 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
14960 let open_handler = handler.clone();
14961 let open = tokio::spawn(async move {
14962 open_handler
14963 .handle_control_frame(
14964 &client_ctx,
14965 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
14966 )
14967 .await
14968 .unwrap()
14969 });
14970 let mut spins = 0;
14972 while module_rx.len() == queued_by_hello {
14973 spins += 1;
14974 assert!(spins < 10_000, "route.open never queued a route.bind");
14975 tokio::task::yield_now().await;
14976 }
14977
14978 for reply in replies {
14980 module_ctx.egress.send(reply).await.unwrap();
14981 }
14982
14983 let first = module_rx.recv().await.unwrap().frame;
14984 assert_eq!(
14985 first.header.ty,
14986 FrameType::HelloAck,
14987 "the first frame a registering module reads must be its HELLO_ACK"
14988 );
14989 assert_eq!(first.header.corr, 7);
14990 let second = module_rx.recv().await.unwrap().frame;
14991 assert_eq!(second.header.ty, FrameType::Request);
14992 assert!(
14993 matches!(
14994 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
14995 ModuleControlRequest::RouteBind { .. }
14996 ),
14997 "the route.bind follows the ack"
14998 );
14999 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
15000
15001 handler
15002 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
15003 .await
15004 .unwrap();
15005 assert!(open.await.unwrap().is_empty());
15006 let _ = client_rx.recv().await.unwrap();
15007 }
15008
15009 #[tokio::test(start_paused = true)]
15010 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
15011 let handler = ControlHandler::with_forwarding(
15012 Arc::new(Registry::default()),
15013 Arc::new(ForwardingTable::default()),
15014 );
15015 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
15016 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
15017 hello_via_sink(
15018 &handler,
15019 &first_ctx,
15020 &mut first_rx,
15021 hello_frame("first", PROTOCOL_VERSION, 1),
15022 )
15023 .await;
15024 hello_via_sink(
15025 &handler,
15026 &second_ctx,
15027 &mut second_rx,
15028 hello_frame("second", PROTOCOL_VERSION, 2),
15029 )
15030 .await;
15031 let root = unique_project_root("second-only");
15032 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
15033 let cloned = handler.clone();
15034 let open = tokio::spawn(async move {
15035 cloned
15036 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
15037 .await
15038 .unwrap()
15039 });
15040 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
15041 .await
15042 .unwrap()
15043 .unwrap();
15044 let first = query_live_roots(&handler, &first_ctx).await;
15045 let second = query_live_roots(&handler, &second_ctx).await;
15046 assert!(
15047 matches!(
15048 first,
15049 ModuleControlResponseToModule::LiveRoots {
15050 total_bindings: 0,
15051 ..
15052 }
15053 ),
15054 "cross-module scope must not expose another module's roots"
15055 );
15056 assert!(
15057 matches!(
15058 second,
15059 ModuleControlResponseToModule::LiveRoots {
15060 total_bindings: 1,
15061 ..
15062 }
15063 ),
15064 "second module must see its pending route"
15065 );
15066 handler
15067 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
15068 .await
15069 .unwrap();
15070 assert!(open.await.unwrap().is_empty());
15071 }
15072
15073 #[tokio::test(start_paused = true)]
15074 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
15075 let handler = ControlHandler::with_forwarding(
15076 Arc::new(Registry::default()),
15077 Arc::new(ForwardingTable::default()),
15078 );
15079 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
15080 hello_via_sink(
15081 &handler,
15082 &target_ctx,
15083 &mut target_rx,
15084 hello_frame("target", PROTOCOL_VERSION, 1),
15085 )
15086 .await;
15087 let actual = query_live_roots(&handler, &target_ctx).await;
15088 let ModuleControlResponseToModule::LiveRoots {
15089 roots,
15090 unknown_root_bindings,
15091 total_bindings,
15092 } = actual
15093 else {
15094 panic!("expected live roots")
15095 };
15096 assert!(roots.is_empty());
15097 assert_eq!(unknown_root_bindings, 0);
15098 assert_eq!(total_bindings, 0);
15099 assert_eq!(
15100 total_bindings,
15101 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15102 );
15103 }
15104
15105 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
15112 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
15113 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
15114 let text = std::fs::read_to_string(&path)
15115 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
15116 let vectors: Vec<(String, Value)> = text
15117 .lines()
15118 .filter(|line| !line.trim().is_empty())
15119 .map(|line| {
15120 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
15121 let id = entry["corpus_id"]
15122 .as_str()
15123 .expect("every vector carries a corpus_id")
15124 .to_string();
15125 (id, entry["package"].clone())
15126 })
15127 .collect();
15128 assert_eq!(
15132 vectors.len(),
15133 3,
15134 "vendored fed corpus changed size; re-sync from subc-federation"
15135 );
15136
15137 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
15152 let richest = vectors
15153 .iter()
15154 .filter_map(|(_, package)| package.as_object())
15155 .map(|object| {
15156 object
15157 .keys()
15158 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
15159 .count()
15160 })
15161 .max()
15162 .unwrap_or(0);
15163 assert!(
15164 richest >= 2,
15165 "vendored corpus no longer carries a package with unmodelled fields, \
15166 so the relay test can no longer distinguish a verbatim relay from a lossy one"
15167 );
15168
15169 vectors
15170 }
15171
15172 #[tokio::test]
15180 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
15181 for (corpus_id, package) in fed_admission_facts_vectors() {
15182 let registry = Arc::new(Registry::default());
15183 let forwarding = Arc::new(ForwardingTable::default());
15184 let supervisor = SupervisorHandle::new();
15185 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15186 let handler =
15187 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15188 .with_supervisor(supervisor)
15189 .with_admission_facts_config(
15190 Some("fed".to_string()),
15191 Some(vec!["target".to_string()]),
15192 );
15193
15194 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
15195 hello_via_sink(
15196 &handler,
15197 &target_ctx,
15198 &mut target_rx,
15199 hello_frame("target", PROTOCOL_VERSION, 1),
15200 )
15201 .await;
15202
15203 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
15204 let route_handler = handler.clone();
15205 let expected = package.clone();
15206 let route_task = tokio::spawn(async move {
15207 route_handler
15208 .handle_control_frame(
15209 &client_ctx,
15210 route_open_frame_with_admission_facts(
15211 20,
15212 "target",
15213 unique_project_root("admission-facts"),
15214 Some(subc_control::ConsumerIdentity {
15215 module_id: "fed".to_string(),
15216 launch_nonce: "fed-nonce".to_string(),
15217 }),
15218 Some(package),
15219 ),
15220 )
15221 .await
15222 .unwrap()
15223 });
15224
15225 let bind_frame = target_rx.recv().await.unwrap();
15226 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15227 let ModuleControlRequest::RouteBind {
15228 admission_facts, ..
15229 } = bind
15230 else {
15231 panic!("{corpus_id}: expected route.bind")
15232 };
15233 assert_eq!(
15234 admission_facts,
15235 Some(expected),
15236 "{corpus_id}: relay must not add, drop or reshape any field"
15237 );
15238
15239 handler
15240 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15241 .await
15242 .unwrap();
15243 route_task.await.unwrap();
15244 }
15245 }
15246
15247 #[tokio::test]
15248 async fn route_bind_never_relays_the_openers_build_class() {
15249 let registry = Arc::new(Registry::default());
15250 let forwarding = Arc::new(ForwardingTable::default());
15251 let handler =
15252 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15253 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
15254 hello_via_sink(
15255 &handler,
15256 &target_ctx,
15257 &mut target_rx,
15258 hello_frame("target", PROTOCOL_VERSION, 1),
15259 )
15260 .await;
15261
15262 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
15263 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
15264 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15265 busy_refusals: None,
15266 target: RouteTarget::ToolProvider {
15267 module_id: "target".to_string(),
15268 },
15269 identity,
15270 consumer_identity: None,
15271 consumer_capabilities: None,
15272 role_versions: None,
15273 admission_facts: None,
15274 scope: None,
15275 })
15276 .unwrap();
15277 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
15278
15279 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
15280 let route_handler = handler.clone();
15281 let route_task = tokio::spawn(async move {
15282 route_handler
15283 .handle_control_frame(&client_ctx, open)
15284 .await
15285 .unwrap()
15286 });
15287 let bind_frame = target_rx.recv().await.unwrap();
15288 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15289 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
15290 panic!("expected route.bind")
15291 };
15292 assert_eq!(
15293 identity.build_class, None,
15294 "route.bind carried the build class"
15295 );
15296 assert!(
15297 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
15298 "the route.bind bytes still name build_class"
15299 );
15300 handler
15301 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15302 .await
15303 .unwrap();
15304 route_task.await.unwrap();
15305 }
15306
15307 #[tokio::test]
15308 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
15309 let registry = Arc::new(Registry::default());
15310 let forwarding = Arc::new(ForwardingTable::default());
15311 let supervisor = SupervisorHandle::new();
15312 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15313 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
15314 let handler =
15315 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15316 .with_supervisor(supervisor)
15317 .with_admission_facts_config(
15318 Some("fed".to_string()),
15319 Some(vec!["target".to_string()]),
15320 );
15321
15322 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
15323 hello_via_sink(
15324 &handler,
15325 &target_ctx,
15326 &mut target_rx,
15327 hello_frame("target", PROTOCOL_VERSION, 1),
15328 )
15329 .await;
15330 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
15331 hello_via_sink(
15332 &handler,
15333 &other_ctx,
15334 &mut other_rx,
15335 hello_frame("other", PROTOCOL_VERSION, 2),
15336 )
15337 .await;
15338
15339 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
15340 let expected_facts = facts.clone();
15341 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
15342 let route_handler = handler.clone();
15343 let route_task = tokio::spawn(async move {
15344 route_handler
15345 .handle_control_frame(
15346 &client_ctx,
15347 route_open_frame_with_admission_facts(
15348 10,
15349 "target",
15350 unique_project_root("admission-facts"),
15351 Some(subc_control::ConsumerIdentity {
15352 module_id: "fed".to_string(),
15353 launch_nonce: "fed-nonce".to_string(),
15354 }),
15355 Some(facts.clone()),
15356 ),
15357 )
15358 .await
15359 .unwrap()
15360 });
15361 let bind_frame = target_rx.recv().await.unwrap();
15362 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15363 let ModuleControlRequest::RouteBind {
15364 admission_facts, ..
15365 } = bind
15366 else {
15367 panic!("expected route.bind")
15368 };
15369 assert_eq!(admission_facts, Some(expected_facts));
15370 handler
15371 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15372 .await
15373 .unwrap();
15374 assert!(route_task.await.unwrap().is_empty());
15375 assert!(matches!(
15376 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
15377 .unwrap(),
15378 ClientControlResponse::RouteOpen { .. }
15379 ));
15380
15381 let direct = handler
15382 .handle_control_frame(
15383 &route_ctx(ConnectionId::new(73)).0,
15384 route_open_frame_with_admission_facts(
15385 11,
15386 "target",
15387 unique_project_root("admission-facts"),
15388 None,
15389 Some(json!({"x": 1})),
15390 ),
15391 )
15392 .await
15393 .unwrap();
15394 assert_eq!(
15395 parse_error(&direct[0])["code"],
15396 "admission_facts_not_permitted"
15397 );
15398
15399 let different_reserved = handler
15400 .handle_control_frame(
15401 &route_ctx(ConnectionId::new(77)).0,
15402 route_open_frame_with_admission_facts(
15403 15,
15404 "target",
15405 unique_project_root("admission-facts"),
15406 Some(subc_control::ConsumerIdentity {
15407 module_id: "other".to_string(),
15408 launch_nonce: "other-nonce".to_string(),
15409 }),
15410 Some(json!({"x": 1})),
15411 ),
15412 )
15413 .await
15414 .unwrap();
15415 assert_eq!(
15416 parse_error(&different_reserved[0])["code"],
15417 "admission_facts_not_permitted"
15418 );
15419
15420 let other_target = handler
15421 .handle_control_frame(
15422 &route_ctx(ConnectionId::new(74)).0,
15423 route_open_frame_with_admission_facts(
15424 12,
15425 "other",
15426 unique_project_root("admission-facts"),
15427 Some(subc_control::ConsumerIdentity {
15428 module_id: "fed".to_string(),
15429 launch_nonce: "fed-nonce".to_string(),
15430 }),
15431 Some(json!({"x": 1})),
15432 ),
15433 )
15434 .await
15435 .unwrap();
15436 assert_eq!(
15437 parse_error(&other_target[0])["code"],
15438 "admission_facts_target_not_allowed"
15439 );
15440
15441 let nonexistent = handler
15442 .handle_control_frame(
15443 &route_ctx(ConnectionId::new(75)).0,
15444 route_open_frame_with_admission_facts(
15445 13,
15446 "missing",
15447 unique_project_root("admission-facts"),
15448 None,
15449 Some(json!({"x": 1})),
15450 ),
15451 )
15452 .await
15453 .unwrap();
15454 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
15455
15456 let described = handler
15457 .handle_control_frame(
15458 &route_ctx(ConnectionId::new(76)).0,
15459 Frame::build(
15460 FrameType::Request,
15461 control_flags(),
15462 0,
15463 0,
15464 14,
15465 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15466 )
15467 .unwrap(),
15468 )
15469 .await
15470 .unwrap();
15471 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15472 serde_json::from_slice(&described[0].body).unwrap()
15473 else {
15474 panic!("expected server.describe response")
15475 };
15476 assert!(capabilities
15477 .iter()
15478 .any(|cap| cap == "admission_facts_relay_v1"));
15479 }
15480
15481 #[tokio::test]
15482 async fn admission_facts_without_configured_carrier_are_rejected() {
15483 let registry = Arc::new(Registry::default());
15484 let forwarding = Arc::new(ForwardingTable::default());
15485 let handler = ControlHandler::with_forwarding(registry, forwarding);
15486 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
15487 hello_via_sink(
15488 &handler,
15489 &target_ctx,
15490 &mut target_rx,
15491 hello_frame("target", PROTOCOL_VERSION, 1),
15492 )
15493 .await;
15494
15495 let responses = handler
15496 .handle_control_frame(
15497 &route_ctx(ConnectionId::new(79)).0,
15498 route_open_frame_with_admission_facts(
15499 16,
15500 "target",
15501 unique_project_root("admission-facts"),
15502 None,
15503 Some(json!({"x": 1})),
15504 ),
15505 )
15506 .await
15507 .unwrap();
15508 assert_eq!(
15509 parse_error(&responses[0])["code"],
15510 "admission_facts_not_permitted"
15511 );
15512 }
15513
15514 #[tokio::test]
15515 async fn route_open_relays_consumer_capabilities_verbatim() {
15516 let registry = Arc::new(Registry::default());
15517 let forwarding = Arc::new(ForwardingTable::default());
15518 let handler =
15519 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15520 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
15521 hello_via_sink(
15522 &handler,
15523 &module_ctx,
15524 &mut module_rx,
15525 hello_frame("aft", PROTOCOL_VERSION, 7),
15526 )
15527 .await;
15528
15529 let expected = vec!["elicitation".to_string(), "roots".to_string()];
15530 let expected_for_request = expected.clone();
15531 let project_root = unique_project_root("consumer-capabilities-present");
15532 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
15533 let route_handler = handler.clone();
15534 let route_task = tokio::spawn(async move {
15535 route_handler
15536 .handle_control_frame(
15537 &client_ctx,
15538 route_open_frame_with_consumer_capabilities(
15539 401,
15540 "aft",
15541 project_root,
15542 Some(expected_for_request),
15543 ),
15544 )
15545 .await
15546 .unwrap()
15547 });
15548 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15549 .await
15550 .unwrap()
15551 .unwrap();
15552 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15553 let ModuleControlRequest::RouteBind {
15554 consumer_capabilities,
15555 ..
15556 } = bind
15557 else {
15558 panic!("expected route.bind request, got {bind:?}");
15559 };
15560 assert_eq!(consumer_capabilities, Some(expected.clone()));
15561
15562 handler
15563 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15564 .await
15565 .unwrap();
15566 let route_response = route_task.await.unwrap();
15567 assert!(route_response.is_empty());
15568 let published = client_rx.recv().await.unwrap();
15569 assert!(matches!(
15570 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15571 ClientControlResponse::RouteOpen { .. }
15572 ));
15573 }
15574
15575 #[tokio::test]
15576 async fn route_open_without_consumer_capabilities_relays_none() {
15577 let registry = Arc::new(Registry::default());
15578 let forwarding = Arc::new(ForwardingTable::default());
15579 let handler =
15580 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15581 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
15582 hello_via_sink(
15583 &handler,
15584 &module_ctx,
15585 &mut module_rx,
15586 hello_frame("aft", PROTOCOL_VERSION, 7),
15587 )
15588 .await;
15589
15590 let project_root = unique_project_root("consumer-capabilities-absent");
15591 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
15592 let route_handler = handler.clone();
15593 let route_task = tokio::spawn(async move {
15594 route_handler
15595 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
15596 .await
15597 .unwrap()
15598 });
15599 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15600 .await
15601 .unwrap()
15602 .unwrap();
15603 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15604 let ModuleControlRequest::RouteBind {
15605 consumer_capabilities,
15606 ..
15607 } = bind
15608 else {
15609 panic!("expected route.bind request, got {bind:?}");
15610 };
15611 assert_eq!(consumer_capabilities, None);
15612
15613 handler
15614 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15615 .await
15616 .unwrap();
15617 let route_response = route_task.await.unwrap();
15618 assert!(route_response.is_empty());
15619 let published = client_rx.recv().await.unwrap();
15620 assert!(matches!(
15621 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15622 ClientControlResponse::RouteOpen { .. }
15623 ));
15624 }
15625
15626 async fn bind_role_versions_for(
15630 sent: Option<BTreeMap<String, String>>,
15631 connection: u64,
15632 ) -> Option<BTreeMap<String, String>> {
15633 let registry = Arc::new(Registry::default());
15634 let forwarding = Arc::new(ForwardingTable::default());
15635 let handler =
15636 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15637 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
15638 hello_via_sink(
15639 &handler,
15640 &module_ctx,
15641 &mut module_rx,
15642 hello_frame("aft", PROTOCOL_VERSION, 7),
15643 )
15644 .await;
15645 let project_root = unique_project_root("role-versions");
15646 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
15647 let route_handler = handler.clone();
15648 let route_task = tokio::spawn(async move {
15649 route_handler
15650 .handle_control_frame(
15651 &client_ctx,
15652 route_open_frame_with_role_versions(403, "aft", project_root, sent),
15653 )
15654 .await
15655 .unwrap()
15656 });
15657 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15658 .await
15659 .expect("a well-formed route.open reaches the module as a bind")
15660 .unwrap();
15661 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15662 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
15663 panic!("expected route.bind request, got {bind:?}");
15664 };
15665 handler
15666 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15667 .await
15668 .unwrap();
15669 assert!(route_task.await.unwrap().is_empty());
15670 let published = client_rx.recv().await.unwrap();
15671 assert!(matches!(
15672 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15673 ClientControlResponse::RouteOpen { .. }
15674 ));
15675 role_versions
15676 }
15677
15678 #[tokio::test]
15679 async fn route_open_relays_role_versions_verbatim() {
15680 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
15681 assert_eq!(
15682 bind_role_versions_for(Some(sent.clone()), 141).await,
15683 Some(sent)
15684 );
15685 }
15686
15687 #[tokio::test]
15690 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
15691 assert_eq!(bind_role_versions_for(None, 143).await, None);
15692 assert_eq!(
15693 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
15694 None
15695 );
15696 }
15697
15698 #[tokio::test]
15699 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
15700 let registry = Arc::new(Registry::default());
15701 let forwarding = Arc::new(ForwardingTable::default());
15702 let handler =
15703 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15704 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
15705 hello_via_sink(
15706 &handler,
15707 &module_ctx,
15708 &mut module_rx,
15709 hello_frame("aft", PROTOCOL_VERSION, 7),
15710 )
15711 .await;
15712
15713 let nine: BTreeMap<String, String> = (0..9)
15714 .map(|index| (format!("role-{index}"), "v1".to_string()))
15715 .collect();
15716 for (label, malformed) in [
15717 (
15718 "invalid role name",
15719 role_versions(&[("Tool_Provider", "v1")]),
15720 ),
15721 ("invalid version", role_versions(&[("tool-provider", "v0")])),
15722 ("nine entries", nine),
15723 ] {
15724 let responses = tokio::time::timeout(
15727 Duration::from_secs(1),
15728 handler.handle_control_frame(
15729 &route_ctx(ConnectionId::new(148)).0,
15730 route_open_frame_with_role_versions(
15731 404,
15732 "aft",
15733 unique_project_root("role-versions-malformed"),
15734 Some(malformed),
15735 ),
15736 ),
15737 )
15738 .await
15739 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
15740 .unwrap();
15741 assert_eq!(responses.len(), 1, "{label}");
15742 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
15743 let error = parse_error(&responses[0]);
15744 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
15745 assert_eq!(
15746 error["detail"]["field"], "role_versions",
15747 "{label}: {error}"
15748 );
15749 assert!(
15750 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
15751 "{label}: a malformed declaration is terminal"
15752 );
15753 assert!(
15754 module_rx.try_recv().is_err(),
15755 "{label}: the module must never see a bind"
15756 );
15757 }
15758 }
15759
15760 #[tokio::test]
15764 async fn route_role_versions_capability_is_advertised() {
15765 let handler = ControlHandler::new(Arc::new(Registry::default()));
15766 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
15767 let ack = hello_via_sink(
15768 &handler,
15769 &ctx,
15770 &mut rx,
15771 hello_frame("m", PROTOCOL_VERSION, 1),
15772 )
15773 .await;
15774 let ack = parse_ack(&ack);
15775 assert!(
15776 ack.subc_capabilities
15777 .iter()
15778 .any(|c| c == "route-role-versions/v1"),
15779 "{:?}",
15780 ack.subc_capabilities
15781 );
15782
15783 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15784 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15785 let reply = handler
15786 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
15787 .await
15788 .unwrap()
15789 .pop()
15790 .unwrap();
15791 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15792 serde_json::from_slice(&reply.body).unwrap()
15793 else {
15794 panic!("not a server.describe reply");
15795 };
15796 assert!(
15797 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
15798 "{capabilities:?}"
15799 );
15800 }
15801
15802 #[tokio::test]
15806 async fn catalog_events_capability_is_advertised() {
15807 let handler = ControlHandler::new(Arc::new(Registry::default()));
15808 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15809 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15810 let reply = handler
15811 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
15812 .await
15813 .unwrap()
15814 .pop()
15815 .unwrap();
15816 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15817 serde_json::from_slice(&reply.body).unwrap()
15818 else {
15819 panic!("not a server.describe reply");
15820 };
15821 assert!(
15822 capabilities
15823 .iter()
15824 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
15825 "{capabilities:?}"
15826 );
15827 }
15828
15829 #[tokio::test]
15830 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
15831 let handler = ControlHandler::new(Arc::new(Registry::default()));
15832 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
15833 let ack = hello_via_sink(
15834 &handler,
15835 &module,
15836 &mut module_rx,
15837 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
15838 )
15839 .await;
15840 let ack = parse_ack(&ack);
15841 assert!(
15842 ack.subc_capabilities
15843 .iter()
15844 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15845 "{:?}",
15846 ack.subc_capabilities
15847 );
15848 assert!(ack
15849 .subc_capabilities
15850 .iter()
15851 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15852
15853 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
15854 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15855 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15856 let reply = handler
15857 .handle_control_frame(&client, frame)
15858 .await
15859 .unwrap()
15860 .pop()
15861 .unwrap();
15862 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15863 serde_json::from_slice(&reply.body).unwrap()
15864 else {
15865 panic!("not a server.describe reply");
15866 };
15867 assert!(
15868 capabilities
15869 .iter()
15870 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15871 "{capabilities:?}"
15872 );
15873 assert!(capabilities
15874 .iter()
15875 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15876 }
15877
15878 #[tokio::test]
15879 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
15880 let handler = ControlHandler::new(Arc::new(Registry::default()));
15881 let (module, mut rx) = route_ctx(ConnectionId::new(156));
15882 let ack = hello_via_sink(
15883 &handler,
15884 &module,
15885 &mut rx,
15886 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
15887 )
15888 .await;
15889 assert!(parse_ack(&ack)
15890 .subc_capabilities
15891 .iter()
15892 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15893 assert!(parse_ack(&ack)
15894 .subc_capabilities
15895 .iter()
15896 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15897 let (client, _rx) = route_ctx(ConnectionId::new(157));
15898 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15899 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15900 let reply = handler
15901 .handle_control_frame(&client, frame)
15902 .await
15903 .unwrap()
15904 .pop()
15905 .unwrap();
15906 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15907 serde_json::from_slice(&reply.body).unwrap()
15908 else {
15909 panic!("server.describe reply")
15910 };
15911 assert!(capabilities
15912 .iter()
15913 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15914 assert!(capabilities
15915 .iter()
15916 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15917 }
15918
15919 #[tokio::test]
15920 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
15921 let handler = ControlHandler::new(Arc::new(Registry::default()));
15922 let (module, mut rx) = route_ctx(ConnectionId::new(158));
15923 let ack = hello_via_sink(
15924 &handler,
15925 &module,
15926 &mut rx,
15927 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
15928 )
15929 .await;
15930 assert!(parse_ack(&ack)
15931 .subc_capabilities
15932 .iter()
15933 .any(|cap| cap == "scope-workdir/v1"));
15934 let (client, _rx) = route_ctx(ConnectionId::new(159));
15935 let frame = Frame::build(
15936 FrameType::Request,
15937 control_flags(),
15938 0,
15939 0,
15940 5,
15941 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15942 )
15943 .unwrap();
15944 let reply = handler
15945 .handle_control_frame(&client, frame)
15946 .await
15947 .unwrap()
15948 .remove(0);
15949 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15950 serde_json::from_slice(&reply.body).unwrap()
15951 else {
15952 panic!("server.describe reply")
15953 };
15954 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
15955 }
15956
15957 #[tokio::test]
15958 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
15959 let handler = ControlHandler::new(Arc::new(Registry::default()));
15960 let (module, mut rx) = route_ctx(ConnectionId::new(160));
15961 let ack = hello_via_sink(
15962 &handler,
15963 &module,
15964 &mut rx,
15965 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
15966 )
15967 .await;
15968 assert!(parse_ack(&ack)
15969 .subc_capabilities
15970 .iter()
15971 .any(|cap| cap == "worker-runs/v1"));
15972 let (client, _rx) = route_ctx(ConnectionId::new(161));
15973 let frame = Frame::build(
15974 FrameType::Request,
15975 control_flags(),
15976 0,
15977 0,
15978 5,
15979 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15980 )
15981 .unwrap();
15982 let reply = handler
15983 .handle_control_frame(&client, frame)
15984 .await
15985 .unwrap()
15986 .remove(0);
15987 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15988 serde_json::from_slice(&reply.body).unwrap()
15989 else {
15990 panic!("server.describe reply")
15991 };
15992 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
15993 }
15994
15995 #[tokio::test]
15996 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
15997 let handler = ControlHandler::new(Arc::new(Registry::default()));
15998 let (module, mut rx) = route_ctx(ConnectionId::new(154));
15999 let ack = hello_via_sink(
16000 &handler,
16001 &module,
16002 &mut rx,
16003 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
16004 )
16005 .await;
16006 assert!(parse_ack(&ack)
16007 .subc_capabilities
16008 .iter()
16009 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16010 let (client, _rx) = route_ctx(ConnectionId::new(155));
16011 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16012 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16013 let reply = handler
16014 .handle_control_frame(&client, frame)
16015 .await
16016 .unwrap()
16017 .pop()
16018 .unwrap();
16019 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16020 serde_json::from_slice(&reply.body).unwrap()
16021 else {
16022 panic!("not a server.describe reply")
16023 };
16024 assert!(capabilities
16025 .iter()
16026 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16027 }
16028
16029 #[tokio::test]
16030 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
16031 let registry = Arc::new(Registry::default());
16032 let forwarding = Arc::new(ForwardingTable::default());
16033 let handler =
16034 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16035 .with_health_probe_timeout(Duration::from_secs(5));
16036 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
16037 hello_via_sink(
16038 &handler,
16039 &module_ctx,
16040 &mut module_rx,
16041 non_routable_hello_frame_with_control_ops(
16042 "mcp",
16043 300,
16044 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16045 ),
16046 )
16047 .await;
16048 assert!(registry
16049 .get_module("mcp")
16050 .unwrap()
16051 .unwrap()
16052 .manifest
16053 .provides
16054 .is_empty());
16055
16056 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
16057 let route_response = handler
16058 .handle_control_frame(
16059 &route_client_ctx,
16060 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
16061 )
16062 .await
16063 .unwrap();
16064 assert_eq!(route_response[0].header.ty, FrameType::Error);
16065 assert_eq!(
16066 parse_error(&route_response[0])["code"],
16067 "target_unavailable"
16068 );
16069 assert!(parse_error(&route_response[0])["message"]
16070 .as_str()
16071 .unwrap()
16072 .contains("does not provide the requested target"));
16073 assert!(module_rx.try_recv().is_err());
16074
16075 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
16076 let health_handler = handler.clone();
16077 let health_task = tokio::spawn(async move {
16078 health_handler
16079 .handle_control_frame(
16080 &health_client_ctx,
16081 supervisor_health_probe_frame(302, "mcp"),
16082 )
16083 .await
16084 .unwrap()
16085 });
16086 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16087 .await
16088 .unwrap()
16089 .unwrap();
16090 assert_eq!(
16091 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
16092 ModuleControlRequest::HealthCheck {}
16093 );
16094 handler
16095 .handle_control_frame(
16096 &module_ctx,
16097 health_response(health_frame.header.corr, HealthStatus::Ok),
16098 )
16099 .await
16100 .unwrap();
16101 let health_response = health_task.await.unwrap();
16102 assert_eq!(health_response[0].header.ty, FrameType::Response);
16103 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
16104 ClientControlResponse::SupervisorHealthProbe {
16105 module_id, status, ..
16106 } => {
16107 assert_eq!(module_id, "mcp");
16108 assert_eq!(status, HealthStatus::Ok);
16109 }
16110 other => panic!("unexpected health response: {other:?}"),
16111 }
16112
16113 forwarding
16118 .cleanup_connection(module_ctx.connection_id)
16119 .unwrap();
16120 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
16121 let cleanup_response = tokio::time::timeout(
16122 Duration::from_millis(200),
16123 handler.handle_control_frame(
16124 &cleanup_probe_ctx,
16125 supervisor_health_probe_frame(303, "mcp"),
16126 ),
16127 )
16128 .await
16129 .expect("probe should fail immediately when the control lane is gone")
16130 .unwrap();
16131 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
16132 assert_eq!(
16133 parse_error(&cleanup_response[0])["code"],
16134 "target_unavailable"
16135 );
16136 assert!(parse_error(&cleanup_response[0])["message"]
16137 .as_str()
16138 .unwrap()
16139 .contains("no module connection"));
16140
16141 handler
16142 .cleanup_connection(module_ctx.connection_id)
16143 .unwrap();
16144 }
16145
16146 #[tokio::test]
16147 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
16148 let registry = Arc::new(Registry::default());
16149 let supervisor_handle = SupervisorHandle::new();
16150 let supervisor =
16151 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
16152 .with_handle(supervisor_handle.clone())
16153 .with_connection_file_path(
16154 std::env::temp_dir()
16155 .join(format!("subc-route-open-warming-{}", std::process::id())),
16156 );
16157 let module = supervisor
16158 .supervise_configured(
16159 ModuleSpec {
16160 script: None,
16161 carry_containment: false,
16162 module_id: "warming".to_string(),
16163 program: fake_aft_stub_path(),
16164 args: Vec::new(),
16165 env: Vec::new(),
16166 reserved: false,
16167 reserved_prefixes: Vec::new(),
16168 protocol: ModuleProtocol::Subc,
16169 overlap: Default::default(),
16170 },
16171 true,
16172 )
16173 .unwrap();
16174 assert_eq!(module.state().unwrap(), ModuleState::Running);
16175
16176 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16177 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
16178 let response = handler
16179 .handle_control_frame(
16180 &ctx,
16181 route_open_frame(304, "warming", unique_project_root("warming")),
16182 )
16183 .await
16184 .unwrap();
16185 module.stop().await.unwrap();
16186
16187 assert_eq!(response[0].header.ty, FrameType::Error);
16188 let error = parse_error(&response[0]);
16189 assert_eq!(error["code"], "module_warming");
16190 assert!(error["message"]
16191 .as_str()
16192 .unwrap()
16193 .contains("state=running, enabled=true, live=false"));
16194 }
16195
16196 #[test]
16197 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
16198 let handler = ControlHandler::new(Arc::new(Registry::default()));
16199 let capture = EventCapture::default();
16200 let _subscriber =
16201 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16202 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
16203 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
16204 let pending = (0..limit).collect::<Vec<_>>();
16205 let response = handler
16206 .route_open_capacity_refusal(
16207 &ctx,
16208 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
16209 "busy",
16210 pending.len(),
16211 limit,
16212 )
16213 .unwrap();
16214 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16215 let event = capture
16216 .events()
16217 .into_iter()
16218 .find(|event| {
16219 event.target == "control"
16220 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
16221 })
16222 .expect("connection admission refusal event");
16223 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16224 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16225 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
16228 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
16229 }
16230
16231 #[test]
16232 fn route_open_target_cap_logs_admission_reason_and_capacity() {
16233 let handler = ControlHandler::new(Arc::new(Registry::default()));
16234 let capture = EventCapture::default();
16235 let _subscriber =
16236 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16237 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
16238 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
16239 let guards = (0..limit)
16240 .map(|_| {
16241 handler
16242 .route_bind_concurrency
16243 .try_admit("busy", limit)
16244 .unwrap()
16245 })
16246 .collect::<Vec<_>>();
16247 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
16248 Err(in_flight) => in_flight,
16249 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
16250 };
16251 let response = handler
16252 .route_open_target_capacity_refusal(
16253 &ctx,
16254 &RouteOpenCaller::unknown(),
16255 &route_open_frame(397, "busy", unique_project_root("target-cap")),
16256 "busy",
16257 in_flight,
16258 )
16259 .unwrap();
16260 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16261 let event = capture
16262 .events()
16263 .into_iter()
16264 .find(|event| {
16265 event.target == "control"
16266 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
16267 })
16268 .expect("target admission refusal event");
16269 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16270 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16271 drop(guards);
16272 }
16273
16274 #[cfg(unix)]
16275 #[tokio::test(start_paused = true)]
16276 async fn supervisor_list_shows_a_failed_modules_next_retry() {
16277 let registry = Arc::new(Registry::default());
16278 let handle = SupervisorHandle::new();
16279 let supervisor =
16280 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
16281 .with_handle(handle.clone());
16282 let module = supervisor
16283 .spawn(crate::ModuleSpec {
16284 script: None,
16285 carry_containment: false,
16286 module_id: "retry-list".into(),
16287 program: PathBuf::from("/bin/sh"),
16288 args: vec!["-c".into(), "exit 17".into()],
16289 env: vec![],
16290 reserved: false,
16291 reserved_prefixes: vec![],
16292 protocol: ModuleProtocol::None,
16293 overlap: Default::default(),
16294 })
16295 .unwrap();
16296 let deadline = std::time::Instant::now() + Duration::from_secs(10);
16297 while module.state().unwrap() != crate::ModuleState::Failed {
16298 assert!(
16299 std::time::Instant::now() < deadline,
16300 "waiting for budget failure"
16301 );
16302 tokio::task::yield_now().await;
16303 }
16304 let handler = ControlHandler::new(registry).with_supervisor(handle);
16305 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16306 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16307 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16308 assert_eq!(body["modules"][0]["state"], "failed");
16309 let retry = body["modules"][0]["next_retry_at_ms"]
16310 .as_u64()
16311 .expect("scheduled retry on the wire");
16312 let now = std::time::SystemTime::now()
16313 .duration_since(std::time::UNIX_EPOCH)
16314 .unwrap()
16315 .as_millis() as u64;
16316 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
16317 module.set_enabled(false).await.unwrap();
16318 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16319 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16320 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16321 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
16322 }
16323
16324 #[tokio::test]
16325 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
16326 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
16327 let supervisor =
16328 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
16329 let module = supervisor
16330 .spawn(crate::ModuleSpec {
16331 script: None,
16332 carry_containment: false,
16333 module_id: "pid-read-budget".into(),
16334 program: std::env::current_exe().unwrap(),
16335 args: vec![
16336 "--exact".into(),
16337 "control::tests::pid_resource_read_fixture".into(),
16338 ],
16339 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16340 .into_iter()
16341 .map(|key| (key.into(), dir.display().to_string()))
16342 .chain(std::iter::once((
16343 "SUBC_SUBTREE_PID_FIXTURE".into(),
16344 "1".into(),
16345 )))
16346 .collect(),
16347 reserved: false,
16348 reserved_prefixes: Vec::new(),
16349 protocol: ModuleProtocol::None,
16350 overlap: Default::default(),
16351 })
16352 .unwrap();
16353 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16354 while !matches!(
16355 module.child_resource_usage(),
16356 subc_control::ChildResourceUsage::Measured(_)
16357 ) {
16358 assert!(
16359 tokio::time::Instant::now() < ready_deadline,
16360 "fixture process was not measurable"
16361 );
16362 tokio::time::sleep(Duration::from_millis(10)).await;
16363 }
16364 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
16365 let reading = {
16366 let _guard = crate::subtree_resources::fault_read_guard().await;
16367 crate::subtree_resources::read_process(module.clone()).await
16368 };
16369 stop_resource_fixture(&module, &dir).await;
16370 assert!(
16371 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
16372 "{reading:?}"
16373 );
16374 }
16375
16376 #[test]
16377 fn pid_resource_read_fixture() {
16378 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
16379 let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
16380 let deadline = std::time::Instant::now() + Duration::from_secs(60);
16381 while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
16382 std::thread::sleep(Duration::from_millis(5));
16383 }
16384 std::fs::write(dir.join("module-stopped"), "").unwrap();
16385 }
16386 }
16387
16388 async fn stop_resource_fixture(
16389 module: &crate::supervise::SupervisedModule,
16390 dir: &std::path::Path,
16391 ) {
16392 let stopping = module.stop();
16393 tokio::pin!(stopping);
16394 tokio::select! {
16395 result = &mut stopping => result.unwrap(),
16396 () = async {
16397 while module.state().unwrap() != crate::supervise::ModuleState::Draining {
16398 tokio::time::sleep(Duration::from_millis(5)).await;
16399 }
16400 std::fs::write(dir.join("stop-module"), "").unwrap();
16401 } => stopping.await.unwrap(),
16402 }
16403 #[cfg(windows)]
16404 assert!(
16405 dir.join("module-stopped").exists(),
16406 "the resource fixture must exit cooperatively, not wait for the drain kill"
16407 );
16408 assert_eq!(
16409 module.state().unwrap(),
16410 crate::supervise::ModuleState::Stopped
16411 );
16412 }
16413
16414 #[tokio::test]
16415 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
16416 let registry = Arc::new(Registry::default());
16417 let handle = SupervisorHandle::new();
16418 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16419 .with_handle(handle.clone());
16420 supervisor
16421 .supervise_configured(
16422 crate::ModuleSpec {
16423 script: None,
16424 module_id: "disabled-resource-reading".into(),
16425 program: std::env::current_exe().unwrap(),
16426 args: Vec::new(),
16427 env: Vec::new(),
16428 reserved: false,
16429 reserved_prefixes: Vec::new(),
16430 protocol: ModuleProtocol::None,
16431 overlap: Default::default(),
16432 carry_containment: false,
16433 },
16434 false,
16435 )
16436 .unwrap();
16437 let handler = ControlHandler::new(registry).with_supervisor(handle);
16438 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16439 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16440 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16441 assert!(body["modules"][0].get("tree").is_none());
16442 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
16443 }
16444
16445 #[tokio::test]
16446 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
16447 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
16448 let registry = Arc::new(Registry::default());
16449 let handle = SupervisorHandle::new();
16450 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16451 .with_handle(handle.clone());
16452 let module = supervisor
16453 .spawn(crate::ModuleSpec {
16454 script: None,
16455 carry_containment: false,
16456 module_id: "list-process-priority".into(),
16457 program: std::env::current_exe().unwrap(),
16458 args: vec![
16459 "--exact".into(),
16460 "control::tests::pid_resource_read_fixture".into(),
16461 ],
16462 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16463 .into_iter()
16464 .map(|key| (key.into(), dir.display().to_string()))
16465 .chain(std::iter::once((
16466 "SUBC_SUBTREE_PID_FIXTURE".into(),
16467 "1".into(),
16468 )))
16469 .collect(),
16470 reserved: false,
16471 reserved_prefixes: Vec::new(),
16472 protocol: ModuleProtocol::None,
16473 overlap: Default::default(),
16474 })
16475 .unwrap();
16476 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16477 loop {
16478 let image = module.running_image_agreement().await;
16479 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
16480 break;
16481 }
16482 assert!(
16483 tokio::time::Instant::now() < ready_deadline,
16484 "the fixture process never became observable: {image:?}"
16485 );
16486 tokio::time::sleep(Duration::from_millis(10)).await;
16487 }
16488 let guard = crate::subtree_resources::fault_read_guard().await;
16489 let (target, release, _) = crate::subtree_resources::held();
16490 module.set_subtree_target_for_test(target);
16491 let handler = ControlHandler::new(registry).with_supervisor(handle);
16492 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16493 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16494 drop(release);
16495 drop(guard);
16496 stop_resource_fixture(&module, &dir).await;
16497 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16498 assert_eq!(
16499 body["modules"][0]["resources"]["status"], "measured",
16500 "the optional tree timeout must not discard the process reading: {body}"
16501 );
16502 assert!(body["modules"][0].get("tree").is_none());
16503 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
16504 assert_eq!(
16505 body["modules"][0]["pending_reload"]["image"]["status"], "match",
16506 "the tree timeout must not hide a completed image observation"
16507 );
16508 }
16509
16510 #[cfg(any(target_os = "linux", target_os = "macos"))]
16511 #[tokio::test]
16512 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
16513 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
16514 .with_daemon_provenance(
16515 std::process::id(),
16516 0,
16517 Some(std::env::current_exe().unwrap()),
16518 None,
16519 None,
16520 );
16521 let (probe, release, began) = ExecutableIdentityProbe::held();
16522 handler.daemon_provenance.probe = probe;
16523 let observe = |handler: ControlHandler, corr| {
16524 tokio::spawn(async move {
16525 let (ctx, _) = route_ctx(ConnectionId::new(corr));
16526 let frame =
16527 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
16528 .unwrap();
16529 handler
16530 .handle_supervisor_provenance(&ctx, frame, None)
16531 .await
16532 .unwrap()
16533 })
16534 };
16535 let explicit = observe(handler.clone(), 114);
16536 began.await.unwrap();
16537 let concurrent = observe(handler.clone(), 115);
16538 tokio::time::sleep(Duration::from_millis(300)).await;
16541 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16542 let list = tokio::time::timeout(
16543 Duration::from_secs(1),
16544 handler.handle_supervisor_list(frame),
16545 )
16546 .await;
16547 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
16548 drop(release);
16549 let replies = explicit.await.unwrap();
16550 let concurrent_replies = concurrent.await.unwrap();
16551 assert!(
16552 explicit_was_pending,
16553 "explicit provenance must await the full observation"
16554 );
16555 let list = list
16556 .expect("a concurrent explicit probe must not stall list")
16557 .unwrap();
16558 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
16559 assert_eq!(list_body["op"], "supervisor.list");
16560 for replies in [replies, concurrent_replies] {
16561 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16562 assert_eq!(
16563 body["daemon"]["daemon_observed"]["running_image"]["status"],
16564 "match"
16565 );
16566 }
16567 }
16568
16569 #[tokio::test]
16570 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
16571 let registry = Arc::new(Registry::default());
16572 let handle = SupervisorHandle::new();
16573 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16574 .with_handle(handle.clone());
16575 let module = supervisor
16576 .supervise_configured(
16577 crate::ModuleSpec {
16578 script: None,
16579 carry_containment: false,
16580 module_id: "memory-read".into(),
16581 program: PathBuf::from("disabled-module"),
16582 args: Vec::new(),
16583 env: Vec::new(),
16584 reserved: false,
16585 reserved_prefixes: Vec::new(),
16586 protocol: ModuleProtocol::Subc,
16587 overlap: Default::default(),
16588 },
16589 true,
16590 )
16591 .unwrap();
16592 let _guard = crate::subtree_resources::fault_read_guard().await;
16593 let (target, release, began) = crate::subtree_resources::held();
16594 let stalled_target = target.clone();
16597 let stalled = tokio::task::spawn_blocking(move || {
16598 stalled_target.tree(&subc_os::ProcessTable::default())
16599 });
16600 began.await.unwrap();
16601 module.set_subtree_target_for_test(target);
16602 let handler = ControlHandler::new(registry).with_supervisor(handle);
16603 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
16604 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16605 let replies = handler
16606 .handle_client_control_request(
16607 &ctx,
16608 frame,
16609 ClientControlRequest::SupervisorList {},
16610 &mut None,
16611 )
16612 .await
16613 .unwrap();
16614 assert!(
16615 replies.is_empty(),
16616 "list reply must be detached from the connection reader"
16617 );
16618 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16619 let replies = handler
16620 .handle_client_control_request(
16621 &ctx,
16622 frame,
16623 ClientControlRequest::SupervisorSpawnSnapshot {},
16624 &mut None,
16625 )
16626 .await
16627 .unwrap();
16628 assert_eq!(
16629 replies.len(),
16630 1,
16631 "another request must complete while the subtree read stalls"
16632 );
16633 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16634 .await
16635 .unwrap()
16636 .unwrap();
16637 release.send(()).unwrap();
16638 drop(release);
16639 stalled.await.unwrap();
16640 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
16641 assert!(
16642 body["modules"][0].get("tree").is_none(),
16643 "an unknown tree reading must be omitted after the bounded read expires"
16644 );
16645 handler
16649 .resource_tracker
16650 .read(handler.supervisor.list(), false, "released_tree_fixture")
16651 .await;
16652 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16653 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16654 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16655 assert_eq!(
16656 body["modules"][0]["tree"]["source"], "process_tree",
16657 "the next request must export the completed refresh"
16658 );
16659 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
16660 }
16661
16662 #[tokio::test]
16663 async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
16664 use subc_control::{
16665 AgentCommandResources, ChildMemoryKind, ChildResourceReading,
16666 ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
16667 TreeReading, TreeSource,
16668 };
16669 let registry = Arc::new(Registry::default());
16670 let handle = SupervisorHandle::new();
16671 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16672 .with_handle(handle.clone());
16673 supervisor
16674 .supervise_configured(
16675 crate::ModuleSpec {
16676 script: None,
16677 carry_containment: false,
16678 module_id: "budget-worker".into(),
16679 program: PathBuf::from("disabled-module"),
16680 args: Vec::new(),
16681 env: Vec::new(),
16682 reserved: false,
16683 reserved_prefixes: Vec::new(),
16684 protocol: ModuleProtocol::Subc,
16685 overlap: Default::default(),
16686 },
16687 true,
16688 )
16689 .unwrap();
16690 let handler = ControlHandler::new(registry).with_supervisor(handle);
16691 let now = u64::try_from(
16692 std::time::SystemTime::now()
16693 .duration_since(std::time::UNIX_EPOCH)
16694 .unwrap()
16695 .as_millis(),
16696 )
16697 .unwrap();
16698 let measured = now - 1_000;
16699 let hour = CpuLastHour::new(100, 60_000);
16700 let daemon = DaemonResources::new(7, 1, "test")
16701 .with_measured_at_ms(Some(measured))
16702 .with_resources(Some(ChildResourceUsage::Measured(
16703 ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
16704 )))
16705 .with_cpu_last_hour(Some(hour.clone()));
16706 let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
16707 .with_measured_at_ms(Some(measured))
16708 .with_cpu_last_hour(Some(hour.clone()));
16709 let tree = TreeReading::new(
16710 150,
16711 50,
16712 ChildMemoryKind::ResidentSet,
16713 1,
16714 TreeSource::ProcessTree,
16715 )
16716 .with_measured_at_ms(Some(measured));
16717 handler.resource_tracker.seed_committed_for_test(
16718 &handler.supervisor.list(),
16719 crate::resource_totals::Readings {
16720 modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
16721 daemon: Some(daemon),
16722 agent_commands: Some(vec![agents]),
16723 ..Default::default()
16724 },
16725 now,
16726 );
16727 list_diagnostics::scope(async {
16728 tokio::time::sleep(Duration::from_millis(110)).await;
16729 assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
16730 let request =
16731 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16732 let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
16733 let ClientControlResponse::SupervisorList {
16734 modules,
16735 daemon,
16736 agent_commands,
16737 ..
16738 } = serde_json::from_slice(&replies[0].body).unwrap()
16739 else {
16740 panic!("expected a list response")
16741 };
16742 assert!(matches!(
16743 modules[0].resources,
16744 Some(ChildResourceUsage::Unavailable {
16745 reason: ChildResourceUnavailableReason::Unreadable
16746 })
16747 ));
16748 assert_eq!(
16749 (
16750 modules[0].tree.as_ref().unwrap().cpu_ms,
16751 modules[0].tree.as_ref().unwrap().measured_at_ms
16752 ),
16753 (150, Some(measured))
16754 );
16755 let daemon = daemon.unwrap();
16756 let agents = agent_commands.unwrap();
16757 assert_eq!(daemon.measured_at_ms, Some(measured));
16758 assert_eq!(
16759 (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
16760 (1, 50, Some(measured))
16761 );
16762 for hour in [
16763 modules[0].cpu_last_hour.as_ref(),
16764 daemon.cpu_last_hour.as_ref(),
16765 agents[0].cpu_last_hour.as_ref(),
16766 ] {
16767 let hour = hour.unwrap();
16768 assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
16769 }
16770 })
16771 .await;
16772 }
16773
16774 #[tokio::test]
16775 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
16776 let registry = Arc::new(Registry::default());
16777 let handle = SupervisorHandle::new();
16778 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16779 .with_handle(handle.clone());
16780 supervisor
16781 .supervise_configured(
16782 crate::ModuleSpec {
16783 script: None,
16784 carry_containment: false,
16785 module_id: "drift".into(),
16786 program: PathBuf::from("disabled-module"),
16787 args: Vec::new(),
16788 env: Vec::new(),
16789 reserved: false,
16790 reserved_prefixes: Vec::new(),
16791 protocol: ModuleProtocol::Subc,
16792 overlap: Default::default(),
16793 },
16794 false,
16795 )
16796 .unwrap();
16797 let handler = ControlHandler::new(registry).with_supervisor(handle);
16798 for configured in [None, Some(false), Some(true)] {
16799 handler.capability_evaluator.configure(
16800 configured.map(|enabled| ("drift".to_string(), enabled)),
16801 BTreeMap::new(),
16802 );
16803 let frame =
16804 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16805 let replies = handler
16806 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
16807 .await
16808 .unwrap();
16809 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16810 assert!(body.get("configured_enabled").is_none());
16811 assert_eq!(body["op"], "supervisor.ack");
16812 let frame =
16813 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16814 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16815 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16816 assert_eq!(
16817 body["modules"][0].get("configured_enabled"),
16818 configured.map(Value::Bool).as_ref()
16819 );
16820 assert!(body["modules"][0].get("tree").is_none());
16821 assert!(body["modules"][0].get("subtree").is_none());
16822 }
16823 }
16824
16825 #[tokio::test(flavor = "current_thread")]
16826 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
16827 let handler = ControlHandler::new(Arc::new(Registry::default()));
16828 let capture = EventCapture::default();
16829 let _subscriber =
16830 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16831 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
16832 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16833
16834 let _ = handler
16835 .handle_client_control_request(
16836 &ctx,
16837 frame,
16838 ClientControlRequest::SupervisorSetEnabled {
16839 module_id: "broca".to_string(),
16840 enabled: false,
16841 },
16842 &mut None,
16843 )
16844 .await
16845 .unwrap();
16846
16847 let events = capture
16848 .events()
16849 .into_iter()
16850 .filter(|event| {
16851 event.target == "control"
16852 && event.fields.get("message").map(String::as_str)
16853 == Some("supervisor request received")
16854 })
16855 .collect::<Vec<_>>();
16856 assert_eq!(events.len(), 1, "one supervisor request log line");
16857 let event = &events[0];
16858 assert_eq!(event.level, tracing::Level::INFO);
16859 assert_eq!(
16860 event.fields.get("op"),
16861 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
16862 );
16863 assert_eq!(
16864 event.fields.get("module_id"),
16865 Some(&"\"broca\"".to_string())
16866 );
16867 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
16868 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
16869 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
16870 }
16871
16872 #[tokio::test(flavor = "current_thread")]
16873 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
16874 let registry = Arc::new(Registry::default());
16875 let module_connection = ConnectionId::new(303);
16876 registry
16877 .register_with_control_ops(
16878 manifest("aft", PROTOCOL_VERSION),
16879 PROTOCOL_VERSION,
16880 module_connection,
16881 module_baseline_control_ops(),
16882 )
16883 .unwrap();
16884 let handler = ControlHandler::new(registry);
16885 let capture = EventCapture::default();
16886 let _subscriber =
16887 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16888 let (ctx, _rx) = route_ctx(module_connection);
16889 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16890
16891 let _ = handler
16892 .handle_client_control_request(
16893 &ctx,
16894 frame,
16895 ClientControlRequest::SupervisorSetEnabled {
16896 module_id: "broca".to_string(),
16897 enabled: false,
16898 },
16899 &mut None,
16900 )
16901 .await
16902 .unwrap();
16903
16904 let callers = capture
16905 .events()
16906 .into_iter()
16907 .filter(|event| {
16908 event.fields.get("message").map(String::as_str)
16909 == Some("supervisor request received")
16910 })
16911 .map(|event| event.fields.get("caller").cloned())
16912 .collect::<Vec<_>>();
16913 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
16914 }
16915
16916 #[tokio::test(flavor = "current_thread")]
16917 async fn supervisor_list_does_not_log_request_received() {
16918 let handler = ControlHandler::new(Arc::new(Registry::default()));
16919 let capture = EventCapture::default();
16920 let _subscriber =
16921 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16922 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
16923 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16924
16925 handler
16926 .handle_client_control_request(
16927 &ctx,
16928 frame,
16929 ClientControlRequest::SupervisorList {},
16930 &mut None,
16931 )
16932 .await
16933 .unwrap();
16934
16935 assert!(capture.events().into_iter().all(|event| {
16936 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
16937 }));
16938 }
16939
16940 #[tokio::test(flavor = "current_thread")]
16941 async fn supervisor_rescan_preview_does_not_log_request_received() {
16942 let handler = ControlHandler::new(Arc::new(Registry::default()));
16943 let capture = EventCapture::default();
16944 let _subscriber =
16945 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16946 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
16947 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16948
16949 handler
16950 .handle_client_control_request(
16951 &ctx,
16952 frame,
16953 ClientControlRequest::SupervisorRescan { preview: true },
16954 &mut None,
16955 )
16956 .await
16957 .unwrap();
16958
16959 assert!(capture.events().into_iter().all(|event| {
16960 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
16961 }));
16962 }
16963
16964 struct ApproveOperator;
16965 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
16966 fn prompt(
16967 &self,
16968 _: &str,
16969 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
16970 ) -> crate::operator_confirm::ProviderResult {
16971 crate::operator_confirm::ProviderResult::Approved
16972 }
16973 }
16974
16975 #[tokio::test]
16976 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
16977 assert_eq!(
16978 ControlHandler::default().privileged_operator_prompt,
16979 PrivilegedOperatorPrompt::Log
16980 );
16981 for (initial, changed, running) in [
16982 ("log", "enforce", PrivilegedOperatorPrompt::Log),
16983 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
16984 ] {
16985 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
16986 let path = dir.join("subc.jsonc");
16987 std::fs::write(
16988 &path,
16989 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
16990 )
16991 .unwrap();
16992 let config = crate::daemon_config::load(&path).unwrap().unwrap();
16993 let registry = Arc::new(Registry::default());
16994 let handle = SupervisorHandle::new();
16995 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
16996 .with_handle(handle.clone());
16997 let handler = ControlHandler::new(registry)
16998 .with_supervisor(handle)
16999 .with_privileged_operator_prompt(config.privileged_operator_prompt)
17000 .with_supervisor_rescan(supervisor, &path, config.port);
17001 handler.forwarding.operator_confirms().configure(
17002 Arc::new(ApproveOperator),
17003 crate::operator_confirm::OperatorLimits::default(),
17004 );
17005 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
17006
17007 for (disk_mode, needs_restart) in [
17008 (Some(initial), false),
17009 (Some(changed), true),
17010 (Some(changed), true),
17011 (None, running == PrivilegedOperatorPrompt::Enforce),
17012 (Some(initial), false),
17013 ] {
17014 let mut doc = json!({"version":1});
17015 if let Some(mode) = disk_mode {
17016 doc["privileged_operator_prompt"] = json!(mode);
17017 }
17018 std::fs::write(&path, doc.to_string()).unwrap();
17019 for preview in [true, false] {
17020 let request = ClientControlRequest::SupervisorRescan { preview };
17021 let frame = Frame::build(
17022 FrameType::Request,
17023 control_flags(),
17024 0,
17025 0,
17026 1,
17027 serde_json::to_vec(&request).unwrap(),
17028 )
17029 .unwrap();
17030 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17031 assert_eq!(frames.len(), 1);
17032 let ClientControlResponse::SupervisorRescan { result } =
17033 serde_json::from_slice(&frames[0].body).unwrap()
17034 else {
17035 panic!("expected a rescan response");
17036 };
17037 let expected = if needs_restart {
17038 vec!["privileged_operator_prompt"]
17039 } else {
17040 Vec::new()
17041 };
17042 assert_eq!(result.restart_required, expected);
17043 assert_eq!(handler.privileged_operator_prompt, running);
17044 assert_eq!(
17045 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
17046 running
17047 );
17048 }
17049 }
17050 }
17051 }
17052
17053 #[tokio::test]
17054 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
17055 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
17056 let path = dir.join("subc.jsonc");
17057 let registry = Arc::new(Registry::default());
17058 let handle = SupervisorHandle::new();
17059 let supervisor =
17060 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
17061 let handler = ControlHandler::new(registry)
17062 .with_supervisor(handle)
17063 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
17064 .with_supervisor_rescan(supervisor, &path, None);
17065 handler.forwarding.operator_confirms().configure(
17066 Arc::new(ApproveOperator),
17067 crate::operator_confirm::OperatorLimits::default(),
17068 );
17069 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
17070 for preview in [true, false] {
17071 std::fs::write(
17072 &path,
17073 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
17074 )
17075 .unwrap();
17076 let request = ClientControlRequest::SupervisorRescan { preview };
17077 let frame = Frame::build(
17078 FrameType::Request,
17079 control_flags(),
17080 0,
17081 0,
17082 1,
17083 serde_json::to_vec(&request).unwrap(),
17084 )
17085 .unwrap();
17086 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17087 assert_eq!(frames.len(), 1);
17088 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
17089 assert_eq!(
17090 handler.privileged_operator_prompt,
17091 PrivilegedOperatorPrompt::Enforce
17092 );
17093 }
17094 }
17095
17096 #[tokio::test]
17101 async fn route_open_refusal_names_the_check_that_refused() {
17102 let handler = ControlHandler::new(Arc::new(Registry::default()));
17103 let capture = EventCapture::default();
17104 let _subscriber =
17105 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17106 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17107 let response = handler
17108 .handle_control_frame(
17109 &ctx,
17110 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
17111 )
17112 .await
17113 .unwrap();
17114
17115 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17116 let event = capture
17117 .events()
17118 .into_iter()
17119 .find(|event| {
17120 event.target == "control"
17121 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17122 })
17123 .expect("route.open refusal event");
17124 assert_eq!(
17125 event.fields.get("reason"),
17126 Some(&"\"not_registered\"".to_string())
17127 );
17128 }
17129
17130 #[tokio::test]
17131 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
17132 let registry = Arc::new(Registry::default());
17133 let supervisor_handle = SupervisorHandle::new();
17134 let supervisor =
17135 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17136 .with_handle(supervisor_handle.clone())
17137 .with_connection_file_path(std::env::temp_dir().join(format!(
17138 "subc-route-open-refusal-info-{}",
17139 std::process::id()
17140 )));
17141 let module = supervisor
17142 .supervise_configured(
17143 ModuleSpec {
17144 script: None,
17145 carry_containment: false,
17146 module_id: "warming".to_string(),
17147 program: fake_aft_stub_path(),
17148 args: Vec::new(),
17149 env: Vec::new(),
17150 reserved: false,
17151 reserved_prefixes: Vec::new(),
17152 protocol: ModuleProtocol::Subc,
17153 overlap: Default::default(),
17154 },
17155 true,
17156 )
17157 .unwrap();
17158 assert_eq!(module.state().unwrap(), ModuleState::Running);
17159
17160 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17161 assert!(handler
17162 .counters()
17163 .snapshot()
17164 .get("route_open_refused_by_code")
17165 .is_none());
17166 let capture = EventCapture::default();
17167 let _subscriber =
17168 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17169 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
17170 let response = handler
17171 .handle_control_frame(
17172 &ctx,
17173 route_open_frame(394, "warming", unique_project_root("refusal-info")),
17174 )
17175 .await
17176 .unwrap();
17177 module.stop().await.unwrap();
17178
17179 assert_eq!(parse_error(&response[0])["code"], "module_warming");
17180 let event = capture
17181 .events()
17182 .into_iter()
17183 .find(|event| {
17184 event.target == "control"
17185 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
17186 })
17187 .expect("route.open refusal event");
17188 assert_eq!(
17189 event.fields.get("module_id"),
17190 Some(&"\"warming\"".to_string())
17191 );
17192 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
17193 assert_eq!(
17194 event.fields.get("reason"),
17195 Some(&"\"supervised_not_registered\"".to_string())
17196 );
17197 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
17198 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
17199 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
17200 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
17201 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
17202 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
17203 assert_eq!(
17204 handler.counters().snapshot()["route_open_refused_by_code"],
17205 json!({ "module_warming": 1 })
17206 );
17207 }
17208
17209 const OUTAGE_START: &str = "route.open refusing module: not serving";
17210 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
17211
17212 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
17213 capture
17214 .events()
17215 .into_iter()
17216 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
17217 .collect()
17218 }
17219
17220 fn supervise_stub(
17221 registry: &Arc<Registry>,
17222 module_id: &str,
17223 enabled: bool,
17224 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
17225 let supervisor_handle = SupervisorHandle::new();
17226 let supervisor =
17227 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
17228 .with_handle(supervisor_handle.clone())
17229 .with_connection_file_path(std::env::temp_dir().join(format!(
17230 "subc-route-outage-{module_id}-{}",
17231 std::process::id()
17232 )));
17233 let module = supervisor
17234 .supervise_configured(
17235 ModuleSpec {
17236 script: None,
17237 carry_containment: false,
17238 module_id: module_id.to_string(),
17239 program: fake_aft_stub_path(),
17240 args: Vec::new(),
17241 env: Vec::new(),
17242 reserved: false,
17243 reserved_prefixes: Vec::new(),
17244 protocol: ModuleProtocol::Subc,
17245 overlap: Default::default(),
17246 },
17247 enabled,
17248 )
17249 .unwrap();
17250 (supervisor_handle, module)
17251 }
17252
17253 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
17254 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
17255 module_id: module_id.to_string(),
17256 drain_timeout_ms: Some(50),
17257 })
17258 .unwrap();
17259 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
17260 }
17261
17262 #[test]
17266 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
17267 let registry = Arc::new(Registry::default());
17268 let forwarding = Arc::new(ForwardingTable::default());
17269 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17270 let second = ControlHandler::with_forwarding(registry, forwarding);
17271 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
17272 }
17273
17274 #[tokio::test(flavor = "current_thread")]
17278 async fn route_open_unknown_module_refusals_add_no_outage_state() {
17279 let handler = ControlHandler::new(Arc::new(Registry::default()));
17280 let capture = EventCapture::default();
17281 let _subscriber =
17282 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17283 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
17284 for corr in 0..8 {
17285 let response = handler
17286 .handle_control_frame(
17287 &ctx,
17288 route_open_frame(
17289 380 + corr,
17290 &format!("nobody-{corr}"),
17291 unique_project_root("outage-unknown"),
17292 ),
17293 )
17294 .await
17295 .unwrap();
17296 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17297 }
17298
17299 assert_eq!(handler.route_outages.tracked_module_count(), 0);
17300 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
17301 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
17302 }
17303
17304 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17309 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
17310 let registry = Arc::new(Registry::default());
17311 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
17312 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17313 let capture = EventCapture::default();
17314 let _subscriber =
17315 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17316 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
17317 handler.route_outages.record_accepted("outage-restart");
17320
17321 let response = handler
17322 .handle_control_frame(
17323 &ctx,
17324 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
17325 )
17326 .await
17327 .unwrap();
17328 assert_eq!(response[0].header.ty, FrameType::Error);
17329 let starts = outage_lines(&capture, OUTAGE_START);
17330 assert_eq!(starts.len(), 1, "{starts:?}");
17331 assert_eq!(starts[0].level, tracing::Level::WARN);
17332 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
17333 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
17334 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
17335 handler.route_outages.record_accepted("outage-restart");
17336 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
17337
17338 let restart = handler
17339 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
17340 .await
17341 .unwrap();
17342 assert_eq!(
17343 restart[0].header.ty,
17344 FrameType::Response,
17345 "{:?}",
17346 parse_error(&restart[0])
17347 );
17348 handler
17349 .handle_control_frame(
17350 &ctx,
17351 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
17352 )
17353 .await
17354 .unwrap();
17355 module.stop().await.unwrap();
17356
17357 let starts = outage_lines(&capture, OUTAGE_START);
17358 assert_eq!(starts.len(), 2, "{starts:?}");
17359 assert_eq!(starts[1].level, tracing::Level::INFO);
17360 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
17361 }
17362
17363 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17367 async fn failed_operator_restart_leaves_no_operator_mark() {
17368 let registry = Arc::new(Registry::default());
17369 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
17370 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17371 let capture = EventCapture::default();
17372 let _subscriber =
17373 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17374 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
17375 handler.route_outages.record_accepted("outage-disabled");
17376
17377 let restart = handler
17378 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
17379 .await
17380 .unwrap();
17381 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
17382 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
17383
17384 handler
17385 .handle_control_frame(
17386 &ctx,
17387 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
17388 )
17389 .await
17390 .unwrap();
17391 let starts = outage_lines(&capture, OUTAGE_START);
17392 assert_eq!(starts.len(), 1, "{starts:?}");
17393 assert_eq!(starts[0].level, tracing::Level::WARN);
17394 }
17395
17396 #[tokio::test(flavor = "current_thread")]
17397 async fn route_open_unknown_module_escapes_target_module_id() {
17398 let handler = ControlHandler::new(Arc::new(Registry::default()));
17399 let capture = EventCapture::default();
17400 let _subscriber =
17401 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17402 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
17403 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17404 let response = handler
17405 .handle_control_frame(
17406 &ctx,
17407 route_open_frame(
17408 395,
17409 hostile_module_id,
17410 unique_project_root("hostile-target-module-id"),
17411 ),
17412 )
17413 .await
17414 .unwrap();
17415
17416 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17417 let event = capture
17418 .events()
17419 .into_iter()
17420 .find(|event| {
17421 event.target == "control"
17422 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17423 })
17424 .expect("route.open unknown-module refusal event");
17425 let logged = event.fields.get("module_id").expect("module_id field");
17426 assert!(!logged.bytes().any(|byte| byte < 0x20));
17427 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17428 }
17429
17430 #[tokio::test(flavor = "current_thread")]
17433 async fn route_open_refusal_carries_daemon_origin() {
17434 let handler = ControlHandler::new(Arc::new(Registry::default()));
17435 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17436 let response = handler
17437 .handle_control_frame(
17438 &ctx,
17439 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
17440 )
17441 .await
17442 .unwrap();
17443
17444 assert_eq!(response.len(), 1);
17445 assert_eq!(response[0].header.ty, FrameType::Error);
17446 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17447 assert!(
17448 response[0].header.flags.is_daemon_origin(),
17449 "route.open refusal must carry daemon origin: {:?}",
17450 response[0].header.flags
17451 );
17452 }
17453
17454 #[tokio::test(flavor = "current_thread")]
17458 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
17459 let registry = Arc::new(Registry::default());
17460 let forwarding = Arc::new(ForwardingTable::default());
17461 let handler =
17462 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17463 let module_connection = ConnectionId::new(95);
17464 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17465 hello_via_sink(
17466 &handler,
17467 &module_ctx,
17468 &mut module_rx,
17469 hello_frame("aft", PROTOCOL_VERSION, 395),
17470 )
17471 .await;
17472 forwarding
17473 .begin_module_drain("aft", RouteCloseReason::Restart)
17474 .unwrap()
17475 .expect("the module endpoint drains");
17476
17477 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
17478 let response = handler
17479 .handle_control_frame(
17480 &client_ctx,
17481 route_open_frame(396, "aft", unique_project_root("drain-origin")),
17482 )
17483 .await
17484 .unwrap();
17485
17486 assert_eq!(response.len(), 1);
17487 assert_eq!(response[0].header.ty, FrameType::Error);
17488 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17489 assert!(
17490 response[0].header.flags.is_daemon_origin(),
17491 "drain module_reloading must carry daemon origin: {:?}",
17492 response[0].header.flags
17493 );
17494 }
17495
17496 #[tokio::test(flavor = "current_thread")]
17500 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
17501 let registry = Arc::new(Registry::default());
17502 let forwarding = Arc::new(ForwardingTable::default());
17503 let handler =
17504 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17505 let module_connection = ConnectionId::new(95);
17506 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17507 hello_via_sink(
17508 &handler,
17509 &module_ctx,
17510 &mut module_rx,
17511 hello_frame("aft", PROTOCOL_VERSION, 395),
17512 )
17513 .await;
17514
17515 let client_connection = ConnectionId::new(96);
17516 let (client_ctx, _client_rx) = route_ctx(client_connection);
17517 let (route_task, bind) = relay_route_open(
17518 &handler,
17519 client_connection,
17520 &client_ctx.egress,
17521 &mut module_rx,
17522 396,
17523 "aft",
17524 "spoofed-reloading",
17525 )
17526 .await;
17527 let rejection = Frame::build(
17530 FrameType::Error,
17531 crate::router::daemon_error_flags(),
17532 0,
17533 0,
17534 bind.header.corr,
17535 serde_json::to_vec(&ErrorBody::new(
17536 "module_reloading",
17537 "module refused route.bind",
17538 ))
17539 .unwrap(),
17540 )
17541 .unwrap();
17542 handler
17543 .handle_control_frame(&module_ctx, rejection)
17544 .await
17545 .unwrap();
17546
17547 let response = route_task.await.unwrap();
17548 assert_eq!(response.len(), 1);
17549 assert_eq!(response[0].header.ty, FrameType::Error);
17550 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17551 assert!(
17552 !response[0].header.flags.is_daemon_origin(),
17553 "a module's own rejection must not carry daemon origin: {:?}",
17554 response[0].header.flags
17555 );
17556 }
17557
17558 #[tokio::test(flavor = "current_thread")]
17559 async fn route_open_module_rejection_uses_daemon_counter_key() {
17560 let registry = Arc::new(Registry::default());
17561 let forwarding = Arc::new(ForwardingTable::default());
17562 let handler =
17563 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17564 let module_connection = ConnectionId::new(95);
17565 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17566 hello_via_sink(
17567 &handler,
17568 &module_ctx,
17569 &mut module_rx,
17570 hello_frame("aft", PROTOCOL_VERSION, 395),
17571 )
17572 .await;
17573
17574 let client_connection = ConnectionId::new(96);
17575 let (client_ctx, _client_rx) = route_ctx(client_connection);
17576 let capture = EventCapture::default();
17577 static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
17584 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
17585 let _subscriber =
17586 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17587 let (route_task, bind) = relay_route_open(
17588 &handler,
17589 client_connection,
17590 &client_ctx.egress,
17591 &mut module_rx,
17592 396,
17593 "aft",
17594 "hostile-module-code",
17595 )
17596 .await;
17597 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
17598 let rejection = Frame::build(
17599 FrameType::Error,
17600 control_flags(),
17601 0,
17602 0,
17603 bind.header.corr,
17604 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
17605 )
17606 .unwrap();
17607 handler
17608 .handle_control_frame(&module_ctx, rejection)
17609 .await
17610 .unwrap();
17611
17612 let response = route_task.await.unwrap();
17613 assert_eq!(parse_error(&response[0])["code"], hostile_code);
17614 let counters = handler.counters().snapshot();
17615 assert_eq!(
17616 counters["route_open_refused_by_code"],
17617 json!({ "module_rejected": 1 })
17618 );
17619 assert!(counters["route_open_refused_by_code"]
17620 .get(hostile_code)
17621 .is_none());
17622
17623 let event = capture
17624 .events()
17625 .into_iter()
17626 .find(|event| {
17627 event.target == "control"
17628 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
17629 })
17630 .expect("route.open module-rejection refusal event");
17631 let logged = event.fields.get("module_code").expect("module_code field");
17632 assert!(!logged.bytes().any(|byte| byte < 0x20));
17633 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17634 }
17635
17636 #[tokio::test]
17637 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
17638 let registry = Arc::new(Registry::default());
17639 let supervisor_handle = SupervisorHandle::new();
17640 let missing_program = std::env::temp_dir().join(format!(
17641 "subc-route-open-missing-program-{}",
17642 std::process::id()
17643 ));
17644 let supervisor =
17645 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17646 .with_handle(supervisor_handle.clone());
17647 let module = supervisor
17648 .supervise_configured(
17649 ModuleSpec {
17650 script: None,
17651 carry_containment: false,
17652 module_id: "failed".to_string(),
17653 program: missing_program,
17654 args: Vec::new(),
17655 env: Vec::new(),
17656 reserved: false,
17657 reserved_prefixes: Vec::new(),
17658 protocol: ModuleProtocol::Subc,
17659 overlap: Default::default(),
17660 },
17661 true,
17662 )
17663 .unwrap();
17664 assert_eq!(module.state().unwrap(), ModuleState::Failed);
17665
17666 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17667 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
17668 let response = handler
17669 .handle_control_frame(
17670 &ctx,
17671 route_open_frame(305, "failed", unique_project_root("failed")),
17672 )
17673 .await
17674 .unwrap();
17675
17676 assert_eq!(response[0].header.ty, FrameType::Error);
17677 let error = parse_error(&response[0]);
17678 assert_eq!(error["code"], "target_unavailable");
17679 assert!(error["message"]
17680 .as_str()
17681 .unwrap()
17682 .contains("state=failed, enabled=true, live=false"));
17683 }
17684
17685 #[tokio::test]
17686 async fn route_open_role_mismatch_remains_target_unavailable() {
17687 let registry = Arc::new(Registry::default());
17688 let handler = ControlHandler::new(Arc::clone(®istry));
17689 handler
17690 .handle_control(
17691 ConnectionId::new(41),
17692 non_routable_hello_frame_with_control_ops("health-only", 306, None),
17693 )
17694 .unwrap();
17695
17696 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
17697 let response = handler
17698 .handle_control_frame(
17699 &ctx,
17700 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
17701 )
17702 .await
17703 .unwrap();
17704
17705 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17706 assert!(parse_error(&response[0])["message"]
17707 .as_str()
17708 .unwrap()
17709 .contains("does not provide the requested target"));
17710 }
17711
17712 #[tokio::test]
17713 async fn route_open_inactive_registration_remains_target_unavailable() {
17714 let registry = Arc::new(Registry::default());
17715 let handler = ControlHandler::new(Arc::clone(®istry));
17716 handler
17717 .handle_control(
17718 ConnectionId::new(43),
17719 hello_frame("inactive", PROTOCOL_VERSION, 308),
17720 )
17721 .unwrap();
17722 assert!(registry
17723 .set_module_state_for_test("inactive", ChannelState::Closed)
17724 .unwrap());
17725
17726 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
17727 let response = handler
17728 .handle_control_frame(
17729 &ctx,
17730 route_open_frame(309, "inactive", unique_project_root("inactive")),
17731 )
17732 .await
17733 .unwrap();
17734
17735 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17736 assert!(parse_error(&response[0])["message"]
17737 .as_str()
17738 .unwrap()
17739 .contains("is not active"));
17740 }
17741
17742 #[tokio::test]
17743 async fn late_health_reply_is_recorded_through_the_module_response_path() {
17744 let registry = Arc::new(Registry::default());
17745 let forwarding = Arc::new(ForwardingTable::default());
17746 let supervisor_handle = SupervisorHandle::new();
17747 let supervisor =
17748 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
17749 .with_forwarding(Arc::clone(&forwarding))
17750 .with_handle(supervisor_handle.clone());
17751 let module = supervisor
17752 .supervise_configured(
17753 crate::ModuleSpec {
17754 script: None,
17755 carry_containment: false,
17756 module_id: "late-health-response".to_string(),
17757 program: PathBuf::from("disabled-module"),
17758 args: Vec::new(),
17759 env: Vec::new(),
17760 reserved: false,
17761 reserved_prefixes: Vec::new(),
17762 protocol: ModuleProtocol::Subc,
17763 overlap: Default::default(),
17764 },
17765 false,
17766 )
17767 .unwrap();
17768 let handler =
17769 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17770 .with_supervisor(supervisor_handle);
17771 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
17772 handler
17773 .handle_control_frame(
17774 &module_ctx,
17775 hello_frame_with_control_ops(
17776 "late-health-response",
17777 PROTOCOL_VERSION,
17778 7,
17779 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17780 ),
17781 )
17782 .await
17783 .unwrap();
17784 let probe_started_at = Instant::now() - Duration::from_millis(80);
17785 let pending = forwarding
17786 .begin_health_probe_rpc_for(
17787 "late-health-response",
17788 MODULE_CONTROL_OP_HEALTH_CHECK,
17789 probe_started_at,
17790 Instant::now() - Duration::from_millis(1),
17791 )
17792 .unwrap();
17793 assert!(forwarding
17794 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
17795 .unwrap());
17796
17797 let responses = handler
17798 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
17799 .await
17800 .unwrap();
17801
17802 assert!(responses.is_empty());
17803 let health = module.status().unwrap().health;
17804 assert_eq!(health.late_answer_count, 1);
17805 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
17806 }
17807
17808 #[tokio::test]
17809 async fn health_probe_timeout_and_module_death_are_typed() {
17810 let registry = Arc::new(Registry::default());
17811 let forwarding = Arc::new(ForwardingTable::default());
17812 let handler =
17813 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17814 .with_health_probe_timeout(Duration::from_millis(50));
17815 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
17816 hello_via_sink(
17817 &handler,
17818 &module_ctx,
17819 &mut module_rx,
17820 hello_frame_with_control_ops(
17821 "aft",
17822 PROTOCOL_VERSION,
17823 7,
17824 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17825 ),
17826 )
17827 .await;
17828
17829 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
17830 let responses = handler
17831 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
17832 .await
17833 .unwrap();
17834 assert_eq!(responses[0].header.ty, FrameType::Error);
17835 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
17836 let _ = module_rx.try_recv();
17837
17838 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
17839 let health_handler = handler.clone();
17840 let death_task = tokio::spawn(async move {
17841 health_handler
17842 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
17843 .await
17844 .unwrap()
17845 });
17846 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
17847 .await
17848 .unwrap()
17849 .unwrap();
17850 handler
17851 .cleanup_connection(module_ctx.connection_id)
17852 .unwrap();
17853 let responses = death_task.await.unwrap();
17854 assert_eq!(responses[0].header.ty, FrameType::Error);
17855 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
17856 }
17857
17858 #[test]
17859 fn hello_requires_exact_protocol_version() {
17860 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
17861 let registry = Arc::new(Registry::default());
17862 let handler = ControlHandler::new(Arc::clone(®istry));
17863 let responses = handler
17864 .handle_control(
17865 ConnectionId::new(connection),
17866 hello_frame("aft", offered, 9),
17867 )
17868 .unwrap();
17869
17870 assert_eq!(responses.len(), 1);
17871 assert_eq!(responses[0].header.ty, FrameType::Error);
17872 let error = parse_error(&responses[0]);
17873 assert_eq!(error["code"], "version_unsupported");
17874 assert!(registry.get_module("aft").unwrap().is_none());
17875 assert_eq!(registry.active_registration_count().unwrap(), 0);
17876 }
17877 }
17878
17879 #[test]
17880 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
17881 let registry = Arc::new(Registry::default());
17882 let forwarding = Arc::new(ForwardingTable::default());
17883 let handler =
17884 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17885 let module_connection = ConnectionId::new(301);
17886 let registration = registry
17887 .register_with_control_ops(
17888 manifest("aft-push", PROTOCOL_VERSION),
17889 PROTOCOL_VERSION,
17890 module_connection,
17891 module_baseline_control_ops(),
17892 )
17893 .unwrap();
17894 let (module_tx, _module_rx) = mpsc::channel(8);
17895 let endpoint = forwarding
17896 .register_module_connection(
17897 module_connection,
17898 "aft-push".to_string(),
17899 PROTOCOL_VERSION,
17900 manifest_concurrency(®istration.manifest),
17901 FrameSink::new(module_tx),
17902 )
17903 .unwrap();
17904
17905 let unknown = Frame::build(
17907 FrameType::Push,
17908 control_flags(),
17909 0,
17910 0,
17911 5,
17912 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
17913 )
17914 .unwrap();
17915 let out = handler.handle_status_update(endpoint, unknown).unwrap();
17916 assert!(
17917 out.is_empty(),
17918 "unknown push op must be ignored, got {out:?}"
17919 );
17920
17921 let malformed = Frame::build(
17923 FrameType::Push,
17924 control_flags(),
17925 0,
17926 0,
17927 6,
17928 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
17929 )
17930 .unwrap();
17931 let out = handler.handle_status_update(endpoint, malformed).unwrap();
17932 assert_eq!(out.len(), 1);
17933 assert_eq!(out[0].header.ty, FrameType::Error);
17934 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
17935 }
17936
17937 #[test]
17938 fn hello_rejected_when_connection_already_owns_client_routes() {
17939 let registry = Arc::new(Registry::default());
17940 let forwarding = Arc::new(ForwardingTable::default());
17941 let handler =
17942 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17943 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
17945 let client_connection = ConnectionId::new(202);
17946
17947 let responses = handler
17950 .handle_control(
17951 client_connection,
17952 hello_frame("aft-second", PROTOCOL_VERSION, 9),
17953 )
17954 .unwrap();
17955 assert_eq!(responses[0].header.ty, FrameType::Error);
17956 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
17957 assert!(registry.get_module("aft-second").unwrap().is_none());
17958 }
17959
17960 #[tokio::test]
17961 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
17962 let registry = Arc::new(Registry::default());
17963 let forwarding = Arc::new(ForwardingTable::default());
17964 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
17965 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
17966 hello_via_sink(
17967 &handler,
17968 &module_ctx,
17969 &mut module_rx,
17970 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
17971 )
17972 .await;
17973 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
17974 let pending = forwarding
17975 .begin_route_bind_relay_for_test(
17976 client_ctx.connection_id,
17977 client_ctx.egress.clone(),
17978 2,
17979 "alpha",
17980 )
17981 .unwrap();
17982 forwarding
17983 .complete_pending_relay(
17984 module_ctx.connection_id,
17985 pending.corr,
17986 RouteBindRelayOutcome::Accepted,
17987 )
17988 .unwrap();
17989 client_rx.try_recv().unwrap();
17990 for module_id in ["beta", "alpha"] {
17991 let replies = handler
17992 .handle_control_frame(
17993 &module_ctx,
17994 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
17995 )
17996 .await
17997 .unwrap();
17998 assert_eq!(replies.len(), 1, "second HELLO must be refused");
17999 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
18000 }
18001 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
18002 assert!(registry.get_module("beta").unwrap().is_none());
18003 assert!(matches!(
18004 forwarding
18005 .lookup_data_route(
18006 client_ctx.connection_id,
18007 pending.client_channel,
18008 pending.client_epoch,
18009 )
18010 .unwrap(),
18011 DataRoute::Client(DataRouteState::Bound(_))
18012 ));
18013 assert!(handler
18014 .hello_launch_nonces
18015 .lock()
18016 .unwrap()
18017 .presented(module_ctx.connection_id, Some("alpha-nonce")));
18018 assert!(module_rx.try_recv().is_err());
18019 }
18020
18021 #[test]
18022 fn reserved_module_hello_requires_matching_launch_nonce() {
18023 let registry = Arc::new(Registry::default());
18024 let supervisor = SupervisorHandle::new();
18025 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
18028 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18029
18030 let no_nonce = handler
18032 .handle_control(
18033 ConnectionId::new(1),
18034 hello_frame("vault", PROTOCOL_VERSION, 1),
18035 )
18036 .unwrap();
18037 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
18038 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
18039 assert!(registry.get_module("vault").unwrap().is_none());
18040
18041 let wrong = handler
18043 .handle_control(
18044 ConnectionId::new(2),
18045 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
18046 )
18047 .unwrap();
18048 assert_eq!(wrong[0].header.ty, FrameType::Error);
18049 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
18050 assert!(registry.get_module("vault").unwrap().is_none());
18051
18052 let ok = handler
18054 .handle_control(
18055 ConnectionId::new(3),
18056 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
18057 )
18058 .unwrap();
18059 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
18060 assert!(registry.get_module("vault").unwrap().is_some());
18061 }
18062
18063 #[test]
18064 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
18065 let registry = Arc::new(Registry::default());
18066 let supervisor = SupervisorHandle::new();
18067 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18068 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18069 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18070
18071 let squat = handler
18072 .handle_control(
18073 ConnectionId::new(1),
18074 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
18075 )
18076 .unwrap();
18077 assert_eq!(squat[0].header.ty, FrameType::Error);
18078 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
18079 assert!(parse_error(&squat[0])["message"]
18080 .as_str()
18081 .unwrap()
18082 .contains("fed:"));
18083
18084 let accepted_peer = handler
18085 .handle_control(
18086 ConnectionId::new(2),
18087 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
18088 )
18089 .unwrap();
18090 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
18091
18092 let accepted_short = handler
18093 .handle_control(
18094 ConnectionId::new(3),
18095 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
18096 )
18097 .unwrap();
18098 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
18099
18100 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
18101 let response = handler
18102 .handle_control(
18103 ConnectionId::new(conn),
18104 hello_frame(module_id, PROTOCOL_VERSION, conn),
18105 )
18106 .unwrap();
18107 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
18108 }
18109 }
18110
18111 #[test]
18112 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
18113 let registry = Arc::new(Registry::default());
18114 let supervisor = SupervisorHandle::new();
18115 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18116 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18117 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
18118 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18119
18120 let owner_nonce = handler
18121 .handle_control(
18122 ConnectionId::new(1),
18123 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
18124 )
18125 .unwrap();
18126 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
18127 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
18128 assert!(registry.get_module("fed:special").unwrap().is_none());
18129
18130 let exact_nonce = handler
18131 .handle_control(
18132 ConnectionId::new(2),
18133 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
18134 )
18135 .unwrap();
18136 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
18137 assert!(registry.get_module("fed:special").unwrap().is_some());
18138 }
18139
18140 #[test]
18141 fn non_reserved_module_ignores_launch_nonce() {
18142 let registry = Arc::new(Registry::default());
18143 let handler = ControlHandler::new(Arc::clone(®istry));
18146 let no_nonce = handler
18147 .handle_control(
18148 ConnectionId::new(1),
18149 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
18150 )
18151 .unwrap();
18152 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
18153 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
18154
18155 let echoed_nonce = handler
18156 .handle_control(
18157 ConnectionId::new(2),
18158 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
18159 )
18160 .unwrap();
18161 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
18162 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
18163 }
18164
18165 #[test]
18166 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
18167 let handler = ControlHandler::default();
18168 let conn = ConnectionId::new(1);
18169 let malformed = Frame::build(
18170 FrameType::Hello,
18171 control_flags(),
18172 0,
18173 0,
18174 3,
18175 b"{not json".to_vec(),
18176 )
18177 .unwrap();
18178
18179 let error = handler.handle_control(conn, malformed).unwrap();
18180 assert_eq!(error[0].header.ty, FrameType::Error);
18181 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
18182
18183 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18184 let pong = handler.handle_control(conn, ping).unwrap();
18185 assert_eq!(pong[0].header.ty, FrameType::Pong);
18186 assert_eq!(pong[0].header.corr, 4);
18187 }
18188
18189 #[test]
18190 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
18191 let registry = Arc::new(Registry::default());
18192 let handler = ControlHandler::new(Arc::clone(®istry));
18193
18194 handler
18195 .handle_control(
18196 ConnectionId::new(1),
18197 hello_frame("aft", PROTOCOL_VERSION, 1),
18198 )
18199 .unwrap();
18200 let duplicate = handler
18201 .handle_control(
18202 ConnectionId::new(2),
18203 hello_frame("aft", PROTOCOL_VERSION, 2),
18204 )
18205 .unwrap();
18206
18207 assert_eq!(duplicate[0].header.ty, FrameType::Error);
18208 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18209 let registration = registry.get_module("aft").unwrap().unwrap();
18210 assert_eq!(registration.connection_id, ConnectionId::new(1));
18211 }
18212
18213 #[test]
18214 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
18215 let registry = Arc::new(Registry::default());
18216 let forwarding = Arc::new(ForwardingTable::default());
18217 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
18218 let handler =
18219 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18220 .with_process_liveness(process_liveness);
18221 let (ctx, route_channel, route_epoch) =
18222 bind_liveness_route(®istry, &forwarding, "aft-dead");
18223 let responses = handler
18224 .handle_route_poll(
18225 &ctx,
18226 route_poll_frame(41, PollKind::Liveness, route_channel),
18227 route_channel,
18228 route_epoch,
18229 PollKind::Liveness,
18230 )
18231 .unwrap();
18232
18233 assert_eq!(responses.len(), 1);
18234 assert_eq!(responses[0].header.ty, FrameType::Response);
18235 assert_route_poll_liveness(&responses[0], false);
18236 }
18237
18238 #[test]
18239 fn liveness_poll_without_process_source_uses_bound_route() {
18240 let registry = Arc::new(Registry::default());
18241 let forwarding = Arc::new(ForwardingTable::default());
18242 let handler =
18243 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18244 let (ctx, route_channel, route_epoch) =
18245 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
18246 let responses = handler
18247 .handle_route_poll(
18248 &ctx,
18249 route_poll_frame(42, PollKind::Liveness, route_channel),
18250 route_channel,
18251 route_epoch,
18252 PollKind::Liveness,
18253 )
18254 .unwrap();
18255
18256 assert_route_poll_liveness(&responses[0], true);
18257 }
18258
18259 #[test]
18260 fn liveness_poll_untracked_process_source_uses_bound_route() {
18261 let registry = Arc::new(Registry::default());
18262 let forwarding = Arc::new(ForwardingTable::default());
18263 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
18264 let handler =
18265 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18266 .with_process_liveness(process_liveness);
18267 let (ctx, route_channel, route_epoch) =
18268 bind_liveness_route(®istry, &forwarding, "aft-untracked");
18269 let responses = handler
18270 .handle_route_poll(
18271 &ctx,
18272 route_poll_frame(43, PollKind::Liveness, route_channel),
18273 route_channel,
18274 route_epoch,
18275 PollKind::Liveness,
18276 )
18277 .unwrap();
18278
18279 assert_route_poll_liveness(&responses[0], true);
18280 }
18281
18282 #[tokio::test]
18283 async fn unknown_op_returns_unknown_control_op() {
18284 let handler = ControlHandler::default();
18285 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
18286 let request = Frame::build(
18287 FrameType::Request,
18288 control_flags(),
18289 0,
18290 0,
18291 55,
18292 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
18293 )
18294 .unwrap();
18295
18296 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18297
18298 assert_eq!(response.len(), 1);
18299 assert_eq!(response[0].header.ty, FrameType::Error);
18300 assert_eq!(response[0].header.corr, 55);
18301 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
18302 }
18303
18304 #[tokio::test]
18305 async fn supervisor_provenance_rejects_unknown_exact_module() {
18306 let handler = ControlHandler::default();
18307 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
18308 let request = Frame::build(
18309 FrameType::Request,
18310 control_flags(),
18311 0,
18312 0,
18313 57,
18314 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
18315 )
18316 .unwrap();
18317
18318 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18319
18320 assert_eq!(response.len(), 1);
18321 assert_eq!(response[0].header.ty, FrameType::Error);
18322 assert_eq!(response[0].header.corr, 57);
18323 let error = parse_error(&response[0]);
18324 assert_eq!(error["code"], "unknown_module");
18325 assert_eq!(error["message"], "module_id 'missing' is not supervised");
18326 }
18327
18328 #[test]
18329 fn provenance_probe_override_keeps_handler_tests_deterministic() {
18330 let expected = subc_control::RunningImageAgreement::Unavailable {
18331 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18332 };
18333 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
18334 assert_eq!(handler.provenance_probe_override, Some(expected));
18335 }
18336
18337 #[test]
18338 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
18339 let verdict = reload_verdict(
18340 std::path::Path::new("/bin/new"),
18341 Some(std::path::Path::new("/bin/old")),
18342 subc_control::RunningImageAgreement::Unavailable {
18343 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18344 },
18345 );
18346 assert!(matches!(
18347 verdict.path,
18348 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
18349 if configured == std::path::Path::new("/bin/new")
18350 && spawned_from == std::path::Path::new("/bin/old")
18351 ));
18352 }
18353
18354 #[test]
18355 fn launch_spec_without_running_digest_is_unavailable_not_match() {
18356 use subc_control::{
18357 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
18358 };
18359 let configured = crate::supervise::ModuleSpec {
18360 script: None,
18361 carry_containment: false,
18362 module_id: "legacy".into(),
18363 program: "/module".into(),
18364 args: Vec::new(),
18365 env: Vec::new(),
18366 reserved: false,
18367 reserved_prefixes: Vec::new(),
18368 protocol: subc_control::ModuleProtocol::Subc,
18369 overlap: Default::default(),
18370 };
18371 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
18372 assert_eq!(
18373 launch_spec_agreement(&configured, false, Some(&digest)),
18374 Agreement::Unavailable {
18375 reason: Reason::NotRunning
18376 }
18377 );
18378 assert_eq!(
18379 launch_spec_agreement(&configured, true, None),
18380 Agreement::Unavailable {
18381 reason: Reason::SpawnedSpecUnavailable
18382 }
18383 );
18384 assert_eq!(
18385 launch_spec_agreement(&configured, true, Some(&digest)),
18386 Agreement::Match
18387 );
18388 }
18389
18390 #[cfg(windows)]
18391 #[test]
18392 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
18393 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
18394 let configured = dir.join("module.exe");
18395 std::fs::write(&configured, b"executable path fixture").unwrap();
18396 let spawned = std::fs::canonicalize(&configured).unwrap();
18397 assert_ne!(
18398 configured, spawned,
18399 "fixture must exercise different path spellings"
18400 );
18401 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
18402 let lower_case = configured.to_string_lossy().to_lowercase();
18403 for spelling in [
18404 &configured,
18405 Path::new(&forward_slashes),
18406 Path::new(&lower_case),
18407 ] {
18408 let verdict = reload_verdict(
18409 spelling,
18410 Some(&spawned),
18411 subc_control::RunningImageAgreement::Unavailable {
18412 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18413 },
18414 );
18415 assert_eq!(
18416 verdict.path,
18417 ReloadPathAgreement::Match,
18418 "configured {spelling:?}, spawned {spawned:?}"
18419 );
18420 }
18421 }
18422
18423 #[cfg(windows)]
18424 #[test]
18425 fn windows_reload_path_keeps_different_existing_program_pending() {
18426 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
18427 let old = dir.join("old.exe");
18428 let new = dir.join("new.exe");
18429 std::fs::write(&old, b"same executable content").unwrap();
18430 std::fs::copy(&old, &new).unwrap();
18431 let spawned = std::fs::canonicalize(&old).unwrap();
18432 let image = subc_control::RunningImageAgreement::Unavailable {
18433 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18434 };
18435 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
18436 assert!(
18437 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
18438 if configured == new && spawned_from == spawned)
18439 );
18440 let missing = dir.join("missing.exe");
18441 assert!(matches!(
18442 reload_verdict(&missing, Some(&spawned), image).path,
18443 ReloadPathAgreement::Mismatch { .. }
18444 ));
18445 }
18446
18447 #[test]
18448 fn reload_verdict_detects_replaced_image_at_same_path() {
18449 let image = subc_control::RunningImageAgreement::Mismatch {
18450 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
18451 digest: "old".into(),
18452 },
18453 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
18454 digest: "new".into(),
18455 },
18456 };
18457 let verdict = reload_verdict(
18458 std::path::Path::new("/bin/same"),
18459 Some(std::path::Path::new("/bin/same")),
18460 image.clone(),
18461 );
18462 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18463 assert_eq!(verdict.image, image);
18464 }
18465
18466 #[test]
18467 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
18468 let image = subc_control::RunningImageAgreement::Unavailable {
18469 reason: subc_control::RunningImageUnavailableReason::NotRunning,
18470 };
18471 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
18472 assert_eq!(
18473 verdict.path,
18474 subc_control::ReloadPathAgreement::Unavailable {
18475 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
18476 }
18477 );
18478 assert_eq!(verdict.image, image);
18479
18480 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
18481 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18482 };
18483 let verdict = reload_verdict(
18484 std::path::Path::new("/bin/same"),
18485 Some(std::path::Path::new("/bin/same")),
18486 unconfirmed.clone(),
18487 );
18488 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18489 assert_eq!(verdict.image, unconfirmed);
18490 }
18491
18492 #[test]
18493 fn reload_verdict_preserves_each_image_unavailability_reason() {
18494 use subc_control::RunningImageUnavailableReason as Reason;
18495
18496 for reason in [
18497 Reason::NotRunning,
18498 Reason::UnsupportedPlatform,
18499 Reason::RunningExecutableUnreadable,
18500 Reason::SpawnedPathUnreadable,
18501 Reason::HashFailed,
18502 Reason::ProcessIdentityUnconfirmed,
18503 Reason::Unknown("future_probe_reason".to_string()),
18504 ] {
18505 let image = subc_control::RunningImageAgreement::Unavailable {
18506 reason: reason.clone(),
18507 };
18508 let verdict = reload_verdict(
18509 std::path::Path::new("/bin/same"),
18510 Some(std::path::Path::new("/bin/same")),
18511 image.clone(),
18512 );
18513 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18514 assert_eq!(verdict.image, image, "{reason:?}");
18515 }
18516 }
18517
18518 #[tokio::test]
18519 async fn malformed_control_bodies_return_invalid_control_body() {
18520 let handler = ControlHandler::default();
18521 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
18522
18523 for (corr, body) in [
18524 (56, br#"{"route_channel":1}"#.as_slice()),
18525 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
18526 (
18527 58,
18528 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
18529 ),
18530 ] {
18531 let request = Frame::build(
18532 FrameType::Request,
18533 control_flags(),
18534 0,
18535 0,
18536 corr,
18537 body.to_vec(),
18538 )
18539 .unwrap();
18540 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18541
18542 assert_eq!(response.len(), 1);
18543 assert_eq!(response[0].header.ty, FrameType::Error);
18544 assert_eq!(response[0].header.corr, corr);
18545 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
18546 }
18547 }
18548
18549 #[tokio::test]
18550 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
18551 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18552 let registry = Arc::new(Registry::default());
18553 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18554 let router = Router::with_control_handler(Arc::clone(&control));
18555 let connection = router.begin_connection();
18556 let (ctx, mut rx) = route_ctx(connection.id());
18557
18558 router
18559 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
18560 .await
18561 .unwrap();
18562 let response = rx.recv().await.unwrap();
18563 let ack = parse_ack(&response);
18564 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18565 let channel = 1;
18566
18567 let goodbye =
18568 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
18569 router.route_for_connection(&ctx, goodbye).await.unwrap();
18570 assert!(rx.try_recv().is_err());
18571 assert!(registry.get_module("aft").unwrap().is_none());
18572
18573 router
18574 .route_for_connection(&ctx, channel_request(channel, 13))
18575 .await
18576 .unwrap();
18577 let error_frame = rx.recv().await.unwrap();
18578 assert_eq!(error_frame.header.ty, FrameType::Error);
18579 assert_eq!(error_frame.header.channel, channel);
18580 let captured = crate::router::test_log::captured_logs(&logs);
18581 assert_eq!(
18582 captured
18583 .lines()
18584 .filter(|line| {
18585 line.contains("module_id=aft")
18586 && line.contains("reason=explicit_goodbye")
18587 && line.contains("module registration ended")
18588 })
18589 .count(),
18590 1,
18591 "unexpected GOODBYE registry log: {captured}"
18592 );
18593 }
18594
18595 #[tokio::test]
18596 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
18597 let registry = Arc::new(Registry::default());
18598 let handler = ControlHandler::new(registry).with_capability_config(
18599 [("prov".to_string(), true), ("cons".to_string(), true)],
18600 BTreeMap::new(),
18601 );
18602 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
18603 register_capability_manifest(
18604 &handler,
18605 &provider_ctx,
18606 &mut provider_rx,
18607 capability_manifest("prov", &["thing/v1"], &[]),
18608 1,
18609 )
18610 .await;
18611 let mut consumer = capability_manifest("cons", &[], &[]);
18612 consumer.capabilities.as_mut().unwrap().requires.push(
18613 subc_protocol::manifest::CapabilityRequirement {
18614 capability: "thing/v1".to_string(),
18615 need: subc_protocol::manifest::CapabilityNeed::Required,
18616 },
18617 );
18618 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
18619 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
18620 assert_eq!(
18621 handler.capability_evaluator.verdict("cons", "thing/v1"),
18622 Some(CapabilityVerdict::Provided)
18623 );
18624 let (mut client_rx, _) = open_route_for_capability_test(
18625 &handler,
18626 &provider_ctx,
18627 &mut provider_rx,
18628 703,
18629 3,
18630 "prov",
18631 None,
18632 )
18633 .await;
18634 let goodbye =
18635 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18636 handler
18637 .handle_control_frame(&provider_ctx, goodbye)
18638 .await
18639 .unwrap();
18640 assert_eq!(
18641 handler.capability_evaluator.verdict("cons", "thing/v1"),
18642 Some(CapabilityVerdict::NeverProvided)
18643 );
18644 let closed = client_rx
18645 .try_recv()
18646 .expect("GOODBYE pushes route.closed before route GOODBYE");
18647 assert!(
18648 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
18649 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
18650 );
18651 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
18652 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
18653 }
18654
18655 struct RouteClosedFixture {
18656 handler: ControlHandler,
18657 #[cfg(unix)]
18658 supervisor: Supervisor,
18659 module_ctx: RouteCtx,
18660 #[cfg(unix)]
18661 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18662 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18663 route: Arc<crate::forwarding::RouteBinding>,
18664 }
18665
18666 async fn route_closed_fixture() -> RouteClosedFixture {
18667 let registry = Arc::new(Registry::default());
18668 let handler = ControlHandler::new(Arc::clone(®istry));
18669 #[cfg(unix)]
18670 let supervisor = Supervisor::new(registry, RestartPolicy::default())
18671 .with_forwarding(Arc::clone(&handler.forwarding));
18672 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
18673 register_capability_manifest(
18674 &handler,
18675 &module_ctx,
18676 &mut module_rx,
18677 capability_manifest("prov", &[], &[]),
18678 1,
18679 )
18680 .await;
18681 let (client_rx, bind) = open_route_for_capability_test(
18682 &handler,
18683 &module_ctx,
18684 &mut module_rx,
18685 702,
18686 2,
18687 "prov",
18688 None,
18689 )
18690 .await;
18691 let ModuleControlRequest::RouteBind {
18692 route_channel: channel,
18693 epoch: route_epoch,
18694 ..
18695 } = bind
18696 else {
18697 panic!("expected route.bind");
18698 };
18699 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
18700 handler
18701 .forwarding
18702 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
18703 .unwrap()
18704 else {
18705 panic!("the acknowledged route must be bound");
18706 };
18707 RouteClosedFixture {
18708 handler,
18709 #[cfg(unix)]
18710 supervisor,
18711 module_ctx,
18712 #[cfg(unix)]
18713 module_rx,
18714 client_rx,
18715 route,
18716 }
18717 }
18718
18719 impl RouteClosedFixture {
18720 #[cfg(unix)]
18721 async fn cut(&mut self) {
18722 let notices = async {
18723 let mut draining = self.module_rx.recv().await.unwrap();
18724 assert!(matches!(
18725 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
18726 &draining.body
18727 )
18728 .unwrap(),
18729 subc_protocol::session::ModuleControlCommand::Draining {
18730 reason: RouteCloseReason::Disable,
18731 ..
18732 }
18733 ));
18734 draining.flushed.take().unwrap().send(()).unwrap();
18735 let mut closing = self.client_rx.recv().await.unwrap();
18736 assert!(matches!(
18737 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
18738 ClientControlPush::RouteClosing {
18739 reason: RouteCloseReason::Restart,
18740 ..
18741 }
18742 ));
18743 closing.flushed.take().unwrap().send(()).unwrap();
18744 };
18745 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
18746 result.unwrap();
18747 }
18748
18749 fn disconnect(&mut self) -> ClientControlPush {
18750 self.handler
18751 .cleanup_connection(self.module_ctx.connection_id)
18752 .unwrap();
18753 let closed = self.client_rx.try_recv().unwrap();
18754 assert_eq!(closed.header.ty, FrameType::Push);
18755 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
18756 assert!(matches!(
18757 &push,
18758 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
18759 if channels == &[self.route.client_channel]
18760 ));
18761 assert_eq!(
18762 self.client_rx.try_recv().unwrap().header.ty,
18763 FrameType::Goodbye
18764 );
18765 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
18766 push
18767 }
18768 }
18769
18770 #[cfg(unix)]
18771 #[tokio::test(start_paused = true)]
18772 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
18773 let mut fixture = route_closed_fixture().await;
18774 fixture.cut().await;
18775 assert!(matches!(
18776 fixture.disconnect(),
18777 ClientControlPush::RouteClosed {
18778 reason: RouteCloseReason::Restart,
18779 drained: true,
18780 ..
18781 }
18782 ));
18783 }
18784
18785 #[cfg(unix)]
18786 #[tokio::test(start_paused = true)]
18787 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
18788 let mut fixture = route_closed_fixture().await;
18789 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
18790 let started = tokio::time::Instant::now();
18791 fixture.cut().await;
18792 assert!(started.elapsed() >= Duration::from_secs(2));
18793 assert_eq!(fixture.route.flow.in_flight(), 1);
18794 assert!(matches!(
18795 fixture.disconnect(),
18796 ClientControlPush::RouteClosed {
18797 reason: RouteCloseReason::Restart,
18798 drained: false,
18799 ..
18800 }
18801 ));
18802 }
18803
18804 #[tokio::test]
18805 async fn module_connection_crash_reports_crash_and_not_drained() {
18806 let mut fixture = route_closed_fixture().await;
18807 assert!(matches!(
18811 fixture.disconnect(),
18812 ClientControlPush::RouteClosed {
18813 reason: RouteCloseReason::Crash,
18814 drained: false,
18815 ..
18816 }
18817 ));
18818 }
18819
18820 #[tokio::test]
18821 async fn dropping_router_connection_releases_registration() {
18822 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18823 let registry = Arc::new(Registry::default());
18824 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18825 let router = Router::with_control_handler(Arc::clone(&control));
18826 let connection = router.begin_connection();
18827 let connection_id = connection.id();
18828 let (ctx, mut rx) = route_ctx(connection_id);
18829
18830 router
18831 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
18832 .await
18833 .unwrap();
18834 let response = rx.recv().await.unwrap();
18835 let ack = parse_ack(&response);
18836 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18837 assert!(registry.get_module("aft").unwrap().is_some());
18838
18839 drop(connection);
18840
18841 assert!(registry.get_module("aft").unwrap().is_none());
18842 assert_eq!(registry.active_registration_count().unwrap(), 0);
18843
18844 control
18845 .cleanup_connection(ConnectionId::new(u64::MAX))
18846 .unwrap();
18847 let captured = crate::router::test_log::captured_logs(&logs);
18848 println!("captured registry lifecycle logs:\n{captured}");
18849 let events: Vec<_> = captured
18850 .lines()
18851 .filter(|line| {
18852 line.contains("module registered module_id=aft ")
18853 || line.contains("module registration ended")
18854 })
18855 .collect();
18856 assert_eq!(
18857 events.len(),
18858 2,
18859 "unexpected registry lifecycle logs: {captured}"
18860 );
18861 assert!(events[0].contains("module registered module_id=aft "));
18862 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
18863 assert!(events[1].contains(&format!(
18864 "module_id=aft connection_id={}",
18865 connection_id.get()
18866 )));
18867 assert!(events[1].contains("reason=connection_closed"));
18868 assert!(events[1].contains("module registration ended"));
18869 assert_eq!(
18870 captured
18871 .lines()
18872 .filter(|line| line.contains("module registered module_id=aft "))
18873 .count(),
18874 1,
18875 "legacy registration admission line must appear once: {captured}"
18876 );
18877 }
18878
18879 #[tokio::test]
18880 async fn hello_registration_keeps_legacy_module_registered_line_once() {
18881 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18882 let registry = Arc::new(Registry::default());
18883 let control = ControlHandler::new(Arc::clone(®istry));
18884 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
18885 hello_via_sink(
18886 &control,
18887 &ctx,
18888 &mut rx,
18889 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
18890 )
18891 .await;
18892
18893 let captured = crate::router::test_log::captured_logs(&logs);
18894 let admissions: Vec<_> = captured
18895 .lines()
18896 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
18897 .collect();
18898 assert_eq!(
18899 admissions.len(),
18900 1,
18901 "legacy admission line must remain exactly once: {captured}"
18902 );
18903 assert!(admissions[0].contains("routable_provider=true"));
18904 assert!(admissions[0].contains("connection_id=777"));
18905 }
18906
18907 fn capability_manifest(
18908 module_id: &str,
18909 provides: &[&str],
18910 must_never_reach: &[&str],
18911 ) -> ModuleManifest {
18912 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
18913 manifest.capabilities = Some(CapabilityDeclarations {
18914 provides: provides
18915 .iter()
18916 .map(|capability| (*capability).to_string())
18917 .collect(),
18918 requires: Vec::new(),
18919 must_never_reach: must_never_reach
18920 .iter()
18921 .map(|capability| (*capability).to_string())
18922 .collect(),
18923 });
18924 manifest
18925 }
18926
18927 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
18928 Frame::build(
18929 FrameType::Hello,
18930 control_flags(),
18931 0,
18932 0,
18933 corr,
18934 serde_json::to_vec(&ModuleHelloBody {
18935 protocol_ver: manifest.protocol_ver,
18936 manifest,
18937 control_ops: None,
18938 launch_nonce: None,
18939 })
18940 .expect("capability test HELLO serializes"),
18941 )
18942 .expect("capability test HELLO frame builds")
18943 }
18944
18945 fn catalog_update_with_capabilities_frame(
18946 corr: u64,
18947 capabilities: CapabilityDeclarations,
18948 ) -> Frame {
18949 Frame::build(
18950 FrameType::Request,
18951 control_flags(),
18952 0,
18953 0,
18954 corr,
18955 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
18956 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
18957 capabilities: Some(capabilities),
18958 ready: None,
18959 })
18960 .expect("capability catalog.update serializes"),
18961 )
18962 .expect("capability catalog.update frame builds")
18963 }
18964
18965 async fn register_capability_manifest(
18966 handler: &ControlHandler,
18967 ctx: &RouteCtx,
18968 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
18969 manifest: ModuleManifest,
18970 corr: u64,
18971 ) {
18972 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
18973 }
18974
18975 async fn open_route_for_capability_test(
18976 handler: &ControlHandler,
18977 target_ctx: &RouteCtx,
18978 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
18979 client_connection_id: u64,
18980 corr: u64,
18981 target_module_id: &str,
18982 consumer_identity: Option<ConsumerIdentity>,
18983 ) -> (
18984 mpsc::Receiver<crate::router::OutboundFrame>,
18985 ModuleControlRequest,
18986 ) {
18987 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
18988 let route_handler = handler.clone();
18989 let target_module_id = target_module_id.to_string();
18990 let route_task = tokio::spawn(async move {
18991 route_handler
18992 .handle_control_frame(
18993 &client_ctx,
18994 route_open_frame_with_admission_facts(
18995 corr,
18996 &target_module_id,
18997 unique_project_root("admission-facts"),
18998 consumer_identity,
18999 None,
19000 ),
19001 )
19002 .await
19003 .expect("capability test route.open succeeds")
19004 });
19005 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
19006 .await
19007 .expect("capability test route.open must reach route.bind")
19008 .expect("target control receiver stays open");
19009 let bind_request: ModuleControlRequest =
19010 serde_json::from_slice(&bind.body).expect("route.bind decodes");
19011 handler
19012 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
19013 .await
19014 .expect("capability test route.bind ACK succeeds");
19015 assert!(route_task.await.expect("route.open task joins").is_empty());
19016 let opened = client_rx
19017 .recv()
19018 .await
19019 .expect("successful route.open publishes a response");
19020 assert!(matches!(
19021 serde_json::from_slice::<ClientControlResponse>(&opened.body),
19022 Ok(ClientControlResponse::RouteOpen { .. })
19023 ));
19024 (client_rx, bind_request)
19025 }
19026
19027 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
19028 assert_eq!(frame.header.ty, FrameType::Push);
19029 assert_eq!(frame.header.channel, 0);
19030 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
19031 .expect("route.closed control push decodes");
19032 let ClientControlPush::RouteClosed { channels, .. } = &push else {
19033 panic!("expected route.closed");
19034 };
19035 assert_eq!(channels.len(), 1, "exactly one violating route closed");
19036 let closed_channel = channels[0];
19037 let channels = channels.clone();
19038 assert_eq!(
19039 push,
19040 ClientControlPush::RouteClosed {
19041 module_id: target_module_id.to_string(),
19042 channels,
19043 reason: RouteCloseReason::CapabilityDenied,
19044 drained: false,
19045 abandoned: 0,
19046 excluded_subscriptions: 0,
19047 terminal: Some(false),
19048 }
19049 );
19050 closed_channel
19051 }
19052
19053 #[tokio::test]
19054 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
19055 let registry = Arc::new(Registry::default());
19056 let forwarding = Arc::new(ForwardingTable::default());
19057 let supervisor = SupervisorHandle::new();
19058 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19059 let handler =
19060 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19061 .with_supervisor(supervisor);
19062 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
19063 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
19064 register_capability_manifest(
19065 &handler,
19066 &target_ctx,
19067 &mut target_rx,
19068 capability_manifest("target", &["credentials-provider/v1"], &[]),
19069 1,
19070 )
19071 .await;
19072 register_capability_manifest(
19073 &handler,
19074 &opener_ctx,
19075 &mut opener_rx,
19076 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19077 2,
19078 )
19079 .await;
19080
19081 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
19082 let replies = handler
19083 .handle_control_frame(
19084 &client_ctx,
19085 route_open_frame_with_admission_facts(
19086 3,
19087 "target",
19088 unique_project_root("admission-facts"),
19089 Some(ConsumerIdentity {
19090 module_id: "opener".to_string(),
19091 launch_nonce: "opener-nonce".to_string(),
19092 }),
19093 None,
19094 ),
19095 )
19096 .await
19097 .expect("denied route.open returns a typed frame");
19098 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19099 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19100 assert!(
19101 target_rx.try_recv().is_err(),
19102 "forbidden route.open must not relay route.bind"
19103 );
19104 }
19105
19106 #[tokio::test]
19107 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
19108 let registry = Arc::new(Registry::default());
19109 let forwarding = Arc::new(ForwardingTable::default());
19110 let supervisor = SupervisorHandle::new();
19111 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19112 let handler =
19113 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19114 .with_supervisor(supervisor);
19115 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
19116 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
19117 register_capability_manifest(
19118 &handler,
19119 &target_ctx,
19120 &mut target_rx,
19121 capability_manifest("target", &["credentials-provider/v1"], &[]),
19122 1,
19123 )
19124 .await;
19125 register_capability_manifest(
19126 &handler,
19127 &old_opener_ctx,
19128 &mut old_opener_rx,
19129 capability_manifest("opener", &[], &[]),
19130 2,
19131 )
19132 .await;
19133 let (mut client_rx, _) = open_route_for_capability_test(
19134 &handler,
19135 &target_ctx,
19136 &mut target_rx,
19137 712,
19138 3,
19139 "target",
19140 Some(ConsumerIdentity {
19141 module_id: "opener".to_string(),
19142 launch_nonce: "opener-nonce".to_string(),
19143 }),
19144 )
19145 .await;
19146 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19147
19148 handler
19149 .cleanup_connection(old_opener_ctx.connection_id)
19150 .expect("old opener registration cleans up");
19151 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
19152 register_capability_manifest(
19153 &handler,
19154 &new_opener_ctx,
19155 &mut new_opener_rx,
19156 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19157 4,
19158 )
19159 .await;
19160
19161 let closed_channel = assert_capability_denied_push(
19162 client_rx
19163 .try_recv()
19164 .expect("HELLO deny addition must emit route.closed")
19165 .frame,
19166 "target",
19167 );
19168 let goodbye = client_rx
19169 .try_recv()
19170 .expect("HELLO deny addition must send the client a route GOODBYE")
19171 .frame;
19172 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19173 assert_eq!(goodbye.header.channel, closed_channel);
19174 assert_ne!(goodbye.header.channel, 0);
19175 assert_ne!(goodbye.header.epoch, 0);
19176 assert!(client_rx.try_recv().is_err());
19177 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19178 assert!(matches!(
19179 target_rx.try_recv(),
19180 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19181 ));
19182 }
19183
19184 #[tokio::test]
19185 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
19186 let registry = Arc::new(Registry::default());
19187 let forwarding = Arc::new(ForwardingTable::default());
19188 let supervisor = SupervisorHandle::new();
19189 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19190 let handler =
19191 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19192 .with_supervisor(supervisor);
19193 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
19194 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
19195 register_capability_manifest(
19196 &handler,
19197 &target_ctx,
19198 &mut target_rx,
19199 capability_manifest("target", &[], &[]),
19200 1,
19201 )
19202 .await;
19203 register_capability_manifest(
19204 &handler,
19205 &opener_ctx,
19206 &mut opener_rx,
19207 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19208 2,
19209 )
19210 .await;
19211 let (mut client_rx, _) = open_route_for_capability_test(
19212 &handler,
19213 &target_ctx,
19214 &mut target_rx,
19215 722,
19216 3,
19217 "target",
19218 Some(ConsumerIdentity {
19219 module_id: "opener".to_string(),
19220 launch_nonce: "opener-nonce".to_string(),
19221 }),
19222 )
19223 .await;
19224 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19225
19226 let replies = handler
19227 .handle_control_frame(
19228 &target_ctx,
19229 catalog_update_with_capabilities_frame(
19230 4,
19231 CapabilityDeclarations {
19232 provides: vec!["credentials-provider/v1".to_string()],
19233 requires: Vec::new(),
19234 must_never_reach: Vec::new(),
19235 },
19236 ),
19237 )
19238 .await
19239 .expect("claim catalog.update succeeds");
19240 assert!(matches!(
19241 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
19242 Ok(ModuleControlResponseToModule::CatalogUpdate {})
19243 ));
19244 let closed_channel = assert_capability_denied_push(
19245 client_rx
19246 .try_recv()
19247 .expect("claim addition must emit route.closed")
19248 .frame,
19249 "target",
19250 );
19251 let goodbye = client_rx
19252 .try_recv()
19253 .expect("claim addition must send the client a route GOODBYE")
19254 .frame;
19255 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19256 assert_eq!(goodbye.header.channel, closed_channel);
19257 assert_ne!(goodbye.header.channel, 0);
19258 assert_ne!(goodbye.header.epoch, 0);
19259 assert!(client_rx.try_recv().is_err());
19260 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19261 assert!(matches!(
19262 target_rx.try_recv(),
19263 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19264 ));
19265 }
19266
19267 #[tokio::test]
19268 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
19269 let registry = Arc::new(Registry::default());
19270 let forwarding = Arc::new(ForwardingTable::default());
19271 let supervisor = SupervisorHandle::new();
19272 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19273 let handler =
19274 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19275 .with_supervisor(supervisor);
19276 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
19277 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
19278 register_capability_manifest(
19279 &handler,
19280 &target_ctx,
19281 &mut target_rx,
19282 capability_manifest("target", &["credentials-provider/v1"], &[]),
19283 1,
19284 )
19285 .await;
19286 register_capability_manifest(
19287 &handler,
19288 &opener_ctx,
19289 &mut opener_rx,
19290 capability_manifest("opener", &[], &[]),
19291 2,
19292 )
19293 .await;
19294 let (mut client_rx, _) = open_route_for_capability_test(
19295 &handler,
19296 &target_ctx,
19297 &mut target_rx,
19298 732,
19299 3,
19300 "target",
19301 Some(ConsumerIdentity {
19302 module_id: "opener".to_string(),
19303 launch_nonce: "opener-nonce".to_string(),
19304 }),
19305 )
19306 .await;
19307 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19308
19309 handler
19310 .handle_control_frame(
19311 &target_ctx,
19312 catalog_update_with_capabilities_frame(
19313 4,
19314 CapabilityDeclarations {
19315 provides: Vec::new(),
19316 requires: Vec::new(),
19317 must_never_reach: Vec::new(),
19318 },
19319 ),
19320 )
19321 .await
19322 .expect("claim removal catalog.update succeeds");
19323 assert_eq!(
19324 forwarding.active_binding_count().unwrap(),
19325 1,
19326 "removing an attested target claim must leave the route census unchanged"
19327 );
19328 assert!(
19329 client_rx.try_recv().is_err(),
19330 "claim removal must not emit route.closed or a client route GOODBYE"
19331 );
19332 assert!(
19333 target_rx.try_recv().is_err(),
19334 "claim removal must not send the target a route GOODBYE"
19335 );
19336 }
19337
19338 #[tokio::test]
19341 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
19342 let registry = Arc::new(Registry::default());
19343 let forwarding = Arc::new(ForwardingTable::default());
19344 let supervisor = SupervisorHandle::new();
19345 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19346 let handler =
19347 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19348 .with_supervisor(supervisor);
19349 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
19350 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
19351 register_capability_manifest(
19352 &handler,
19353 &target_ctx,
19354 &mut target_rx,
19355 capability_manifest("target", &["credentials-provider/v1"], &[]),
19356 1,
19357 )
19358 .await;
19359 register_capability_manifest(
19360 &handler,
19361 &opener_ctx,
19362 &mut opener_rx,
19363 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19364 2,
19365 )
19366 .await;
19367
19368 let (_client_rx, bind) = open_route_for_capability_test(
19369 &handler,
19370 &target_ctx,
19371 &mut target_rx,
19372 742,
19373 3,
19374 "target",
19375 None,
19376 )
19377 .await;
19378 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
19379 panic!("direct scope-honesty route must bind");
19380 };
19381 assert_eq!(principal, Some(Principal::Direct));
19382 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19383 }
19384
19385 #[tokio::test]
19388 async fn must_never_reach_self_route_is_capability_forbidden() {
19389 let registry = Arc::new(Registry::default());
19390 let forwarding = Arc::new(ForwardingTable::default());
19391 let supervisor = SupervisorHandle::new();
19392 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
19393 let handler =
19394 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19395 .with_supervisor(supervisor);
19396 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
19397 register_capability_manifest(
19398 &handler,
19399 &self_ctx,
19400 &mut self_rx,
19401 capability_manifest(
19402 "self-provider",
19403 &["credentials-provider/v1"],
19404 &["credentials-provider/v1"],
19405 ),
19406 1,
19407 )
19408 .await;
19409
19410 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
19411 let replies = handler
19412 .handle_control_frame(
19413 &client_ctx,
19414 route_open_frame_with_admission_facts(
19415 2,
19416 "self-provider",
19417 unique_project_root("admission-facts"),
19418 Some(ConsumerIdentity {
19419 module_id: "self-provider".to_string(),
19420 launch_nonce: "self-nonce".to_string(),
19421 }),
19422 None,
19423 ),
19424 )
19425 .await
19426 .expect("self-route refusal returns a typed frame");
19427 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19428 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19429 assert!(
19430 self_rx.try_recv().is_err(),
19431 "self denial must not relay route.bind"
19432 );
19433 }
19434
19435 #[test]
19436 fn unsupported_channel_zero_frame_returns_error() {
19437 let handler = ControlHandler::default();
19438 let request = Frame::build(
19439 FrameType::Request,
19440 control_flags(),
19441 0,
19442 0,
19443 21,
19444 b"opaque".to_vec(),
19445 )
19446 .unwrap();
19447
19448 let response = handler
19449 .handle_control(ConnectionId::new(1), request)
19450 .unwrap();
19451
19452 assert_eq!(response[0].header.ty, FrameType::Error);
19453 assert_eq!(
19454 parse_error(&response[0])["code"],
19455 "unsupported_control_frame"
19456 );
19457 }
19458
19459 mod swap {
19464 use super::*;
19465
19466 const INCUMBENT: ConnectionId = ConnectionId::new(30);
19467 const CANDIDATE: ConnectionId = ConnectionId::new(40);
19468
19469 struct Swap {
19470 registry: Arc<Registry>,
19471 forwarding: Arc<ForwardingTable>,
19472 handler: ControlHandler,
19473 incumbent_ctx: RouteCtx,
19474 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19475 candidate_ctx: RouteCtx,
19476 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19477 }
19478
19479 async fn swap_with_incumbent() -> Swap {
19480 let registry = Arc::new(Registry::default());
19481 let forwarding = Arc::new(ForwardingTable::default());
19482 let handler =
19483 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
19484 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
19485 hello_via_sink(
19486 &handler,
19487 &incumbent_ctx,
19488 &mut incumbent_rx,
19489 hello_frame("aft", PROTOCOL_VERSION, 7),
19490 )
19491 .await;
19492 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
19493 Swap {
19494 registry,
19495 forwarding,
19496 handler,
19497 incumbent_ctx,
19498 incumbent_rx,
19499 candidate_ctx,
19500 candidate_rx,
19501 }
19502 }
19503
19504 fn register_candidate(swap: &Swap, ready: Option<bool>) {
19505 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
19506 candidate_manifest.ready = ready;
19507 let registration = swap
19508 .registry
19509 .register_candidate_with_control_ops(
19510 candidate_manifest,
19511 PROTOCOL_VERSION,
19512 CANDIDATE,
19513 module_baseline_control_ops(),
19514 )
19515 .unwrap();
19516 swap.forwarding
19517 .register_candidate_module_connection(
19518 CANDIDATE,
19519 "aft".to_string(),
19520 PROTOCOL_VERSION,
19521 manifest_concurrency(®istration.manifest),
19522 swap.candidate_ctx.egress.clone(),
19523 )
19524 .unwrap();
19525 }
19526
19527 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
19528 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
19529 swap.registry.promote_candidate("aft").unwrap().unwrap();
19530 cutover.incumbent.unwrap()
19531 }
19532
19533 #[tokio::test]
19542 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
19543 let mut swap = swap_with_incumbent().await;
19544 let hold = Arc::new(std::sync::Barrier::new(2));
19545 let handler = swap
19546 .handler
19547 .clone()
19548 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
19549 handler
19550 .supervisor
19551 .open_swap("aft", "candidate-nonce".to_string());
19552 let hello = {
19553 let handler = handler.clone();
19554 let candidate_ctx = swap.candidate_ctx.clone();
19555 std::thread::spawn(move || {
19556 handler
19557 .handle_hello(
19558 candidate_ctx.connection_id,
19559 Some(candidate_ctx.egress.clone()),
19560 hello_frame_with_nonce(
19561 "aft",
19562 PROTOCOL_VERSION,
19563 8,
19564 Some("candidate-nonce"),
19565 ),
19566 )
19567 .unwrap()
19568 })
19569 };
19570
19571 hold.wait();
19572 assert!(swap
19575 .registry
19576 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
19577 .unwrap()
19578 .is_some());
19579 assert!(swap
19580 .registry
19581 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19582 .unwrap()
19583 .is_none());
19584 assert!(swap
19585 .forwarding
19586 .module_endpoint_for_connection(CANDIDATE)
19587 .unwrap()
19588 .is_none());
19589
19590 hold.wait();
19591 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
19592 assert_eq!(
19593 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19594 FrameType::HelloAck
19595 );
19596 let published = swap
19597 .registry
19598 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19599 .unwrap()
19600 .expect("candidate is published once its HELLO completes");
19601 assert_eq!(published.connection_id, CANDIDATE);
19602 assert!(swap
19603 .forwarding
19604 .module_endpoint_for_connection(CANDIDATE)
19605 .unwrap()
19606 .is_some());
19607 }
19608
19609 #[tokio::test]
19610 async fn candidate_lookup_after_hello_ack_waits_for_publication() {
19611 let mut swap = swap_with_incumbent().await;
19612 let hold = Arc::new(std::sync::Barrier::new(2));
19613 swap.handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
19614 swap.handler
19615 .supervisor
19616 .open_swap("aft", "candidate-nonce".to_string());
19617 let hello = {
19618 let handler = swap.handler.clone();
19619 let candidate_ctx = swap.candidate_ctx.clone();
19620 std::thread::spawn(move || {
19621 handler
19622 .handle_hello(
19623 CANDIDATE,
19624 Some(candidate_ctx.egress),
19625 hello_frame_with_nonce(
19626 "aft",
19627 PROTOCOL_VERSION,
19628 8,
19629 Some("candidate-nonce"),
19630 ),
19631 )
19632 .unwrap()
19633 })
19634 };
19635 hold.wait();
19636 assert_eq!(
19637 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19638 FrameType::HelloAck
19639 );
19640 let reader_registry = Arc::clone(&swap.registry);
19641 let published = lookup_during_hello_publication(&swap.registry, &hold, move || {
19642 reader_registry
19643 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19644 .unwrap()
19645 });
19646 assert!(hello.join().unwrap().is_empty());
19647 assert_eq!(
19648 published
19649 .expect("ACK makes candidate publication visible")
19650 .connection_id,
19651 CANDIDATE
19652 );
19653 assert_eq!(
19654 swap.registry.generation().unwrap(),
19655 1,
19656 "candidate is not a second catalog entry"
19657 );
19658 }
19659
19660 #[tokio::test]
19661 async fn audit4_ungated_removed_candidate_publication_rolls_back_admission() {
19662 let mut swap = swap_with_incumbent().await;
19663 let hold = Arc::new(std::sync::Barrier::new(2));
19664 let handler = swap
19665 .handler
19666 .clone()
19667 .with_hello_registry_to_forwarding_hold(hold.clone());
19668 handler
19669 .supervisor
19670 .open_swap("aft", "candidate-nonce".into());
19671 let hello = {
19672 let handler = handler.clone();
19673 let ctx = swap.candidate_ctx.clone();
19674 std::thread::spawn(move || {
19675 handler
19676 .handle_hello(
19677 CANDIDATE,
19678 Some(ctx.egress),
19679 hello_frame_with_nonce(
19680 "aft",
19681 PROTOCOL_VERSION,
19682 8,
19683 Some("candidate-nonce"),
19684 ),
19685 )
19686 .unwrap()
19687 })
19688 };
19689 hold.wait();
19690 assert_eq!(
19691 swap.registry
19692 .deregister_connection(CANDIDATE)
19693 .unwrap()
19694 .len(),
19695 1
19696 );
19697 hold.wait();
19698 assert!(hello.join().unwrap().is_empty());
19699 assert_eq!(
19700 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19701 FrameType::HelloAck
19702 );
19703 assert!(swap.registry.get_candidate("aft").unwrap().is_none());
19704 assert_eq!(
19705 handler
19706 .supervisor
19707 .swap_hello_admission("aft", Some("candidate-nonce")),
19708 SwapHelloAdmission::Candidate,
19709 "failed publication must roll back candidate_admitted for an ungated nonce swap"
19710 );
19711 handler.supervisor.close_swap("aft");
19712 }
19713
19714 fn keyed_total(counters: &Value, key: &str) -> u64 {
19715 counters[key]
19716 .as_object()
19717 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
19718 .unwrap_or(0)
19719 }
19720
19721 #[tokio::test]
19722 async fn replacement_logs_old_end_and_new_admission_once() {
19723 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19724 let mut swap = swap_with_incumbent().await;
19725 swap.handler
19726 .supervisor
19727 .open_swap("aft", "candidate-nonce".to_string());
19728 hello_via_sink(
19729 &swap.handler,
19730 &swap.candidate_ctx,
19731 &mut swap.candidate_rx,
19732 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
19733 )
19734 .await;
19735 cutover(&swap);
19736 swap.handler.cleanup_connection(INCUMBENT).unwrap();
19737
19738 let captured = crate::router::test_log::captured_logs(&logs);
19739 println!("captured replacement registry logs:\n{captured}");
19740 let new_admissions: Vec<_> = captured
19741 .lines()
19742 .filter(|line| {
19743 line.contains("connection_id=40")
19744 && line.contains("swap candidate registered; not routable until cutover")
19745 })
19746 .collect();
19747 assert_eq!(
19748 new_admissions.len(),
19749 1,
19750 "unexpected admission logs: {captured}"
19751 );
19752 assert!(new_admissions[0].contains("module_id=aft"));
19753 assert!(new_admissions[0].contains("connection_id=40"));
19754 assert!(
19755 new_admissions[0].contains("swap candidate registered; not routable until cutover")
19756 );
19757 assert!(new_admissions[0].contains("ready=true"));
19758
19759 let old_admissions: Vec<_> = captured
19760 .lines()
19761 .filter(|line| {
19762 line.contains("module registered module_id=aft ")
19763 && line.contains("connection_id=30")
19764 })
19765 .collect();
19766 assert_eq!(
19767 old_admissions.len(),
19768 1,
19769 "unexpected incumbent admission: {captured}"
19770 );
19771
19772 let old_ends: Vec<_> = captured
19773 .lines()
19774 .filter(|line| {
19775 line.contains("connection_id=30") && line.contains("module registration ended")
19776 })
19777 .collect();
19778 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
19779 assert!(old_ends[0].contains("module_id=aft"));
19780 assert!(old_ends[0].contains("connection_id=30"));
19781 assert!(old_ends[0].contains("reason=replaced"));
19782 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
19783 assert_eq!(
19784 captured
19785 .lines()
19786 .filter(|line| line.contains("module registration promoted"))
19787 .count(),
19788 1,
19789 "unexpected promotion logs: {captured}"
19790 );
19791 }
19792
19793 #[tokio::test]
19799 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
19800 let mut swap = swap_with_incumbent().await;
19801 let handler = swap.handler.clone();
19802
19803 let cotenant = ConnectionId::new(31);
19805 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
19806 let (cotenant_task, cotenant_bind) = relay_route_open(
19807 &handler,
19808 cotenant,
19809 &cotenant_ctx.egress,
19810 &mut swap.incumbent_rx,
19811 100,
19812 "aft",
19813 "swap-cotenant",
19814 )
19815 .await;
19816 handler
19817 .handle_control_frame(
19818 &swap.incumbent_ctx,
19819 route_bind_ack(cotenant_bind.header.corr),
19820 )
19821 .await
19822 .unwrap();
19823 assert!(cotenant_task.await.unwrap().is_empty());
19824 let (cotenant_channel, cotenant_epoch) =
19825 published_route(&cotenant_rx.recv().await.unwrap());
19826
19827 let caller = ConnectionId::new(32);
19829 let (caller_ctx, mut caller_rx) = route_ctx(caller);
19830 let (caller_task, caller_bind) = relay_route_open(
19831 &handler,
19832 caller,
19833 &caller_ctx.egress,
19834 &mut swap.incumbent_rx,
19835 101,
19836 "aft",
19837 "swap-caller",
19838 )
19839 .await;
19840 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
19841
19842 register_candidate(&swap, None);
19843 cutover(&swap);
19844
19845 let ack = handler
19847 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
19848 .await;
19849 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
19850 if module_loop_error.is_some() {
19851 handler.cleanup_connection(INCUMBENT).unwrap();
19854 }
19855
19856 assert!(
19858 cotenant_rx.try_recv().is_err(),
19859 "the co-tenant route on the incumbent was torn down by one late ack: \
19860 {module_loop_error:?}"
19861 );
19862 assert!(matches!(
19863 swap.forwarding
19864 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
19865 .unwrap(),
19866 DataRoute::Client(DataRouteState::Bound(_))
19867 ));
19868 assert_eq!(module_loop_error, None);
19869 assert!(swap
19870 .registry
19871 .get_module_by_connection(INCUMBENT)
19872 .unwrap()
19873 .is_some());
19874
19875 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
19877 .await
19878 .expect("the incumbent is told to drop the abandoned binding")
19879 .unwrap()
19880 .frame;
19881 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19882 assert_eq!(goodbye.header.channel, abandoned_channel);
19883 assert_eq!(goodbye.header.epoch, abandoned_epoch);
19884 assert!(swap.incumbent_rx.try_recv().is_err());
19885
19886 let response = caller_task.await.unwrap();
19888 assert_eq!(response.len(), 1);
19889 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
19890 assert!(caller_rx.try_recv().is_err());
19891
19892 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
19896 let counters = handler.counters().snapshot();
19897 assert_eq!(
19898 keyed_total(&counters, "route_open_accepted_by_principal"),
19899 1
19900 );
19901 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
19902 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
19903 }
19904
19905 #[tokio::test]
19909 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
19910 let mut swap = swap_with_incumbent().await;
19911 register_candidate(&swap, None);
19912 let incumbent = cutover(&swap);
19913 swap.forwarding
19914 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
19915 .unwrap()
19916 .expect("the incumbent is still registered");
19917
19918 let client = ConnectionId::new(33);
19919 let (client_ctx, mut client_rx) = route_ctx(client);
19920 let route_handler = swap.handler.clone();
19921 let open_ctx = RouteCtx {
19922 connection_id: client,
19923 egress: client_ctx.egress.clone(),
19924 };
19925 let mut route_task = tokio::spawn(async move {
19926 route_handler
19927 .handle_control_frame(
19928 &open_ctx,
19929 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
19930 )
19931 .await
19932 .unwrap()
19933 });
19934 let bind = tokio::select! {
19935 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
19936 response = &mut route_task => {
19937 let response = response.unwrap();
19938 panic!(
19939 "post-cutover route.open was refused instead of relayed to the candidate: {}",
19940 parse_error(&response[0])["code"]
19941 );
19942 }
19943 };
19944 swap.handler
19945 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
19946 .await
19947 .unwrap();
19948 assert!(route_task.await.unwrap().is_empty());
19949 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
19950 match swap
19951 .forwarding
19952 .lookup_data_route(client, channel, epoch)
19953 .unwrap()
19954 {
19955 DataRoute::Client(DataRouteState::Bound(route)) => {
19956 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
19957 }
19958 other => panic!("expected a bound route on the candidate, got {other:?}"),
19959 }
19960 assert!(swap.incumbent_rx.try_recv().is_err());
19961 }
19962
19963 #[tokio::test]
19968 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
19969 let swap = swap_with_incumbent().await;
19970 register_candidate(&swap, Some(false));
19971 let update = Frame::build(
19972 FrameType::Request,
19973 control_flags(),
19974 0,
19975 0,
19976 55,
19977 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
19978 provides: manifest("aft", PROTOCOL_VERSION).provides,
19979 capabilities: None,
19980 ready: Some(true),
19981 })
19982 .unwrap(),
19983 )
19984 .unwrap();
19985
19986 let replies = swap
19987 .handler
19988 .handle_control_frame(&swap.candidate_ctx, update)
19989 .await
19990 .unwrap();
19991
19992 assert_eq!(replies.len(), 1);
19993 assert_eq!(
19994 replies[0].header.ty,
19995 FrameType::Response,
19996 "candidate catalog.update was refused: {:?}",
19997 serde_json::from_slice::<Value>(&replies[0].body).ok()
19998 );
19999 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
20000 assert_eq!(
20001 swap.registry
20002 .get_module("aft")
20003 .unwrap()
20004 .unwrap()
20005 .connection_id,
20006 INCUMBENT
20007 );
20008 }
20009 }
20010
20011 mod swap_admission {
20015 use super::*;
20016
20017 const INCUMBENT_NONCE: &str = "incumbent-nonce";
20018 const CANDIDATE_NONCE: &str = "candidate-nonce";
20019
20020 fn handler_with_incumbent(
20021 module_id: &str,
20022 reserved: bool,
20023 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
20024 let registry = Arc::new(Registry::default());
20025 let supervisor = SupervisorHandle::new();
20026 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
20027 if reserved {
20028 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
20029 }
20030 let handler =
20031 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
20032 let incumbent = handler
20033 .handle_control(
20034 ConnectionId::new(1),
20035 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
20036 )
20037 .unwrap();
20038 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
20039 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
20040 (registry, supervisor, handler)
20041 }
20042
20043 #[test]
20049 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
20050 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
20051
20052 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
20053 let replies = handler
20054 .handle_control(
20055 ConnectionId::new(connection),
20056 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
20057 )
20058 .unwrap();
20059 assert_eq!(replies[0].header.ty, FrameType::Error);
20060 assert_eq!(
20061 parse_error(&replies[0])["code"],
20062 "swap_token_invalid",
20063 "nonce {nonce:?}"
20064 );
20065 }
20066 assert!(registry.get_candidate("aft").unwrap().is_none());
20067 assert_eq!(
20068 registry.get_module("aft").unwrap().unwrap().connection_id,
20069 ConnectionId::new(1)
20070 );
20071
20072 let admitted = handler
20075 .handle_control(
20076 ConnectionId::new(4),
20077 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
20078 )
20079 .unwrap();
20080 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
20081 assert_eq!(
20082 registry
20083 .get_candidate("aft")
20084 .unwrap()
20085 .unwrap()
20086 .connection_id,
20087 ConnectionId::new(4)
20088 );
20089 assert_eq!(
20090 registry.get_module("aft").unwrap().unwrap().connection_id,
20091 ConnectionId::new(1),
20092 "the candidate must not take the active slot"
20093 );
20094 let replayed = handler
20095 .handle_control(
20096 ConnectionId::new(5),
20097 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
20098 )
20099 .unwrap();
20100 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
20101
20102 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
20106 let squatter = handler
20107 .handle_control(
20108 ConnectionId::new(6),
20109 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
20110 )
20111 .unwrap();
20112 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
20113 assert!(
20114 registry.get_module("aft").unwrap().is_none(),
20115 "a squatter took the active slot of an id being swapped"
20116 );
20117 }
20118
20119 #[test]
20124 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
20125 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
20126
20127 let replies = handler
20128 .handle_control(
20129 ConnectionId::new(2),
20130 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20131 )
20132 .unwrap();
20133
20134 assert_eq!(
20135 replies[0].header.ty,
20136 FrameType::HelloAck,
20137 "reserved candidate refused: {:?}",
20138 serde_json::from_slice::<Value>(&replies[0].body).ok()
20139 );
20140 assert_eq!(
20141 registry
20142 .get_candidate("vault")
20143 .unwrap()
20144 .unwrap()
20145 .connection_id,
20146 ConnectionId::new(2)
20147 );
20148 }
20149
20150 #[test]
20153 fn without_an_open_swap_the_ordinary_gates_decide() {
20154 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
20155 supervisor.close_swap("vault");
20156
20157 let candidate = handler
20158 .handle_control(
20159 ConnectionId::new(2),
20160 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20161 )
20162 .unwrap();
20163 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
20164 let duplicate = handler
20165 .handle_control(
20166 ConnectionId::new(3),
20167 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
20168 )
20169 .unwrap();
20170 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
20171 assert!(registry.get_candidate("vault").unwrap().is_none());
20172 }
20173 }
20174
20175 mod scopes {
20179 mod close_reasons {
20180 include!("route_close_reason_tests.rs");
20181 }
20182 mod idle_tests {
20183 use super::*;
20184 include!("scope_idle_tests.rs");
20185 }
20186 mod route_open_timings {
20187 use super::*;
20188 include!("route_open_timing_tests.rs");
20189 }
20190 mod route_open_refusal_log {
20191 use super::*;
20192 include!("route_open_refusal_log_tests.rs");
20193 }
20194 mod activity {
20195 include!("activity_scope_tests.rs");
20196 }
20197 mod delta {
20198 include!("scope_handler_delta_tests.rs");
20199 }
20200 mod call_modes {
20201 include!("scope_call_mode_tests.rs");
20202 }
20203 mod federated {
20204 include!("scope_federated_tests.rs");
20205 }
20206 mod authorize {
20207 include!("route_authorize_tests.rs");
20208 }
20209 #[tokio::test]
20210 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
20211 let rig = rig().await;
20212 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
20213 "s".into(),
20214 1,
20215 "tool".into(),
20216 ))
20217 .unwrap();
20218 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
20219 let (direct, _rx) = wide_ctx(999);
20220 for (ctx, expected) in [
20221 (&rig.owner, FrameType::Response),
20222 (&direct, FrameType::Error),
20223 ] {
20224 let frame =
20225 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
20226 .unwrap();
20227 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
20228 assert_eq!(replies[0].header.ty, expected);
20229 if expected == FrameType::Response {
20230 let reply: subc_control::ScopeServedByReply =
20231 serde_json::from_slice(&replies[0].body).unwrap();
20232 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
20233 } else {
20234 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
20235 }
20236 }
20237 let reply = call(
20238 &rig.handler,
20239 &rig.owner,
20240 &ModuleControlRequestFromModule::ScopeDescribe {
20241 owner: Principal::Reserved {
20242 module_id: OWNER.into(),
20243 },
20244 scope_ref: "s".into(),
20245 },
20246 )
20247 .await;
20248 assert!(matches!(
20249 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
20250 ModuleControlResponseToModule::ScopeDescribe { .. }
20251 ));
20252 }
20253
20254 use subc_protocol::scope::{
20255 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
20256 ScopeStatus,
20257 };
20258
20259 use super::*;
20260
20261 const OWNER: &str = "prefrontal-core";
20262
20263 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
20264 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
20265 }
20266
20267 async fn call(
20268 handler: &ControlHandler,
20269 ctx: &RouteCtx,
20270 request: &ModuleControlRequestFromModule,
20271 ) -> Frame {
20272 let body = serde_json::to_vec(request).unwrap();
20273 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
20274 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
20275 assert_eq!(replies.len(), 1, "{replies:?}");
20276 replies.pop().unwrap()
20277 }
20278
20279 async fn sync(
20280 handler: &ControlHandler,
20281 ctx: &RouteCtx,
20282 generation: u64,
20283 scopes: Vec<ScopeRecord>,
20284 ) -> Result<ModuleControlResponseToModule, String> {
20285 let reply = call(
20286 handler,
20287 ctx,
20288 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
20289 )
20290 .await;
20291 match reply.header.ty {
20292 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
20293 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
20294 }
20295 }
20296
20297 async fn describe(
20298 handler: &ControlHandler,
20299 ctx: &RouteCtx,
20300 owner: &str,
20301 scope_ref: &str,
20302 ) -> ModuleControlResponseToModule {
20303 let reply = call(
20304 handler,
20305 ctx,
20306 &ModuleControlRequestFromModule::ScopeDescribe {
20307 owner: Principal::Reserved {
20308 module_id: owner.to_string(),
20309 },
20310 scope_ref: scope_ref.to_string(),
20311 },
20312 )
20313 .await;
20314 assert_eq!(
20315 reply.header.ty,
20316 FrameType::Response,
20317 "{:?}",
20318 parse_error(&reply)
20319 );
20320 serde_json::from_slice(&reply.body).unwrap()
20321 }
20322
20323 #[tokio::test]
20324 async fn scope_describe_never_echoes_bind_proof_or_hash() {
20325 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
20326 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
20327 let mut rig = rig().await;
20328 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
20329 .await;
20330 let _bound = rig
20331 .bound(
20332 Some(AFT),
20333 PLEXUS,
20334 Some(
20335 rig.selector("s", Some(1))
20336 .with_bind_proof(Some(PROOF.into())),
20337 ),
20338 )
20339 .await;
20340 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
20341 let output = serde_json::to_string(&reply).unwrap();
20342 assert!(output.contains("\"status\":\"live\""));
20343 assert!(!output.contains(PROOF));
20344 assert!(!output.contains(HASH));
20345 }
20346
20347 async fn module(
20349 handler: &ControlHandler,
20350 connection: u64,
20351 module_id: &str,
20352 nonce: Option<&str>,
20353 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20354 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
20355 hello_via_sink(
20356 handler,
20357 &ctx,
20358 &mut rx,
20359 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
20360 )
20361 .await;
20362 (ctx, rx)
20363 }
20364
20365 #[tokio::test]
20368 async fn a_client_connection_cannot_sync_or_describe() {
20369 let handler = ControlHandler::new(Arc::new(Registry::default()));
20370 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
20371 for request in [
20372 ModuleControlRequestFromModule::ScopeSync {
20373 generation: 1,
20374 scopes: vec![head("s", 1)],
20375 },
20376 ModuleControlRequestFromModule::ScopeDescribe {
20377 owner: Principal::Direct,
20378 scope_ref: "s".to_string(),
20379 },
20380 ] {
20381 let reply = call(&handler, &ctx, &request).await;
20382 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
20383 }
20384 assert!(
20385 !handler
20386 .scopes
20387 .read()
20388 .unwrap()
20389 .describe(
20390 &Principal::Reserved {
20391 module_id: OWNER.to_string()
20392 },
20393 "s"
20394 )
20395 .owner_synced
20396 );
20397 }
20398
20399 #[tokio::test]
20402 async fn a_module_without_a_supervised_launch_cannot_sync() {
20403 let handler = ControlHandler::new(Arc::new(Registry::default()));
20404 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
20405 assert_eq!(
20406 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
20407 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20408 );
20409 }
20410
20411 #[tokio::test]
20412 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
20413 let supervisor = SupervisorHandle::new();
20414 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20415 let handler = ControlHandler::new(Arc::new(Registry::default()))
20416 .with_supervisor(supervisor.clone());
20417 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20418 sync(&handler, &incumbent, 1, vec![head("s", 1)])
20419 .await
20420 .expect("the current launch syncs");
20421
20422 supervisor.open_swap(OWNER, "n2".to_string());
20425 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
20426 assert_eq!(
20427 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20428 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20429 );
20430 sync(&handler, &incumbent, 2, vec![head("s", 1)])
20431 .await
20432 .expect("the serving owner syncs during the swap");
20433
20434 supervisor.close_swap(OWNER);
20437 assert_eq!(
20438 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20439 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20440 );
20441 sync(&handler, &incumbent, 3, vec![head("s", 1)])
20442 .await
20443 .expect("the serving owner syncs after the rollback");
20444 handler.cleanup_connection(candidate.connection_id).unwrap();
20445
20446 supervisor.open_swap(OWNER, "n3".to_string());
20451 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
20452 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
20453 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
20454 .await
20455 .expect("the promoted launch takes authority");
20456 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20457 panic!("unexpected reply {reply:?}");
20458 };
20459 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
20460 assert_eq!(
20461 sync(&handler, &incumbent, 4, Vec::new()).await,
20462 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20463 );
20464 }
20465
20466 #[tokio::test]
20469 async fn closing_the_authority_connection_frees_sync_authority() {
20470 let supervisor = SupervisorHandle::new();
20471 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20472 let handler = ControlHandler::new(Arc::new(Registry::default()))
20473 .with_supervisor(supervisor.clone());
20474 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20475 sync(&handler, &first, 10, vec![head("s", 1)])
20476 .await
20477 .unwrap();
20478 handler.cleanup_connection(first.connection_id).unwrap();
20479
20480 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
20481 sync(&handler, &second, 1, vec![head("s", 1)])
20482 .await
20483 .expect("the next connection takes the released authority");
20484 }
20485
20486 #[tokio::test]
20487 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
20488 let supervisor = SupervisorHandle::new();
20489 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20490 let handler =
20491 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
20492 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
20493 sync(&handler, &first, 10, vec![head("s", 1)])
20494 .await
20495 .unwrap();
20496 handler
20497 .handle_control_frame(
20498 &first,
20499 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
20500 )
20501 .await
20502 .unwrap();
20503 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
20504 sync(&handler, &second, 1, vec![head("s", 1)])
20505 .await
20506 .expect("GOODBYE releases authority even if the old socket remains open");
20507 }
20508
20509 #[tokio::test]
20510 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
20511 let registry = Arc::new(Registry::default());
20512 let supervisor_handle = SupervisorHandle::new();
20513 let supervisor =
20514 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20515 .with_handle(supervisor_handle.clone())
20516 .with_daemon_incarnation("incarnation-7".to_string());
20517 supervisor
20520 .supervise_configured(
20521 ModuleSpec {
20522 script: None,
20523 carry_containment: false,
20524 module_id: OWNER.to_string(),
20525 program: PathBuf::from("/nonexistent/prefrontal-core"),
20526 args: Vec::new(),
20527 env: Vec::new(),
20528 reserved: false,
20529 reserved_prefixes: Vec::new(),
20530 protocol: ModuleProtocol::Subc,
20531 overlap: Default::default(),
20532 },
20533 false,
20534 )
20535 .unwrap();
20536 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
20537 let handler =
20538 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
20539 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
20540
20541 let ModuleControlResponseToModule::ScopeDescribe {
20543 status,
20544 daemon_incarnation,
20545 owner_synced,
20546 owner_configured,
20547 scope,
20548 ..
20549 } = describe(&handler, &reader, OWNER, "s").await
20550 else {
20551 panic!("not a describe reply");
20552 };
20553 assert_eq!(status, ScopeStatus::NotLive);
20554 assert_eq!(daemon_incarnation, "incarnation-7");
20555 assert!(!owner_synced);
20556 assert!(owner_configured);
20557 assert!(scope.is_none());
20558
20559 let ModuleControlResponseToModule::ScopeDescribe {
20562 status,
20563 owner_configured,
20564 ..
20565 } = describe(&handler, &reader, "ghost", "s").await
20566 else {
20567 panic!("not a describe reply");
20568 };
20569 assert_eq!(status, ScopeStatus::NotLive);
20570 assert!(!owner_configured);
20571
20572 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
20574 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
20575 let ModuleControlResponseToModule::ScopeDescribe {
20576 status,
20577 scope_epoch,
20578 owner_synced,
20579 scope,
20580 ..
20581 } = describe(&handler, &reader, OWNER, "s").await
20582 else {
20583 panic!("not a describe reply");
20584 };
20585 assert_eq!(status, ScopeStatus::Live);
20586 assert_eq!(scope_epoch, Some(4));
20587 assert!(owner_synced);
20588 let stamp = scope.expect("a live scope carries its stamp");
20589 assert!(
20590 stamp.owner_authorized,
20591 "prefrontal-core is the default authority"
20592 );
20593 assert_eq!(stamp.kind, ScopeKind::Head);
20594 }
20595
20596 #[tokio::test]
20597 async fn scope_authority_owners_decides_owner_authorized() {
20598 let supervisor = SupervisorHandle::new();
20599 supervisor.set_spawn_nonce("broca", "b1".to_string());
20600 let handler = ControlHandler::new(Arc::new(Registry::default()))
20601 .with_supervisor(supervisor)
20602 .with_scope_authority_owners(vec!["broca".to_string()]);
20603 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
20604 let mut gated = head("s", 1);
20605 gated.attributes.agent_id = Some("agent".to_string());
20606 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
20607 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
20608 describe(&handler, &broca, "broca", "s").await
20609 else {
20610 panic!("not a describe reply");
20611 };
20612 assert!(scope.unwrap().owner_authorized);
20613 }
20614
20615 #[tokio::test]
20619 async fn scope_ops_and_the_scopes_capability_are_advertised() {
20620 let handler = ControlHandler::new(Arc::new(Registry::default()));
20621 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
20622 let ack = hello_via_sink(
20623 &handler,
20624 &ctx,
20625 &mut rx,
20626 hello_frame("m", PROTOCOL_VERSION, 1),
20627 )
20628 .await;
20629 let ack = parse_ack(&ack);
20630 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
20631 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
20632 }
20633 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
20634
20635 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
20636 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
20637 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
20638 let reply = handler
20639 .handle_control_frame(&client, frame)
20640 .await
20641 .unwrap()
20642 .pop()
20643 .unwrap();
20644 let ClientControlResponse::ServerDescribe { capabilities, .. } =
20645 serde_json::from_slice(&reply.body).unwrap()
20646 else {
20647 panic!("not a server.describe reply");
20648 };
20649 assert!(
20650 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
20651 "{capabilities:?}"
20652 );
20653 }
20654
20655 const PLEXUS: &str = "plexus";
20658 const OTHER: &str = "other";
20659 const AFT: &str = "aft";
20660 const BROCA: &str = "broca";
20661 const MAGIC: &str = "magic-context";
20662
20663 fn nonce(module_id: &str) -> String {
20664 format!("nonce-{module_id}")
20665 }
20666
20667 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20668 let (tx, rx) = mpsc::channel(64);
20669 (
20670 RouteCtx {
20671 connection_id: ConnectionId::new(connection),
20672 egress: FrameSink::new(tx),
20673 },
20674 rx,
20675 )
20676 }
20677
20678 struct Rig {
20682 handler: ControlHandler,
20683 forwarding: Arc<ForwardingTable>,
20684 owner: RouteCtx,
20685 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20686 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
20687 generation: u64,
20688 next_connection: u64,
20689 _supervisor: Supervisor,
20690 }
20691
20692 async fn rig() -> Rig {
20693 rig_with_flow_support(true).await
20694 }
20695
20696 async fn rig_with_flow_support(flow_support: bool) -> Rig {
20697 rig_with_scope_capabilities(flow_support, false).await
20698 }
20699
20700 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
20701 let mut provides = Vec::new();
20702 if flow_support {
20703 provides.push("flow-scopes/v1");
20704 }
20705 if run_support {
20706 provides.push("agent-run-scopes/v1");
20707 }
20708 rig_with_target_capabilities(&provides).await
20709 }
20710
20711 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
20712 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
20713 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20714 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20717 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
20718 Frame::build(
20719 FrameType::Hello,
20720 control_flags(),
20721 0,
20722 0,
20723 connection,
20724 serde_json::to_vec(&body).unwrap(),
20725 )
20726 .unwrap()
20727 }
20728
20729 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
20730 let registry = Arc::new(Registry::default());
20731 let forwarding = Arc::new(ForwardingTable::default());
20732 let supervisor_handle = SupervisorHandle::new();
20733 let supervisor =
20734 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20735 .with_handle(supervisor_handle.clone());
20736 supervisor
20737 .supervise_configured(
20738 ModuleSpec {
20739 script: None,
20740 carry_containment: false,
20741 module_id: OWNER.to_string(),
20742 program: PathBuf::from("/nonexistent/prefrontal-core"),
20743 args: Vec::new(),
20744 env: Vec::new(),
20745 reserved: false,
20746 reserved_prefixes: Vec::new(),
20747 protocol: ModuleProtocol::Subc,
20748 overlap: Default::default(),
20749 },
20750 false,
20751 )
20752 .unwrap();
20753 for module_id in [OWNER, AFT, BROCA, MAGIC] {
20754 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
20755 }
20756 let handler =
20757 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
20758 .with_supervisor(supervisor_handle);
20759 let (owner, mut owner_rx) = wide_ctx(1);
20760 hello_via_sink(
20761 &handler,
20762 &owner,
20763 &mut owner_rx,
20764 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
20765 )
20766 .await;
20767 let mut modules = BTreeMap::new();
20768 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
20769 let (ctx, mut rx) = wide_ctx(connection);
20770 let hello = hello_with_capabilities(module_id, connection, provides);
20771 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
20772 modules.insert(module_id.to_string(), (ctx, rx));
20773 }
20774 Rig {
20775 handler,
20776 forwarding,
20777 owner,
20778 _owner_rx: owner_rx,
20779 modules,
20780 generation: 0,
20781 next_connection: 100,
20782 _supervisor: supervisor,
20783 }
20784 }
20785
20786 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
20787 ScopeCarrier::new(Principal::Reserved {
20788 module_id: module_id.to_string(),
20789 })
20790 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
20791 }
20792
20793 fn session(scope_epoch: u64) -> ScopeRecord {
20796 let mut record = head("s", scope_epoch);
20797 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
20798 record.attributes.agent_id = Some("agent-1".to_string());
20799 record.attributes.delegates = true;
20800 record
20801 }
20802
20803 impl Rig {
20804 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
20805 self.generation += 1;
20806 sync(&self.handler, &self.owner, self.generation, scopes)
20807 .await
20808 .expect("the owner's sync is accepted");
20809 }
20810
20811 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
20812 ScopeSelector::new(
20813 Principal::Reserved {
20814 module_id: OWNER.to_string(),
20815 },
20816 scope_ref.to_string(),
20817 scope_epoch,
20818 )
20819 }
20820
20821 fn open_frame(
20822 &mut self,
20823 opener: Option<&str>,
20824 target: &str,
20825 scope: Option<ScopeSelector>,
20826 ) -> (
20827 RouteCtx,
20828 mpsc::Receiver<crate::router::OutboundFrame>,
20829 Frame,
20830 ) {
20831 self.next_connection += 1;
20832 let (ctx, rx) = wide_ctx(self.next_connection);
20833 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
20834 busy_refusals: None,
20835 target: RouteTarget::ToolProvider {
20836 module_id: target.to_string(),
20837 },
20838 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
20839 consumer_identity: opener.map(|module_id| ConsumerIdentity {
20840 module_id: module_id.to_string(),
20841 launch_nonce: nonce(module_id),
20842 }),
20843 consumer_capabilities: None,
20844 role_versions: None,
20845 admission_facts: None,
20846 scope,
20847 })
20848 .unwrap();
20849 let frame = Frame::build(
20850 FrameType::Request,
20851 control_flags(),
20852 0,
20853 0,
20854 self.next_connection,
20855 body,
20856 )
20857 .unwrap();
20858 (ctx, rx, frame)
20859 }
20860
20861 async fn refused(
20863 &mut self,
20864 opener: Option<&str>,
20865 target: &str,
20866 scope: Option<ScopeSelector>,
20867 ) -> String {
20868 self.refusal_body(opener, target, scope).await["code"]
20869 .as_str()
20870 .unwrap()
20871 .to_string()
20872 }
20873
20874 async fn refusal_body(
20875 &mut self,
20876 opener: Option<&str>,
20877 target: &str,
20878 scope: Option<ScopeSelector>,
20879 ) -> Value {
20880 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
20881 let replies = tokio::time::timeout(
20882 Duration::from_secs(2),
20883 self.handler.handle_control_frame(&ctx, frame),
20884 )
20885 .await
20886 .expect("the open must be refused before waiting for a bind ack")
20887 .unwrap();
20888 assert_eq!(replies.len(), 1, "{replies:?}");
20889 assert_eq!(replies[0].header.ty, FrameType::Error);
20890 let (_, module_rx) = self.modules.get_mut(target).unwrap();
20891 assert!(
20892 module_rx.try_recv().is_err(),
20893 "a refused open relays nothing"
20894 );
20895 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
20896 parse_error(&replies[0])
20897 }
20898
20899 async fn relayed(
20901 &mut self,
20902 opener: Option<&str>,
20903 target: &str,
20904 scope: Option<ScopeSelector>,
20905 ) -> Relayed {
20906 self.relayed_with_legacy_root(opener, target, scope, None)
20907 .await
20908 }
20909
20910 async fn relayed_with_legacy_root(
20911 &mut self,
20912 opener: Option<&str>,
20913 target: &str,
20914 scope: Option<ScopeSelector>,
20915 legacy_root: Option<&std::path::Path>,
20916 ) -> Relayed {
20917 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
20918 if let Some(root) = legacy_root {
20919 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
20920 body["identity"]["project_root"] = serde_json::json!(root);
20921 frame.body = serde_json::to_vec(&body).unwrap();
20922 }
20923 let handler = self.handler.clone();
20924 let task_ctx = ctx.clone();
20925 let task = tokio::spawn(async move {
20926 handler
20927 .handle_control_frame(&task_ctx, frame)
20928 .await
20929 .unwrap()
20930 });
20931 let (_, module_rx) = self.modules.get_mut(target).unwrap();
20932 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
20933 .await
20934 .expect("the target receives the relayed route.bind")
20935 .unwrap()
20936 .frame;
20937 Relayed {
20938 target: target.to_string(),
20939 client: ctx,
20940 client_rx: rx,
20941 task,
20942 bind,
20943 }
20944 }
20945
20946 async fn ack(&self, relayed: &Relayed) {
20947 let (module, _) = &self.modules[&relayed.target];
20948 self.handler
20949 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
20950 .await
20951 .unwrap();
20952 }
20953
20954 async fn bound(
20956 &mut self,
20957 opener: Option<&str>,
20958 target: &str,
20959 scope: Option<ScopeSelector>,
20960 ) -> Bound {
20961 let relayed = self.relayed(opener, target, scope).await;
20962 self.commit_relayed(relayed).await
20963 }
20964
20965 async fn bound_with_legacy_root(
20966 &mut self,
20967 opener: Option<&str>,
20968 target: &str,
20969 scope: Option<ScopeSelector>,
20970 root: &std::path::Path,
20971 ) -> Bound {
20972 let relayed = self
20973 .relayed_with_legacy_root(opener, target, scope, Some(root))
20974 .await;
20975 self.commit_relayed(relayed).await
20976 }
20977
20978 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
20979 self.ack(&relayed).await;
20980 let Relayed {
20981 target,
20982 client,
20983 mut client_rx,
20984 task,
20985 bind,
20986 } = relayed;
20987 assert!(
20988 task.await.unwrap().is_empty(),
20989 "the open is answered by commit"
20990 );
20991 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
20992 Bound {
20993 target,
20994 client,
20995 client_rx,
20996 channel,
20997 epoch,
20998 bind,
20999 }
21000 }
21001
21002 fn live(&self, route: &Bound) -> bool {
21003 matches!(
21004 self.forwarding
21005 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21006 .unwrap(),
21007 DataRoute::Client(DataRouteState::Bound(_))
21008 )
21009 }
21010 }
21011
21012 struct Relayed {
21013 target: String,
21014 client: RouteCtx,
21015 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21016 task: tokio::task::JoinHandle<Vec<Frame>>,
21017 bind: Frame,
21018 }
21019
21020 struct Bound {
21021 target: String,
21022 client: RouteCtx,
21023 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21024 channel: u16,
21025 epoch: u32,
21026 bind: Frame,
21027 }
21028
21029 impl Bound {
21030 fn closed_reason(&mut self) -> RouteCloseReason {
21033 let mut reason = None;
21034 let mut goodbye = false;
21035 while let Ok(outbound) = self.client_rx.try_recv() {
21036 let frame = outbound.frame;
21037 match frame.header.ty {
21038 FrameType::Goodbye => {
21039 assert_eq!(
21040 (frame.header.channel, frame.header.epoch),
21041 (self.channel, self.epoch)
21042 );
21043 goodbye = true;
21044 }
21045 FrameType::Push => {
21046 let ClientControlPush::RouteClosed {
21047 reason: r,
21048 module_id,
21049 ..
21050 } = serde_json::from_slice(&frame.body).unwrap()
21051 else {
21052 panic!("unexpected push");
21053 };
21054 assert_eq!(module_id, self.target);
21055 reason = Some(r);
21056 }
21057 other => panic!("unexpected frame {other:?}"),
21058 }
21059 }
21060 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
21061 reason.expect("the client is told why the route closed")
21062 }
21063
21064 fn untouched(&mut self) -> bool {
21065 self.client_rx.try_recv().is_err()
21066 }
21067
21068 fn stamp(&self) -> Option<ScopeStamp> {
21069 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
21070 ModuleControlRequest::RouteBind { scope, .. } => scope,
21071 other => panic!("expected a route.bind, got {other:?}"),
21072 }
21073 }
21074 }
21075
21076 #[tokio::test]
21077 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
21078 ) {
21079 let mut rig = rig().await;
21080 let mut record = session(1);
21081 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
21082 record.child_owners = vec![Principal::Reserved {
21083 module_id: MAGIC.to_string(),
21084 }];
21085 rig.sync(vec![record]).await;
21086 let scope = || Some(rig_selector("s", Some(1)));
21087
21088 rig.bound(Some(OWNER), PLEXUS, scope()).await;
21091 rig.bound(Some(AFT), OTHER, scope()).await;
21092 rig.bound(Some(BROCA), PLEXUS, scope()).await;
21093
21094 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
21098 assert_eq!(
21099 rig.refused(opener, target, scope()).await,
21100 error_codes::SCOPE_NOT_CARRIER,
21101 "{opener:?} -> {target}"
21102 );
21103 }
21104 }
21105
21106 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
21107 ScopeSelector::new(
21108 Principal::Reserved {
21109 module_id: OWNER.to_string(),
21110 },
21111 scope_ref.to_string(),
21112 scope_epoch,
21113 )
21114 }
21115
21116 #[tokio::test]
21117 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
21118 let mut rig = rig().await;
21119 rig.sync(vec![session(1)]).await;
21120 for opener in [OWNER, AFT] {
21121 assert_eq!(
21122 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
21123 .await,
21124 error_codes::SCOPE_EPOCH_REQUIRED,
21125 "{opener}"
21126 );
21127 }
21128 }
21129
21130 #[tokio::test]
21131 async fn admission_separates_not_synced_not_live_and_ended() {
21132 let mut rig = rig().await;
21133 let code = rig
21135 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21136 .await;
21137 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
21138 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
21139
21140 let ghost = ScopeSelector::new(
21142 Principal::Reserved {
21143 module_id: "ghost".to_string(),
21144 },
21145 "s".to_string(),
21146 Some(1),
21147 );
21148 assert_eq!(
21149 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
21150 error_codes::SCOPE_NOT_LIVE
21151 );
21152
21153 rig.sync(vec![session(2)]).await;
21154 assert_eq!(
21155 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
21156 .await,
21157 error_codes::SCOPE_NOT_LIVE
21158 );
21159 for epoch in [1, 3] {
21160 assert_eq!(
21161 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
21162 .await,
21163 error_codes::SCOPE_ENDED,
21164 "epoch {epoch}"
21165 );
21166 }
21167 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
21169 .await;
21170 }
21171
21172 #[tokio::test]
21173 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
21174 let mut rig = rig().await;
21175 rig.sync(vec![session(1)]).await;
21176 let route = rig
21177 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21178 .await;
21179 let stamp = route.stamp().expect("a scoped bind carries the stamp");
21180 assert_eq!(stamp.scope_ref, "s");
21181 assert_eq!(stamp.scope_epoch, 1);
21182 assert_eq!(stamp.kind, ScopeKind::Head);
21183 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
21184 assert!(stamp.attributes.delegates);
21185 assert!(stamp.owner_authorized);
21186 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21187 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
21188
21189 let (broca, mut broca_rx) = wide_ctx(50);
21192 hello_via_sink(
21193 &rig.handler,
21194 &broca,
21195 &mut broca_rx,
21196 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
21197 )
21198 .await;
21199 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
21200 .await
21201 .unwrap();
21202 let own = ScopeSelector::new(
21203 Principal::Reserved {
21204 module_id: BROCA.to_string(),
21205 },
21206 "b".to_string(),
21207 Some(1),
21208 );
21209 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
21210 assert!(!route.stamp().unwrap().owner_authorized);
21211 }
21212
21213 #[tokio::test]
21214 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
21215 let mut rig = rig().await;
21216 let mut record = head("s", 1);
21217 record.carriers = vec![carrier(AFT, None)];
21218 let flow_id = "Flow:run-7/step_2!~";
21219 record.attributes.flow_id = Some(flow_id.to_string());
21220 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
21221 .await
21222 .unwrap();
21223 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
21224 panic!("not a sync reply");
21225 };
21226 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
21227 let route = rig
21228 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21229 .await;
21230 assert!(rig.live(&route), "the stamped bind committed");
21231 let stamp = route.stamp().expect("a flow scope carries a stamp");
21232 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
21233 assert_eq!(stamp.attributes.agent_id, None);
21234 assert!(!stamp.attributes.delegates);
21235 assert!(stamp.owner_authorized);
21236 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21237 assert_eq!(unscoped.stamp(), None);
21238 }
21239
21240 #[tokio::test]
21241 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
21242 let mut rig = rig_with_flow_support(false).await;
21243 let mut record = session(1);
21244 record.attributes.flow_id = Some("flow:7".to_string());
21245 rig.sync(vec![record]).await;
21246 for opener in [OWNER, AFT] {
21247 let body = rig
21248 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21249 .await;
21250 assert_eq!(body["code"], "target_flow_unsupported");
21251 let message = body["message"].as_str().unwrap();
21252 for required in [PLEXUS, "flow-scopes/v1"] {
21253 assert!(message.contains(required), "{message}");
21254 }
21255 }
21256 }
21257
21258 #[tokio::test]
21259 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
21260 let mut rig = rig_with_flow_support(true).await;
21261 let mut record = session(1);
21262 record.attributes.flow_id = Some("flow:7".to_string());
21263 rig.sync(vec![record]).await;
21264 for opener in [OWNER, AFT] {
21265 let route = rig
21266 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21267 .await;
21268 assert!(rig.live(&route));
21269 assert_eq!(
21270 route.stamp().unwrap().attributes.flow_id.as_deref(),
21271 Some("flow:7")
21272 );
21273 }
21274 }
21275
21276 #[tokio::test]
21277 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
21278 use std::future::Future;
21279
21280 let mut rig = rig_with_flow_support(true).await;
21281 let mut record = session(1);
21282 record.attributes.flow_id = Some("flow:7".to_string());
21283 rig.sync(vec![record]).await;
21284 let (client, mut client_rx, frame) =
21285 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
21286 for _ in 0..64 {
21289 client.egress.try_send(route_bind_ack(1)).unwrap();
21290 }
21291 let handler = rig.handler.clone();
21292 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21293 std::future::poll_fn(|cx| {
21294 assert!(open.as_mut().poll(cx).is_pending());
21295 std::task::Poll::Ready(())
21296 })
21297 .await;
21298 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21299
21300 let old_connection = rig.modules[PLEXUS].0.connection_id;
21301 rig.handler.cleanup_connection(old_connection).unwrap();
21302 let (replacement, mut replacement_rx) = wide_ctx(200);
21303 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
21304 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
21305 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
21306 let hello = Frame::build(
21307 FrameType::Hello,
21308 control_flags(),
21309 0,
21310 0,
21311 200,
21312 serde_json::to_vec(&body).unwrap(),
21313 )
21314 .unwrap();
21315 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
21316
21317 client_rx.try_recv().unwrap();
21318 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21319 .await
21320 .expect("the replacement is refused without waiting for a bind ack")
21321 .unwrap();
21322 assert_eq!(replies.len(), 1);
21323 let body = parse_error(&replies[0]);
21324 assert_eq!(body["code"], "target_flow_unsupported");
21325 for required in [PLEXUS, "flow-scopes/v1"] {
21326 assert!(body["message"].as_str().unwrap().contains(required));
21327 }
21328 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
21329 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21330 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21331 assert!(rig
21332 .handler
21333 .registry
21334 .get_module_by_connection(replacement.connection_id)
21335 .unwrap()
21336 .is_some());
21337 }
21338
21339 #[tokio::test]
21340 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
21341 use std::future::Future;
21342 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
21343 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21344 const NEXT_HASH: &str =
21345 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21346 let mut rig = rig().await;
21347 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
21348 record.carriers.push(ScopeCarrier::new(Principal::Direct));
21349 rig.sync(vec![record.clone()]).await;
21350 let (client, mut rx, frame) = rig.open_frame(
21351 None,
21352 PLEXUS,
21353 Some(
21354 rig.selector("s", Some(1))
21355 .with_bind_proof(Some(PROOF.into())),
21356 ),
21357 );
21358 for _ in 0..64 {
21361 client.egress.try_send(route_bind_ack(1)).unwrap();
21362 }
21363 let handler = rig.handler.clone();
21364 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21365 std::future::poll_fn(|cx| {
21366 assert!(open.as_mut().poll(cx).is_pending());
21367 std::task::Poll::Ready(())
21368 })
21369 .await;
21370 record.bind_proof_sha256 = Some(NEXT_HASH.into());
21371 rig.sync(vec![record]).await;
21372 rx.try_recv().unwrap();
21373 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21374 .await
21375 .unwrap()
21376 .unwrap();
21377 assert_eq!(
21378 parse_error(&replies[0])["code"],
21379 error_codes::SCOPE_BIND_PROOF_MISMATCH
21380 );
21381 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
21382 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21383 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21384 }
21385
21386 #[tokio::test]
21387 async fn scope_bind_proof_reserved_openers_ignore_proof() {
21388 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21389 let mut rig = rig().await;
21390 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21391 .await;
21392 for opener in [OWNER, AFT] {
21393 for proof in [None, Some("malformed+proof=")] {
21394 let route = rig
21395 .bound(
21396 Some(opener),
21397 PLEXUS,
21398 Some(
21399 rig.selector("s", Some(1))
21400 .with_bind_proof(proof.map(str::to_string)),
21401 ),
21402 )
21403 .await;
21404 assert!(rig.live(&route));
21405 }
21406 }
21407 }
21408
21409 #[tokio::test]
21410 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
21411 use std::future::Future;
21412 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21413 const NEXT_HASH: &str =
21414 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21415 let mut rig = rig().await;
21416 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21417 .await;
21418 let (client, mut rx, frame) =
21419 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
21420 for _ in 0..64 {
21421 client.egress.try_send(route_bind_ack(1)).unwrap();
21422 }
21423 let handler = rig.handler.clone();
21424 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21425 std::future::poll_fn(|cx| {
21426 assert!(open.as_mut().poll(cx).is_pending());
21427 std::task::Poll::Ready(())
21428 })
21429 .await;
21430 rig.sync(vec![
21431 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
21432 ])
21433 .await;
21434 rx.try_recv().unwrap();
21435 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21436 let bind = std::future::poll_fn(|cx| {
21437 assert!(open.as_mut().poll(cx).is_pending());
21438 module_rx.poll_recv(cx)
21439 })
21440 .await
21441 .unwrap()
21442 .frame;
21443 handler
21444 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
21445 .await
21446 .unwrap();
21447 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21448 .await
21449 .unwrap()
21450 .unwrap();
21451 assert_eq!(replies.len(), 1);
21452 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
21455 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21456 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
21457 let route = rig
21458 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
21459 .await;
21460 assert!(rig.live(&route));
21461 }
21462
21463 #[tokio::test]
21464 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
21465 {
21466 let mut rig = rig_with_flow_support(false).await;
21467 rig.sync(vec![session(1)]).await;
21468 let route = rig
21469 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21470 .await;
21471 assert!(rig.live(&route));
21472 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
21473 let mut record = session(1);
21474 record.attributes.flow_id = Some("flow:7".to_string());
21475 rig.sync(vec![record]).await;
21476 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
21477 assert!(rig.live(&unscoped));
21478 assert_eq!(unscoped.stamp(), None);
21479 }
21480
21481 #[tokio::test]
21482 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
21483 let mut rig = rig().await;
21484 let mut record = session(1);
21485 record.attributes.flow_id = Some("flow:7".to_string());
21486 rig.sync(vec![record.clone()]).await;
21487 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21488 let mut owner_route = rig
21489 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21490 .await;
21491 let mut carrier_route = rig
21492 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
21493 .await;
21494 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21495 rig.sync(vec![record.clone()]).await;
21496 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21497 assert!(rig.live(&owner_route) && owner_route.untouched());
21498 assert!(rig.live(&carrier_route) && carrier_route.untouched());
21499
21500 record.attributes.flow_id = Some("flow:8".to_string());
21501 rig.sync(vec![record]).await;
21502 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
21503 let before = before.unwrap();
21504 assert_eq!(after.scope_epoch, before.scope_epoch);
21505 assert!(after.version > before.version);
21506 for route in [&mut owner_route, &mut carrier_route] {
21507 assert!(!rig.live(route));
21508 assert_eq!(
21509 route.closed_reason(),
21510 RouteCloseReason::ScopeDelegationChanged
21511 );
21512 }
21513 assert!(rig.live(&unscoped) && unscoped.untouched());
21514 for target in [PLEXUS, OTHER] {
21517 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
21518 let goodbye = module_rx
21519 .try_recv()
21520 .expect("the provider sees the drain")
21521 .frame;
21522 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21523 assert!(module_rx.try_recv().is_err());
21524 }
21525 let rebound = rig
21526 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21527 .await;
21528 assert_eq!(
21529 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
21530 Some("flow:8")
21531 );
21532 }
21533
21534 #[tokio::test]
21539 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
21540 let mut rig = rig().await;
21541 rig.sync(vec![session(1)]).await;
21542 let mut cotenant = rig
21543 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21544 .await;
21545
21546 let mut changed = session(1);
21547 changed.child_owners.push(Principal::Reserved {
21548 module_id: MAGIC.to_string(),
21549 });
21550 let mut ended = None;
21551 for (code, next) in [
21552 (error_codes::SCOPE_CHANGED, vec![changed]),
21553 (error_codes::SCOPE_ENDED, Vec::new()),
21554 ] {
21555 let relayed = rig
21556 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21557 .await;
21558 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
21559 ended = Some(next.is_empty());
21560 rig.sync(next).await;
21561 rig.ack(&relayed).await;
21562 let replies = relayed.task.await.unwrap();
21563 assert_eq!(replies.len(), 1, "{replies:?}");
21564 assert_eq!(parse_error(&replies[0])["code"], code);
21565 assert_eq!(
21566 subc_protocol::error_codes::is_retryable_route_open(code),
21567 code == error_codes::SCOPE_CHANGED
21568 );
21569 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21570 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21572 let mut goodbyes = Vec::new();
21575 while let Ok(outbound) = plexus_rx.try_recv() {
21576 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
21577 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
21578 }
21579 assert!(
21580 goodbyes.contains(&(bind_channel, bind_epoch)),
21581 "{goodbyes:?}"
21582 );
21583 assert!(rig
21584 .handler
21585 .registry
21586 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
21587 .unwrap()
21588 .is_some());
21589 }
21590 assert_eq!(ended, Some(true));
21591 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
21594 }
21595
21596 #[tokio::test]
21599 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
21600 struct Case {
21601 name: &'static str,
21602 change: fn(&mut ScopeRecord),
21603 closed: [Option<RouteCloseReason>; 4],
21606 }
21607 use RouteCloseReason::*;
21608 let cases = [
21609 Case {
21610 name: "a carrier entry removed",
21611 change: |r| {
21612 r.carriers.retain(|c| {
21613 c.principal
21614 != Principal::Reserved {
21615 module_id: AFT.to_string(),
21616 }
21617 })
21618 },
21619 closed: [None, Some(ScopeCarrierRemoved), None, None],
21620 },
21621 Case {
21622 name: "a target removed from a carrier",
21623 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
21624 closed: [None, None, None, Some(ScopeCarrierRemoved)],
21625 },
21626 Case {
21627 name: "a bare carrier narrowed to targets",
21628 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
21629 closed: [None, Some(ScopeCarrierRemoved), None, None],
21630 },
21631 Case {
21632 name: "delegates turned off",
21633 change: |r| r.attributes.delegates = false,
21634 closed: [Some(ScopeDelegationChanged); 4],
21635 },
21636 Case {
21637 name: "agent_id changed",
21638 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
21639 closed: [Some(ScopeDelegationChanged); 4],
21640 },
21641 Case {
21642 name: "a carrier added, child owners changed, the record re-sent",
21643 change: |r| {
21644 r.carriers.push(carrier(MAGIC, None));
21645 r.child_owners.push(Principal::Reserved {
21646 module_id: MAGIC.to_string(),
21647 });
21648 },
21649 closed: [None; 4],
21650 },
21651 Case {
21652 name: "a target added",
21653 change: |r| {
21654 r.carriers[1]
21655 .targets
21656 .as_mut()
21657 .unwrap()
21658 .push("third".to_string())
21659 },
21660 closed: [None; 4],
21661 },
21662 Case {
21663 name: "delegates turned on",
21664 change: |r| r.attributes.delegates = true,
21665 closed: [None; 4],
21666 },
21667 ];
21668 for case in cases {
21669 let mut rig = rig().await;
21670 rig.sync(vec![session(1)]).await;
21671 let scope = || Some(rig_selector("s", Some(1)));
21672 let mut routes = [
21673 rig.bound(Some(OWNER), PLEXUS, scope()).await,
21674 rig.bound(Some(AFT), PLEXUS, scope()).await,
21675 rig.bound(Some(BROCA), PLEXUS, scope()).await,
21676 rig.bound(Some(BROCA), OTHER, scope()).await,
21677 ];
21678 let mut record = session(1);
21679 (case.change)(&mut record);
21680 rig.sync(vec![record]).await;
21681 for (index, expected) in case.closed.iter().enumerate() {
21682 let route = &mut routes[index];
21683 match expected {
21684 Some(reason) => {
21685 assert!(!rig.live(route), "{}: route {index} still live", case.name);
21686 assert_eq!(
21687 route.closed_reason(),
21688 *reason,
21689 "{}: route {index}",
21690 case.name
21691 );
21692 }
21693 None => {
21694 assert!(rig.live(route), "{}: route {index} closed", case.name);
21695 assert!(
21696 route.untouched(),
21697 "{}: route {index} was told something",
21698 case.name
21699 );
21700 }
21701 }
21702 }
21703 }
21704 }
21705
21706 #[tokio::test]
21707 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
21708 for next in [Vec::new(), vec![session(2)]] {
21710 let mut rig = rig().await;
21711 rig.sync(vec![session(1)]).await;
21712 let mut route = rig
21713 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21714 .await;
21715 rig.sync(next).await;
21716 assert!(!rig.live(&route));
21717 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
21718 }
21719
21720 let mut rig = rig().await;
21723 let mut child = session(1);
21724 child.scope_ref = "child".to_string();
21725 child.kind = ScopeKind::Worker;
21726 child.parent = Some(ScopeParent::new(
21727 Principal::Reserved {
21728 module_id: OWNER.to_string(),
21729 },
21730 "s".to_string(),
21731 1,
21732 ));
21733 rig.sync(vec![session(1), child.clone()]).await;
21734 let mut child_route = rig
21735 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
21736 .await;
21737 assert_eq!(
21738 child_route.stamp().unwrap().parent_state,
21739 Some(ParentState::Linked)
21740 );
21741 rig.sync(vec![child]).await;
21742 assert!(!rig.live(&child_route));
21743 assert_eq!(
21744 child_route.closed_reason(),
21745 RouteCloseReason::ScopeParentEnded
21746 );
21747 }
21748
21749 #[tokio::test]
21750 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
21751 ) {
21752 let mut rig = rig().await;
21753 rig.sync(vec![session(1)]).await;
21754 let mut route = rig
21755 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21756 .await;
21757 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21758 rig.sync(vec![session(1)]).await;
21759 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21760 assert!(rig.live(&route) && route.untouched());
21761
21762 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
21766 .forwarding
21767 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21768 .unwrap()
21769 else {
21770 panic!("the route is bound");
21771 };
21772 binding.flow.acquire_tagged(9, false).await.unwrap();
21773 let mut widened = session(1);
21774 widened.carriers.push(carrier(MAGIC, None));
21775 rig.sync(vec![widened]).await;
21776 assert!(rig.live(&route) && route.untouched());
21777 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21778 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
21779 assert_eq!(binding.flow.in_flight(), 1);
21782 binding
21783 .flow
21784 .acquire_tagged(10, false)
21785 .await
21786 .expect("the flow is still open");
21787 }
21788
21789 #[tokio::test]
21792 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
21793 let mut rig = rig().await;
21794 rig.sync(vec![session(1)]).await;
21795 let mut on_incumbent = rig
21796 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21797 .await;
21798
21799 let (candidate, _candidate_rx) = wide_ctx(9);
21802 let registration = rig
21803 .handler
21804 .registry
21805 .register_candidate_with_control_ops(
21806 manifest(PLEXUS, PROTOCOL_VERSION),
21807 PROTOCOL_VERSION,
21808 candidate.connection_id,
21809 module_baseline_control_ops(),
21810 )
21811 .unwrap();
21812 rig.forwarding
21813 .register_candidate_module_connection(
21814 candidate.connection_id,
21815 PLEXUS.to_string(),
21816 PROTOCOL_VERSION,
21817 manifest_concurrency(®istration.manifest),
21818 candidate.egress.clone(),
21819 )
21820 .unwrap();
21821 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
21822 rig.handler
21823 .registry
21824 .promote_candidate(PLEXUS)
21825 .unwrap()
21826 .unwrap();
21827 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
21828
21829 rig.sync(Vec::new()).await;
21830 assert!(!rig.live(&on_incumbent));
21831 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
21832 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21833 let goodbye = incumbent_rx
21834 .try_recv()
21835 .expect("the superseded endpoint is told")
21836 .frame;
21837 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21838 }
21839 }
21840}
21841
21842#[cfg(test)]
21843mod concurrency_default_exposure_tests {
21844 use super::*;
21845
21846 fn hello_body(role_json: &str) -> Vec<u8> {
21847 format!(
21848 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
21849 )
21850 .into_bytes()
21851 }
21852
21853 fn manifest_from(body: &[u8]) -> ModuleManifest {
21854 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
21855 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
21856 .expect("manifest parses")
21857 }
21858
21859 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
21860
21861 #[test]
21862 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
21863 let body = hello_body(&format!(
21864 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
21865 ));
21866 let manifest = manifest_from(&body);
21867 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21870 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
21871 }
21872
21873 #[test]
21874 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
21875 let body = hello_body(&format!(
21876 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
21877 ));
21878 let manifest = manifest_from(&body);
21879 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21880 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21881 }
21882
21883 #[test]
21884 fn non_management_roles_are_never_reported() {
21885 let body = hello_body(
21886 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
21887 );
21888 let manifest = manifest_from(&body);
21889 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21890 }
21891}