1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26 error_codes,
27 manifest::{
28 validate_hello_capability_grammar, validate_hello_confirm_operations,
29 validate_hello_display_name, validate_hello_event_declarations,
30 validate_hello_flow_operations, validate_hello_operation_deferral,
31 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32 validate_hello_self_signal_declarations, validate_role_declarations_value,
33 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34 ProviderRole,
35 },
36 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
37 provider_declarations::ProviderDeclarations,
38 scope::{
39 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
40 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
41 SCOPE_SYNC_OP,
42 },
43 session::{
44 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
45 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
46 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
47 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
48 },
49 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
50 Priority, RouteTarget, PROTOCOL_VERSION,
51};
52use tokio::time::{timeout_at, Instant};
53use tracing::{debug, info, instrument::WithSubscriber, warn};
54
55use crate::{
56 caller_identity::CallerIdentity,
57 capability_requirements::{
58 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
59 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
60 RequirementStatus, RuntimeModule,
61 },
62 child_roster::DaemonShutdownFlag,
63 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
64 forwarding::{
65 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
66 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
67 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
68 },
69 observability::{
70 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
71 },
72 privileged_prompt::{prompt_text, PrivilegedOperation},
73 provenance::{
74 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
75 },
76 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
77 router::{daemon_error_flags, RouteCtx, RouterError},
78 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
79 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
80 stderr_tail::{CaptureState, TailEntry},
81 supervise::{
82 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
83 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
84 },
85 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
86};
87
88pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
94
95const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
96const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
97const CAP_PING_PONG: &str = "ping_pong_v1";
98const CAP_SESSION_ATTACH: &str = "session_attach_v1";
99const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
100
101const SUBC_CONTROL_OPS: &[&str] = &[
102 "claim.mint",
103 "claim.present",
104 ops::DAEMON_SHUTDOWN_INTENT,
105 ops::ACTIVITY_RECORD,
106 ops::ACTIVITY_QUERY,
107 ops::ACTIVITY_SUBSCRIBE,
108 ops::SERVER_DESCRIBE,
109 ops::CATALOG_LIST,
110 ops::CATALOG_WATCH,
111 ops::ROUTE_OPEN,
112 ops::ROUTE_POLL,
113 ops::ROUTE_CLOSING,
114 ops::ROUTE_CLOSED,
115 ops::SUPERVISOR_LIST,
116 ops::SUPERVISOR_RESTART,
117 ops::SUPERVISOR_SWAP,
118 ops::SUPERVISOR_RELOAD,
119 ops::SUPERVISOR_RESCAN,
120 ops::SUPERVISOR_RELEASE_RESERVED,
121 ops::SUPERVISOR_SET_ENABLED,
122 ops::SUPERVISOR_HEALTH_PROBE,
123 ops::SUPERVISOR_HEALTH,
124 ops::SUPERVISOR_STDERR_TAIL,
125 ops::SUPERVISOR_TERMINALS,
126 ops::SUPERVISOR_ROUTES,
127 ops::SUPERVISOR_PROVENANCE,
128 ops::SUPERVISOR_SPAWN_SNAPSHOT,
129 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
130];
131
132const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[
139 ops::SUPERVISOR_RESCAN_CHECKED,
140 "config.status",
141 "config.plan",
142 "config.apply",
143 "config.stamp",
144 "config.unstamp",
145 "config.history",
146 "config.rollback",
147];
148
149mod caller_attestation;
150mod channel_operations;
151#[cfg(all(test, unix, feature = "test-support"))]
152#[path = "control/channel_operations_tests.rs"]
153mod channel_operations_tests;
154pub(crate) mod claims;
155#[cfg(test)]
156#[path = "control/config_advertisement_tests.rs"]
157mod config_advertisement_tests;
158mod config_gate;
159use caller_attestation::{caller_for_target, scope_acts_for_another};
160#[cfg(all(test, unix, feature = "test-support"))]
161#[path = "control/caller_attestation_tests.rs"]
162mod caller_attestation_tests;
163mod scope_expiry;
164
165mod agent_commands;
166pub(crate) mod catalog;
167mod rescan_checked;
168use rescan_checked::{RescanMode, RescanPlan};
169
170const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
171 "claim.mint",
172 "claim.present",
173 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
174 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
175 "supervisor.live_roots",
176 "supervisor.agent_commands",
177 SCOPE_SYNC_OP,
178 SCOPE_APPLY_OP,
179 SCOPE_DESCRIBE_OP,
180 subc_protocol::scope::ROUTE_AUTHORIZE_OP,
181 "operator.confirm",
182];
183
184const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
188
189const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
190
191pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
199
200pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
215
216pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
234
235const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
236const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
237
238fn reload_verdict(
239 configured: &Path,
240 spawned_from: Option<&Path>,
241 image: subc_control::RunningImageAgreement,
242) -> PendingReloadVerdict {
243 let path = match spawned_from {
244 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
245 ReloadPathAgreement::Match
246 }
247 Some(spawned_from) => ReloadPathAgreement::Mismatch {
248 configured: configured.to_path_buf(),
249 spawned_from: spawned_from.to_path_buf(),
250 },
251 None => ReloadPathAgreement::Unavailable {
252 reason: if matches!(
253 image,
254 subc_control::RunningImageAgreement::Unavailable {
255 reason: subc_control::RunningImageUnavailableReason::NotRunning
256 }
257 ) {
258 ReloadPathUnavailableReason::NotRunning
259 } else {
260 ReloadPathUnavailableReason::SpawnedPathUnavailable
261 },
262 },
263 };
264 PendingReloadVerdict::new(path, image)
265}
266
267fn launch_spec_agreement(
268 configured: &crate::supervise::ModuleSpec,
269 running: bool,
270 spawned: Option<&crate::supervise::LaunchSpecDigest>,
271) -> subc_control::LaunchSpecAgreement {
272 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
273 if !running {
274 return Agreement::Unavailable {
275 reason: Reason::NotRunning,
276 };
277 }
278 match spawned {
279 None => Agreement::Unavailable {
280 reason: Reason::SpawnedSpecUnavailable,
281 },
282 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
283 Agreement::Match
284 }
285 Some(_) => Agreement::Mismatch,
286 }
287}
288
289fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
290 if configured == spawned_from {
291 return true;
292 }
293 #[cfg(windows)]
294 {
295 match (
301 std::fs::canonicalize(configured),
302 std::fs::canonicalize(spawned_from),
303 ) {
304 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
305 _ => false,
306 }
307 }
308 #[cfg(not(windows))]
309 {
310 false
311 }
312}
313
314#[derive(Clone)]
315struct DaemonProvenanceFacts {
316 build: DaemonBuildProvenance,
317 pid: Option<u32>,
318 started_at_ms: Option<u64>,
319 start_clock: Option<crate::clock::StartClock>,
320 executable_path: Option<PathBuf>,
321 executable_identity: Option<SpawnedFileIdentity>,
322 process_start_time: Option<u64>,
323 probe: ExecutableIdentityProbe,
324}
325
326impl Default for DaemonProvenanceFacts {
327 fn default() -> Self {
328 Self {
329 build: DaemonBuildProvenance {
330 build_git_sha: None,
331 build_lock_digest: None,
332 },
333 pid: None,
334 started_at_ms: None,
335 start_clock: None,
336 executable_path: None,
337 executable_identity: None,
338 process_start_time: None,
339 probe: ExecutableIdentityProbe::default(),
340 }
341 }
342}
343
344#[derive(Debug, Clone)]
345struct SupervisorRescanContext {
346 supervisor: Supervisor,
347 config_path: PathBuf,
348 configured_port: Option<u16>,
349 storage_config: Option<crate::daemon_config::StorageConfig>,
350 admission_facts_carrier_module_id: Option<String>,
351 admission_facts_targets: Option<Vec<String>>,
352 scope_authority_owners: Vec<String>,
353 federated_scope_owner: String,
354 privileged_operator_prompt: PrivilegedOperatorPrompt,
355 generation: Arc<std::sync::atomic::AtomicU64>,
361}
362
363const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
384 "reloading",
385 "supervisor_not_live",
386 "registration_not_active",
387 "no_forwarding_connection",
388 "relay_send_failed",
389];
390
391#[cfg(test)]
393struct ActivityRequestPause {
394 connection: ConnectionId,
395 started: tokio::sync::Notify,
396 cancelled: tokio::sync::Notify,
397 cleanup_started: std::sync::atomic::AtomicBool,
398 cancelled_after_cleanup: std::sync::atomic::AtomicBool,
399}
400
401#[cfg(test)]
402struct ActivityRequestCancellation(Arc<ActivityRequestPause>);
403
404#[cfg(test)]
405impl Drop for ActivityRequestCancellation {
406 fn drop(&mut self) {
407 self.0.cancelled_after_cleanup.store(
408 self.0
409 .cleanup_started
410 .load(std::sync::atomic::Ordering::Acquire),
411 std::sync::atomic::Ordering::Release,
412 );
413 self.0.cancelled.notify_one();
414 }
415}
416
417#[cfg(test)]
419type TestPause = (
420 tokio::sync::oneshot::Sender<()>,
421 std::sync::mpsc::Receiver<()>,
422);
423
424#[cfg(test)]
426type TestHook = Box<dyn FnOnce() + Send>;
427
428#[cfg(test)]
430type AsyncTestPause = (
431 tokio::sync::oneshot::Sender<()>,
432 tokio::sync::oneshot::Receiver<()>,
433);
434
435#[derive(Clone)]
437pub struct ControlHandler {
438 #[cfg(test)]
439 activity_request_pause: Arc<Mutex<Option<Arc<ActivityRequestPause>>>>,
440 #[cfg(test)]
444 cleanup_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
445 #[cfg(test)]
448 after_nonce_attestation: Arc<Mutex<Option<TestHook>>>,
449 #[cfg(test)]
452 hello_ingress_pause: Arc<Mutex<Option<AsyncTestPause>>>,
453 #[cfg(test)]
455 agent_commands_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
456 claims: Arc<claims::Claims>,
457 registry: Arc<Registry>,
458 config_runtime: Option<Arc<crate::config_gate::runtime::RuntimeOwner>>,
459 config_review_admission: Arc<dyn crate::config_gate::ReviewAdmission>,
460 config_review_admission_is_default: bool,
461 activity: Option<crate::activity::ActivityLedger>,
462 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
463 catalog_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
464 forwarding: Arc<ForwardingTable>,
465 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
466 supervisor: SupervisorHandle,
467 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
468 agent_declarations: agent_commands::Queues,
469 list_admission: Arc<tokio::sync::Semaphore>,
470 subc_capabilities: Arc<[String]>,
471 route_bind_relay_timeout: Duration,
475 route_bind_relay_timeouts: BTreeMap<String, Duration>,
479 route_bind_breakers: RouteBindBreakers,
482 route_bind_concurrency: RouteBindConcurrency,
485 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
490 route_bind_breaker_threshold: u32,
492 route_bind_breaker_cooldown: Duration,
494 health_probe_timeout: Duration,
495 storage_config: Option<crate::daemon_config::StorageConfig>,
498 machine_id: Option<crate::machine_id::MachineId>,
502 admission_facts_carrier_module_id: Option<String>,
503 admission_facts_targets: Option<Vec<String>>,
504 scopes: Arc<RwLock<ScopeTable>>,
508 scope_deadline_added: Arc<tokio::sync::Notify>,
512 scope_authority_owners: Vec<String>,
515 federated_scope_owner: String,
516 privileged_operator_prompt: PrivilegedOperatorPrompt,
518 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
521 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
522 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
523 captured_peers: Arc<Mutex<HashMap<ConnectionId, subc_os::local_peer::CapturedPeer>>>,
524 config_desktop_stamps:
525 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
526 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
527 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
528 pub(crate) operator_apps:
529 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
530 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
535 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
541 #[cfg(test)]
543 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
544 #[cfg(test)]
545 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
546 #[cfg(test)]
547 recovery_exit_test_after_step: Option<fn(usize) -> Result<(), crate::config_gate::GateError>>,
548 #[cfg(test)]
549 recovery_exit_test_completed: Option<fn(u64)>,
550 #[cfg(test)]
551 apply_reconcile_pause: Arc<Mutex<Option<AsyncTestPause>>>,
552 #[cfg(test)]
553 recovery_exit_test_quiet_retry: bool,
554 #[cfg(test)]
555 recovery_exit_test_retry_pause: Option<fn(u64) -> Option<PathBuf>>,
556 #[cfg(test)]
557 config_delivery_test_wakes:
558 Option<tokio::sync::mpsc::UnboundedSender<(&'static str, tokio::time::Instant)>>,
559 #[cfg(all(test, unix))]
560 config_delivery_test_idle:
561 Option<tokio::sync::mpsc::UnboundedSender<tokio::sync::oneshot::Sender<()>>>,
562 #[allow(dead_code)]
566 #[cfg(windows)]
567 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
568 #[allow(dead_code)]
569 #[cfg(windows)]
570 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
571 #[allow(dead_code)]
575 #[cfg(windows)]
576 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
577 #[cfg(all(test, windows))]
580 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
581 #[cfg(test)]
582 caller_lookup: Option<TestCallerLookup>,
583 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
586 rescan: Option<SupervisorRescanContext>,
587 connected_clients: ConnectedClients,
588 counters: DaemonCounters,
589 local_endpoint_unavailable: Option<String>,
592 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
593 daemon_shutdown: DaemonShutdownFlag,
598 daemon_provenance: DaemonProvenanceFacts,
599 #[cfg(test)]
600 control_dispatch_delay: Option<Duration>,
601 #[cfg(test)]
602 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
603 #[cfg(test)]
607 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
608 #[cfg(test)]
614 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
615 #[cfg(test)]
616 hello_ack_to_publication_hold: Option<Arc<std::sync::Barrier>>,
617 #[cfg(test)]
618 hello_before_publication_hold: Option<Arc<std::sync::Barrier>>,
619 #[cfg(test)]
620 hello_after_launch_check_hold: Option<Arc<std::sync::Barrier>>,
621 #[cfg(test)]
622 hello_inside_publication_hold: Option<Arc<std::sync::Barrier>>,
623}
624
625impl fmt::Debug for ControlHandler {
626 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
627 f.debug_struct("ControlHandler")
628 .field("registry", &self.registry)
629 .field("forwarding", &self.forwarding)
630 .field("process_liveness", &self.process_liveness.is_some())
631 .field("supervisor", &self.supervisor)
632 .field("subc_capabilities", &self.subc_capabilities)
633 .finish()
634 }
635}
636
637struct RouteOpenRequest {
638 target: RouteTarget,
639 identity: BindIdentity,
640 consumer_identity: Option<ConsumerIdentity>,
641 consumer_capabilities: Option<Vec<String>>,
642 role_versions: Option<BTreeMap<String, String>>,
643 busy_refusals: Option<bool>,
644 admission_facts: Option<serde_json::Value>,
645 scope: Option<ScopeSelector>,
646 accepts_close_reasons: Option<Vec<String>>,
647}
648
649#[derive(Clone, Copy)]
650enum RouteOpenPhase {
651 Admission,
652 ProjectRoot,
653 EgressReservation,
654 BindSend,
655 BindAckWait,
656}
657
658struct RouteOpenTimings {
662 module_id: String,
663 principal: String,
664 admission: Duration,
665 project_root: Duration,
666 egress_reservation: Duration,
667 bind_send: Duration,
668 bind_ack_wait: Duration,
669 phase: Option<RouteOpenPhase>,
670 phase_started_at: Instant,
671}
672
673impl RouteOpenTimings {
674 fn new() -> Self {
675 Self {
676 module_id: String::new(),
677 principal: "unresolved".to_string(),
679 admission: Duration::ZERO,
680 project_root: Duration::ZERO,
681 egress_reservation: Duration::ZERO,
682 bind_send: Duration::ZERO,
683 bind_ack_wait: Duration::ZERO,
684 phase: Some(RouteOpenPhase::Admission),
685 phase_started_at: Instant::now(),
686 }
687 }
688
689 fn finish(&mut self) {
690 let elapsed = self.phase_started_at.elapsed();
691 match self.phase.take() {
692 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
693 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
694 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
695 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
696 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
697 None => {}
698 }
699 }
700
701 fn enter(&mut self, phase: RouteOpenPhase) {
702 self.finish();
703 self.phase = Some(phase);
704 self.phase_started_at = Instant::now();
705 }
706}
707
708struct RouteBindReservationGuard {
709 forwarding: Arc<ForwardingTable>,
710 endpoint: ModuleEndpointId,
711 relay_corr: u64,
712 armed: bool,
713}
714
715struct ModuleControlRpcGuard {
716 forwarding: Arc<ForwardingTable>,
717 endpoint: ModuleEndpointId,
718 corr: u64,
719 armed: bool,
720}
721
722impl ModuleControlRpcGuard {
723 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
724 Self {
725 forwarding,
726 endpoint,
727 corr,
728 armed: true,
729 }
730 }
731
732 fn disarm(&mut self) {
733 self.armed = false;
734 }
735}
736
737impl Drop for ModuleControlRpcGuard {
738 fn drop(&mut self) {
739 if self.armed {
740 let _ = self
741 .forwarding
742 .cancel_module_control_rpc(self.endpoint, self.corr);
743 }
744 }
745}
746
747impl RouteBindReservationGuard {
748 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
749 Self {
750 forwarding,
751 endpoint,
752 relay_corr,
753 armed: true,
754 }
755 }
756
757 fn release_and_disarm(&mut self) {
758 if !self.armed {
759 return;
760 }
761 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
762 self.endpoint,
763 self.relay_corr,
764 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
765 ) {
766 send_goodbye_target_best_effort(
767 &self.forwarding.counters(),
768 &target,
769 "canceled route.bind",
770 );
771 }
772 self.armed = false;
773 }
774
775 fn disarm(&mut self) {
776 self.armed = false;
777 }
778}
779
780impl Drop for RouteBindReservationGuard {
781 fn drop(&mut self) {
782 self.release_and_disarm();
783 }
784}
785
786#[derive(Debug, Clone, Default)]
816pub(crate) struct RouteBindBreakers {
817 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
818}
819
820#[derive(Debug, Clone, Default)]
821pub(crate) struct RouteBindConcurrency {
822 modules: Arc<Mutex<HashMap<String, usize>>>,
823}
824
825struct RouteBindConcurrencyGuard {
826 concurrency: RouteBindConcurrency,
827 module_id: String,
828}
829
830impl RouteBindConcurrency {
831 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
834 let mut modules = self
835 .modules
836 .lock()
837 .expect("route.bind concurrency mutex poisoned");
838 let in_flight = modules.entry(module_id.to_string()).or_default();
839 if *in_flight >= limit {
840 return Err(*in_flight);
841 }
842 *in_flight += 1;
843 Ok(RouteBindConcurrencyGuard {
844 concurrency: self.clone(),
845 module_id: module_id.to_string(),
846 })
847 }
848}
849
850impl Drop for RouteBindConcurrencyGuard {
851 fn drop(&mut self) {
852 let mut modules = self
853 .concurrency
854 .modules
855 .lock()
856 .expect("route.bind concurrency mutex poisoned");
857 let remove = {
858 let in_flight = modules
859 .get_mut(&self.module_id)
860 .expect("admitted route.bind has a concurrency entry");
861 *in_flight -= 1;
862 *in_flight == 0
863 };
864 if remove {
865 modules.remove(&self.module_id);
866 }
867 }
868}
869
870#[derive(Debug, Default)]
871struct ModuleBreakerState {
872 consecutive_timeouts: u32,
874 cooldown_until: Option<Instant>,
877 probe_in_flight: Option<Arc<()>>,
882}
883
884enum RouteBindAdmission<'a> {
886 Admitted {
887 guard: RouteBindBreakerGuard<'a>,
888 probe: bool,
891 },
892 Refused {
893 consecutive_timeouts: u32,
894 retry_in: Duration,
898 probe_in_flight: bool,
899 },
900}
901
902struct RouteBindBreakerGuard<'a> {
909 breakers: RouteBindBreakers,
910 module_id: &'a str,
911 probe_token: Option<Arc<()>>,
912 settled: bool,
913}
914
915impl RouteBindBreakerGuard<'_> {
916 fn record_accepted(&mut self) -> bool {
920 self.settled = true;
921 self.breakers.record_accepted(self.module_id)
922 }
923
924 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
927 self.settled = true;
928 self.breakers.record_timeout(
929 self.module_id,
930 self.probe_token.as_ref(),
931 threshold,
932 cooldown,
933 )
934 }
935
936 fn record_inconclusive(&mut self) {
945 self.settled = true;
946 self.breakers
947 .record_inconclusive(self.module_id, self.probe_token.as_ref());
948 }
949}
950
951impl Drop for RouteBindBreakerGuard<'_> {
952 fn drop(&mut self) {
953 if !self.settled {
954 self.breakers
955 .record_inconclusive(self.module_id, self.probe_token.as_ref());
956 }
957 }
958}
959
960struct BreakerOpened {
964 consecutive_timeouts: u32,
965 reopened_after_probe: bool,
968}
969
970impl RouteBindBreakers {
971 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
972 self.modules
973 .lock()
974 .expect("route.bind breaker mutex poisoned")
975 }
976
977 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
980 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
981 probe: probe_token.is_some(),
982 guard: RouteBindBreakerGuard {
983 breakers: self.clone(),
984 module_id,
985 probe_token,
986 settled: false,
987 },
988 };
989
990 let mut modules = self.lock();
991 let Some(state) = modules.get_mut(module_id) else {
992 return admitted(None);
993 };
994 let Some(cooldown_until) = state.cooldown_until else {
995 return admitted(None);
996 };
997 if state.probe_in_flight.is_some() {
998 return RouteBindAdmission::Refused {
999 consecutive_timeouts: state.consecutive_timeouts,
1000 retry_in: Duration::ZERO,
1001 probe_in_flight: true,
1002 };
1003 }
1004 let now = Instant::now();
1005 if now < cooldown_until {
1006 return RouteBindAdmission::Refused {
1007 consecutive_timeouts: state.consecutive_timeouts,
1008 retry_in: cooldown_until - now,
1009 probe_in_flight: false,
1010 };
1011 }
1012 let token = Arc::new(());
1013 state.probe_in_flight = Some(Arc::clone(&token));
1014 admitted(Some(token))
1015 }
1016
1017 fn record_accepted(&self, module_id: &str) -> bool {
1018 self.lock()
1019 .remove(module_id)
1020 .is_some_and(|state| state.cooldown_until.is_some())
1021 }
1022
1023 fn record_timeout(
1024 &self,
1025 module_id: &str,
1026 probe_token: Option<&Arc<()>>,
1027 threshold: u32,
1028 cooldown: Duration,
1029 ) -> Option<BreakerOpened> {
1030 let mut modules = self.lock();
1031 let state = modules.entry(module_id.to_string()).or_default();
1032 let was_open = state.cooldown_until.is_some();
1033 let was_probe = Self::owns_probe(state, probe_token);
1034 if was_probe {
1035 state.probe_in_flight = None;
1036 }
1037 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
1038 if state.consecutive_timeouts < threshold {
1039 return None;
1040 }
1041 state.cooldown_until = Some(Instant::now() + cooldown);
1042 Some(BreakerOpened {
1043 consecutive_timeouts: state.consecutive_timeouts,
1044 reopened_after_probe: was_open && was_probe,
1045 })
1046 }
1047
1048 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
1049 state
1053 .probe_in_flight
1054 .as_ref()
1055 .zip(token)
1056 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
1057 }
1058
1059 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
1060 if let Some(state) = self.lock().get_mut(module_id) {
1061 if Self::owns_probe(state, probe_token) {
1062 state.probe_in_flight = None;
1063 }
1064 }
1065 }
1066
1067 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
1082 self.lock()
1083 .remove(module_id)
1084 .map(|state| state.consecutive_timeouts)
1085 .filter(|discarded| *discarded > 0)
1086 }
1087
1088 fn open_snapshot(&self) -> Option<serde_json::Value> {
1095 let now = Instant::now();
1096 let modules = self.lock();
1097 let open = modules
1098 .iter()
1099 .filter_map(|(module_id, state)| {
1100 let cooldown_until = state.cooldown_until?;
1101 Some((
1102 module_id.clone(),
1103 serde_json::json!({
1104 "consecutive_timeouts": state.consecutive_timeouts,
1105 "cooldown_remaining_ms":
1106 cooldown_until.saturating_duration_since(now).as_millis() as u64,
1107 "probe_in_flight": state.probe_in_flight.is_some(),
1108 }),
1109 ))
1110 })
1111 .collect::<serde_json::Map<String, serde_json::Value>>();
1112 (!open.is_empty()).then_some(serde_json::Value::Object(open))
1113 }
1114}
1115
1116impl ControlHandler {
1117 pub(crate) fn install_registration_service(
1118 &self,
1119 router: std::sync::Weak<crate::Router>,
1120 auth: crate::server::ServerAuth,
1121 ) {
1122 self.supervisor.install_registration_service(router, auth);
1123 }
1124 pub fn new(registry: Arc<Registry>) -> Self {
1125 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
1126 }
1127
1128 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
1129 let counters = forwarding.counters();
1130 let route_bind_breakers = forwarding.route_bind_breakers();
1134 let route_bind_concurrency = forwarding.route_bind_concurrency();
1135 let route_outages = forwarding.route_outages();
1136 Self {
1137 registry,
1138 #[cfg(test)]
1139 activity_request_pause: Arc::new(Mutex::new(None)),
1140 #[cfg(test)]
1141 cleanup_pause: Arc::new(Mutex::new(None)),
1142 #[cfg(test)]
1143 after_nonce_attestation: Arc::new(Mutex::new(None)),
1144 #[cfg(test)]
1145 hello_ingress_pause: Arc::new(Mutex::new(None)),
1146 #[cfg(test)]
1147 agent_commands_pause: Arc::new(Mutex::new(None)),
1148 claims: Arc::new(claims::Claims::default()),
1149 config_runtime: None,
1150 config_review_admission: Arc::new(crate::config_gate::MissingReviewAdmission),
1151 config_review_admission_is_default: true,
1152 activity: None,
1153 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1154 catalog_streams: Arc::new(Mutex::new(HashMap::new())),
1155 forwarding,
1156 process_liveness: None,
1157 supervisor: SupervisorHandle::new(),
1158 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1159 env!("CARGO_PKG_VERSION").to_owned(),
1160 )),
1161 agent_declarations: agent_commands::Queues::default(),
1162 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1163 subc_capabilities: Arc::from([
1164 CAP_MANIFEST_REGISTRATION.to_string(),
1165 CAP_CHANNEL_LIFECYCLE.to_string(),
1166 CAP_PING_PONG.to_string(),
1167 CAP_SESSION_ATTACH.to_string(),
1168 CAP_ADMISSION_FACTS_RELAY.to_string(),
1169 CAP_SCOPES_V1.to_string(),
1170 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1171 subc_protocol::peer::CALLER_ATTESTATION_CAPABILITY.to_string(),
1172 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1173 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1174 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1175 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1176 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1177 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1178 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1179 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1180 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1181 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1182 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1183 subc_protocol::ROUTE_CLOSE_REASONS_CAPABILITY.to_string(),
1184 ]),
1185 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1186 route_bind_relay_timeouts: BTreeMap::new(),
1187 route_bind_breakers,
1188 route_bind_concurrency,
1189 route_outages,
1190 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1191 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1192 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1193 storage_config: None,
1194 machine_id: None,
1195 admission_facts_carrier_module_id: None,
1196 admission_facts_targets: None,
1197 scopes: Arc::new(RwLock::new(ScopeTable::new(
1198 crate::daemon_config::default_scope_authority_owners(),
1199 ))),
1200 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1201 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1202 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1203 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1204 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1205 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1206 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1207 captured_peers: Arc::new(Mutex::new(HashMap::new())),
1208 config_desktop_stamps: Arc::new(Mutex::new(HashMap::new())),
1209 local_connections: Arc::new(Mutex::new(HashSet::new())),
1210 operator_app_policy: Default::default(),
1211 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1212 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1213 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1214 #[cfg(test)]
1215 app_open_test_platform: None,
1216 #[cfg(test)]
1217 app_open_test_launch: None,
1218 #[cfg(test)]
1219 recovery_exit_test_after_step: None,
1220 #[cfg(test)]
1221 recovery_exit_test_completed: None,
1222 #[cfg(test)]
1223 apply_reconcile_pause: Arc::new(Mutex::new(None)),
1224 #[cfg(test)]
1225 recovery_exit_test_quiet_retry: false,
1226 #[cfg(test)]
1227 recovery_exit_test_retry_pause: None,
1228 #[cfg(test)]
1229 config_delivery_test_wakes: None,
1230 #[cfg(all(test, unix))]
1231 config_delivery_test_idle: None,
1232 #[cfg(windows)]
1233 windows_verify: None,
1234 #[cfg(windows)]
1235 windows_flash: None,
1236 #[cfg(windows)]
1237 windows_track: None,
1238 #[cfg(all(test, windows))]
1239 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1240 #[cfg(test)]
1241 caller_lookup: None,
1242 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1243 rescan: None,
1244 connected_clients: ConnectedClients::new(),
1245 local_endpoint_unavailable: None,
1246 counters,
1247 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1248 daemon_shutdown: DaemonShutdownFlag::default(),
1249 daemon_provenance: DaemonProvenanceFacts::default(),
1250 #[cfg(test)]
1251 control_dispatch_delay: None,
1252 #[cfg(test)]
1253 provenance_probe_override: None,
1254 #[cfg(test)]
1255 route_bind_relay_sent_hold: None,
1256 #[cfg(test)]
1257 hello_registry_to_forwarding_hold: None,
1258 #[cfg(test)]
1259 hello_ack_to_publication_hold: None,
1260 #[cfg(test)]
1261 hello_before_publication_hold: None,
1262 #[cfg(test)]
1263 hello_after_launch_check_hold: None,
1264 #[cfg(test)]
1265 hello_inside_publication_hold: None,
1266 }
1267 }
1268
1269 pub fn with_storage_config(
1272 mut self,
1273 storage_config: Option<crate::daemon_config::StorageConfig>,
1274 ) -> Self {
1275 self.storage_config = storage_config;
1276 self
1277 }
1278
1279 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1282 self.activity = Some(crate::activity::ActivityLedger::open(
1283 run_dir.join("activity.jsonl"),
1284 incarnation,
1285 ));
1286 self.supervisor.set_activity(self.activity.clone());
1287 self
1288 }
1289
1290 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1293 self.machine_id = machine_id;
1294 self
1295 }
1296
1297 pub fn with_admission_facts_config(
1301 mut self,
1302 carrier_module_id: Option<String>,
1303 targets: Option<Vec<String>>,
1304 ) -> Self {
1305 self.admission_facts_carrier_module_id = carrier_module_id;
1306 self.admission_facts_targets = targets;
1307 self
1308 }
1309
1310 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1314 self.scopes = Arc::new(RwLock::new(
1315 ScopeTable::new(owners.iter().cloned())
1316 .with_federated_owner(self.federated_scope_owner.clone()),
1317 ));
1318 self.scope_authority_owners = owners;
1319 self
1320 }
1321
1322 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1324 self.scopes = Arc::new(RwLock::new(
1325 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1326 .with_federated_owner(owner.clone()),
1327 ));
1328 self.federated_scope_owner = owner;
1329 self
1330 }
1331
1332 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1334 self.privileged_operator_prompt = mode;
1335 self
1336 }
1337
1338 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1341 self.route_bind_relay_timeout = timeout;
1342 self
1343 }
1344
1345 pub fn with_route_bind_relay_timeouts(
1351 mut self,
1352 timeouts: impl IntoIterator<Item = (String, Duration)>,
1353 ) -> Self {
1354 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1355 self
1356 }
1357
1358 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1364 self.route_bind_relay_timeouts
1365 .get(module_id)
1366 .copied()
1367 .unwrap_or(self.route_bind_relay_timeout)
1368 }
1369
1370 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1378 self.route_bind_breaker_threshold = threshold.max(1);
1379 self.route_bind_breaker_cooldown = cooldown;
1380 self
1381 }
1382
1383 #[cfg(test)]
1384 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1385 self.health_probe_timeout = timeout;
1386 self
1387 }
1388
1389 #[cfg(test)]
1390 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1391 self.control_dispatch_delay = Some(delay);
1392 self
1393 }
1394
1395 #[cfg(test)]
1399 pub(crate) fn with_route_bind_relay_sent_hold(
1400 mut self,
1401 hold: Arc<tokio::sync::Barrier>,
1402 ) -> Self {
1403 self.route_bind_relay_sent_hold = Some(hold);
1404 self
1405 }
1406
1407 #[cfg(test)]
1412 pub(crate) fn with_hello_registry_to_forwarding_hold(
1413 mut self,
1414 hold: Arc<std::sync::Barrier>,
1415 ) -> Self {
1416 self.hello_registry_to_forwarding_hold = Some(hold);
1417 self
1418 }
1419
1420 #[cfg(test)]
1421 fn hold_between_registry_and_forwarding(&self) {
1422 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1423 hold.wait();
1424 hold.wait();
1425 }
1426 }
1427
1428 #[cfg(test)]
1431 fn hold_between_ack_and_publication(&self) {
1432 if let Some(hold) = &self.hello_ack_to_publication_hold {
1433 hold.wait();
1434 hold.wait();
1435 }
1436 }
1437
1438 #[cfg(all(test, unix))]
1439 pub(crate) fn with_hello_before_publication_hold(
1440 mut self,
1441 hold: Arc<std::sync::Barrier>,
1442 ) -> Self {
1443 self.hello_before_publication_hold = Some(hold);
1444 self
1445 }
1446
1447 #[cfg(all(test, unix))]
1448 pub(crate) fn with_hello_after_launch_check_hold(
1449 mut self,
1450 hold: Arc<std::sync::Barrier>,
1451 ) -> Self {
1452 self.hello_after_launch_check_hold = Some(hold);
1453 self
1454 }
1455
1456 #[cfg(all(test, unix))]
1457 pub(crate) fn with_hello_inside_publication_hold(
1458 mut self,
1459 hold: Arc<std::sync::Barrier>,
1460 ) -> Self {
1461 self.hello_inside_publication_hold = Some(hold);
1462 self
1463 }
1464
1465 pub fn with_process_liveness(
1466 mut self,
1467 process_liveness: Arc<dyn ModuleProcessLiveness>,
1468 ) -> Self {
1469 self.process_liveness = Some(process_liveness);
1470 self
1471 }
1472
1473 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1474 self.supervisor = supervisor;
1475 if let Some(owner) = &self.config_runtime {
1476 self.supervisor.install_binding_retirement(owner);
1477 }
1478 self.supervisor.set_activity(self.activity.clone());
1479 self
1480 }
1481
1482 pub fn with_daemon_provenance(
1483 mut self,
1484 pid: u32,
1485 started_at_ms: u64,
1486 executable_path: Option<PathBuf>,
1487 build_git_sha: Option<String>,
1488 build_lock_digest: Option<String>,
1489 ) -> Self {
1490 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1491 let process_start_time = process_start_time(pid);
1492 self.daemon_provenance = DaemonProvenanceFacts {
1493 build: DaemonBuildProvenance {
1494 build_git_sha,
1495 build_lock_digest,
1496 },
1497 pid: Some(pid),
1498 started_at_ms: Some(started_at_ms),
1499 start_clock: None,
1500 executable_path,
1501 executable_identity,
1502 process_start_time,
1503 probe: ExecutableIdentityProbe::default(),
1504 };
1505 self
1506 }
1507
1508 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1509 self.daemon_provenance.start_clock = Some(clock);
1510 self
1511 }
1512
1513 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1515 self.scopes
1516 .write()
1517 .unwrap_or_else(|poisoned| poisoned.into_inner())
1518 .set_wall_clock(Arc::new(clock));
1519 self
1520 }
1521
1522 #[cfg(test)]
1523 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1524 self.provenance_probe_override = Some(result);
1525 self
1526 }
1527
1528 pub fn with_capability_config(
1532 self,
1533 modules: impl IntoIterator<Item = (String, bool)>,
1534 reserved_capabilities: BTreeMap<String, String>,
1535 ) -> Self {
1536 self.capability_evaluator
1537 .configure(modules, reserved_capabilities);
1538 self
1539 }
1540
1541 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1545 self.daemon_shutdown = flag;
1546 self
1547 }
1548
1549 pub fn with_supervisor_rescan(
1550 mut self,
1551 supervisor: Supervisor,
1552 config_path: impl Into<PathBuf>,
1553 configured_port: Option<u16>,
1554 ) -> Self {
1555 self.rescan = Some(SupervisorRescanContext {
1556 supervisor,
1557 config_path: config_path.into(),
1558 configured_port,
1559 storage_config: self.storage_config.clone(),
1560 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1561 admission_facts_targets: self.admission_facts_targets.clone(),
1562 scope_authority_owners: self.scope_authority_owners.clone(),
1563 federated_scope_owner: self.federated_scope_owner.clone(),
1564 privileged_operator_prompt: self.privileged_operator_prompt,
1565 generation: Arc::default(),
1566 });
1567 self
1568 }
1569
1570 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1571 self.connected_clients = connected_clients;
1572 self
1573 }
1574
1575 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1578 self.local_endpoint_unavailable = reason;
1579 self
1580 }
1581
1582 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1583 Arc::clone(&self.forwarding)
1584 }
1585
1586 pub(crate) fn counters(&self) -> DaemonCounters {
1587 self.counters.clone()
1588 }
1589
1590 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1593 let handler = Arc::downgrade(&self);
1594 let evaluator = self.capability_evaluator.clone();
1595 tokio::spawn(async move {
1596 loop {
1597 evaluator.wait_for_change_or_deadline().await;
1598 let Some(handler) = handler.upgrade() else {
1599 break;
1600 };
1601 handler.refresh_capability_requirements();
1602 }
1603 });
1604 }
1605
1606 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1612 let context = scope_expiry::ScopeExpiryContext::new(&self);
1613 let deadline_added = self.scope_deadline_added.clone();
1614 tokio::spawn(async move {
1615 loop {
1616 if !context.has_deadlines() {
1617 deadline_added.notified().await;
1618 continue;
1619 }
1620 tokio::time::sleep(Duration::from_secs(1)).await;
1621 if let Err(error) = context.sweep() {
1622 warn!(%error, "scope expiry sweep failed");
1623 }
1624 }
1625 });
1626 }
1627
1628 #[cfg(test)]
1629 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1630 scope_expiry::ScopeExpiryContext::new(self).sweep()
1631 }
1632
1633 fn publish_scope_changes(
1634 &self,
1635 changes: &[crate::scopes::ScopeTagChange],
1636 expired: &[crate::scopes::ScopeExpired],
1637 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1638 scope_expiry::ScopeExpiryContext::new(self).publish_scope_changes(changes, expired)
1639 }
1640
1641 fn runtime_capability_snapshot(
1642 &self,
1643 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1644 let runtime = self
1645 .supervisor
1646 .list()
1647 .into_iter()
1648 .map(|module| {
1649 let status = module.status().map_err(|err| {
1650 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1651 })?;
1652 Ok(RuntimeModule {
1653 module_id: status.module_id,
1654 state: status.state,
1655 enabled: status.enabled,
1656 })
1657 })
1658 .collect::<Result<Vec<_>, RouterError>>()?;
1659 let registrations = self.registry.list_registered_modules().map_err(|err| {
1665 RouterError::backend(
1666 0,
1667 0,
1668 format!("failed to list capability registrations: {err}"),
1669 )
1670 })?;
1671 let registrations = registrations
1672 .into_iter()
1673 .map(|registration| RegisteredModule {
1674 module_id: registration.manifest.module_id,
1675 module_version: registration.manifest.module_version,
1676 capabilities: registration.manifest.capabilities,
1677 })
1678 .collect();
1679 Ok((runtime, registrations))
1680 }
1681
1682 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1689 let cached_registration = RegisteredModule {
1690 module_id: registration.manifest.module_id.clone(),
1691 module_version: registration.manifest.module_version.clone(),
1692 capabilities: registration.manifest.capabilities.clone(),
1693 };
1694 let change = self.capability_evaluator.record_hello(&cached_registration);
1695 log_manifest_change(&cached_registration.module_id, &change);
1696 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1697 self.enforce_capability_denies();
1698 }
1699 self.refresh_capability_requirements();
1700 }
1701
1702 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1706 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1707 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1708 self.supervisor.set_swap_promotion_observer(observer);
1709 }
1710
1711 pub fn refresh_capability_requirements(&self) {
1712 match self.runtime_capability_snapshot() {
1713 Ok((runtime, registrations)) => {
1714 log_requirement_events(
1715 self.capability_evaluator
1716 .evaluate_now(&runtime, ®istrations),
1717 &self.daemon_shutdown,
1718 );
1719 }
1720 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1721 }
1722 }
1723
1724 fn enforce_capability_denies(&self) {
1728 let registrations = match self.registry.list_registered_modules() {
1729 Ok(snapshot) => snapshot,
1730 Err(err) => {
1731 warn!(error = %err, "failed to read registrations for capability deny census");
1732 return;
1733 }
1734 };
1735 let manifests = registrations
1736 .into_iter()
1737 .map(|registration| {
1738 (
1739 registration.manifest.module_id.clone(),
1740 registration.manifest,
1741 )
1742 })
1743 .collect::<BTreeMap<_, _>>();
1744 let census = match self.forwarding.route_census(None) {
1745 Ok(census) => census,
1746 Err(err) => {
1747 warn!(error = %err, "failed to read route census for capability deny enforcement");
1748 return;
1749 }
1750 };
1751
1752 for (target_module_id, routes) in census {
1753 let Some(target_manifest) = manifests.get(&target_module_id) else {
1754 continue;
1755 };
1756 let mut closed_routes = Vec::new();
1757 let mut goodbyes = Vec::new();
1758 for route in routes {
1759 let Principal::Reserved {
1760 module_id: opening_module_id,
1761 } = &route.principal
1762 else {
1763 continue;
1764 };
1765 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1766 continue;
1767 };
1768 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1769 continue;
1770 };
1771
1772 match self.forwarding.release_client_route(
1773 route.goodbye_target.connection_id,
1774 route.goodbye_target.channel,
1775 route.goodbye_target.epoch,
1776 ) {
1777 Ok(RouteRelease::Removed(module_goodbye)) => {
1778 warn!(
1779 opening_module_id,
1780 target_module_id,
1781 capability,
1782 "force-closing route because an attested capability deny edge now matches"
1783 );
1784 goodbyes.push(module_goodbye);
1785 goodbyes.push(route.goodbye_target.clone());
1786 closed_routes.push(route);
1787 }
1788 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1789 Err(err) => warn!(
1790 opening_module_id,
1791 target_module_id,
1792 capability,
1793 error = %err,
1794 "failed to force-close capability-denied route"
1795 ),
1796 }
1797 }
1798
1799 if closed_routes.is_empty() {
1800 continue;
1801 }
1802 send_route_control_pushes(
1803 &self.forwarding,
1804 closed_routes,
1805 ClientControlPush::RouteClosed {
1806 module_id: target_module_id,
1807 channels: Vec::new(),
1808 reason: RouteCloseReason::CapabilityDenied,
1809 drained: false,
1810 abandoned: 0,
1811 excluded_subscriptions: 0,
1812 terminal: Some(false),
1813 },
1814 );
1815 self.emit_route_goodbyes(goodbyes);
1818 }
1819 }
1820
1821 fn not_ready_reason(
1827 &self,
1828 registration: &crate::registry::ModuleRegistration,
1829 ) -> Option<NotReadyReason> {
1830 if !registration.ready {
1831 return Some(NotReadyReason {
1832 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1833 capability: None,
1834 });
1835 }
1836 self.first_unprovided_required_capability(registration)
1837 .map(|capability| NotReadyReason {
1838 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1839 capability: Some(capability),
1840 })
1841 }
1842
1843 fn first_unprovided_required_capability(
1861 &self,
1862 registration: &crate::registry::ModuleRegistration,
1863 ) -> Option<String> {
1864 let required = registration
1865 .manifest
1866 .capabilities
1867 .iter()
1868 .flat_map(|declarations| declarations.requires.iter())
1869 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1870 .map(|requirement| requirement.capability.as_str())
1871 .collect::<BTreeSet<_>>();
1872 if required.is_empty() {
1873 return None;
1874 }
1875 let module_id = registration.manifest.module_id.as_str();
1876 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1877 if required
1878 .iter()
1879 .any(|capability| verdict(capability).is_none())
1880 {
1881 self.refresh_capability_requirements();
1882 }
1883 required
1884 .into_iter()
1885 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1886 .map(str::to_string)
1887 }
1888
1889 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1890 self.capability_evaluator
1891 .statuses()
1892 .into_iter()
1893 .map(capability_requirement_status)
1894 .collect()
1895 }
1896
1897 fn deregister_connection(
1904 &self,
1905 connection_id: ConnectionId,
1906 reason: RegistrationEndReason,
1907 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1908 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
1909 spawn.end_primary_registration();
1910 }
1911 self.registry
1912 .deregister_connection_with_reason(connection_id, reason)
1913 }
1914
1915 fn rollback_unpublished_hello(
1920 &self,
1921 connection: ConnectionId,
1922 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1923 self.registry.deregister_connection_with_reason(
1924 connection,
1925 RegistrationEndReason::RegistrationFailed,
1926 )
1927 }
1928
1929 fn request_refusal_close(
1930 &self,
1931 connection: ConnectionId,
1932 sink: Option<&crate::FrameSink>,
1933 reason: CloseReason,
1934 ) {
1935 if let Some(sink) = sink {
1936 sink.terminate_ingress();
1937 }
1938 if self.claims.close_notifications_enabled() {
1939 self.forwarding.request_connection_close(connection, reason);
1940 }
1941 }
1942
1943 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1944 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1945 return None;
1946 }
1947 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1948 parse_client_control_request(&frame.body)
1949 else {
1950 return None;
1951 };
1952 Some(target_module_id(&target).to_string())
1953 }
1954
1955 pub(crate) fn route_open_capacity_refusal(
1956 &self,
1957 ctx: &RouteCtx,
1958 frame: &Frame,
1959 target_module_id: &str,
1960 in_flight: usize,
1961 limit: usize,
1962 ) -> Result<Frame, RouterError> {
1963 let caller = self.route_open_caller_from_frame(ctx, frame);
1966 self.route_open_admission_refusal_frame(
1967 ctx,
1968 &caller,
1969 frame,
1970 target_module_id,
1971 "open_admission_full",
1972 (in_flight, limit),
1973 format!(
1974 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1975 ),
1976 )
1977 }
1978
1979 fn route_open_target_capacity_refusal(
1980 &self,
1981 ctx: &RouteCtx,
1982 caller: &RouteOpenCaller,
1983 frame: &Frame,
1984 target_module_id: &str,
1985 in_flight: usize,
1986 ) -> Result<Frame, RouterError> {
1987 self.route_open_admission_refusal_frame(
1988 ctx,
1989 caller,
1990 frame,
1991 target_module_id,
1992 "target_binds_full",
1993 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1994 format!(
1995 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1996 ),
1997 )
1998 }
1999
2000 #[allow(clippy::too_many_arguments)]
2008 fn route_open_admission_refusal_frame(
2009 &self,
2010 ctx: &RouteCtx,
2011 caller: &RouteOpenCaller,
2012 frame: &Frame,
2013 target_module_id: &str,
2014 reason: &'static str,
2015 (in_flight, limit): (usize, usize),
2016 message: impl Into<String>,
2017 ) -> Result<Frame, RouterError> {
2018 let code = error_codes::TARGET_UNAVAILABLE;
2019 self.counters.increment_route_open_refused(code);
2020 info!(
2021 target: "control",
2022 code,
2023 reason,
2024 module_id = ?target_module_id,
2025 connection_id = ctx.connection_id.get(),
2026 principal = %caller.principal,
2027 scope = %caller.scope,
2028 harness = %caller.harness,
2029 in_flight,
2030 limit,
2031 "route.open refused"
2032 );
2033 control_error_frame(frame, code, message.into())
2034 }
2035
2036 #[cfg(test)]
2042 pub fn handle_control(
2043 &self,
2044 connection_id: ConnectionId,
2045 frame: Frame,
2046 ) -> Result<Vec<Frame>, RouterError> {
2047 match frame.header.ty {
2048 FrameType::Ping => Ok(vec![pong(&frame)?]),
2049 FrameType::Hello => self.handle_hello(connection_id, None, frame),
2050 FrameType::Goodbye => self.handle_goodbye(connection_id),
2051 ty => Ok(vec![control_error_frame(
2052 &frame,
2053 "unsupported_control_frame",
2054 format!("unsupported channel-0 frame {ty:?}"),
2055 )?]),
2056 }
2057 }
2058
2059 pub async fn handle_control_frame(
2060 &self,
2061 ctx: &RouteCtx,
2062 frame: Frame,
2063 ) -> Result<Vec<Frame>, RouterError> {
2064 self.handle_control_frame_timed(ctx, frame, None).await
2065 }
2066
2067 #[cfg(any(test, feature = "test-support"))]
2068 pub fn with_test_operator_app_policy(
2069 mut self,
2070 policy: crate::operator_apps::TestOperatorAppPolicy,
2071 ) -> Self {
2072 self.operator_app_policy = policy.into();
2073 self
2074 }
2075
2076 #[cfg(all(windows, any(test, feature = "test-support")))]
2080 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
2081 self.windows_verify = Some(verify);
2082 self
2083 }
2084
2085 #[cfg(all(windows, any(test, feature = "test-support")))]
2087 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
2088 self.windows_flash = Some(flash);
2089 self
2090 }
2091
2092 #[cfg(all(windows, any(test, feature = "test-support")))]
2096 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
2097 self.windows_track = Some(track);
2098 self
2099 }
2100
2101 pub(crate) fn record_captured_peer(
2102 &self,
2103 connection: ConnectionId,
2104 peer: subc_os::local_peer::CapturedPeer,
2105 ) {
2106 self.record_observed_peer(connection, peer.observation().clone());
2107 let identity = self.operator_app_policy.config_identity(peer.clone());
2108 #[cfg(windows)]
2109 let identity = identity.or_else(|| self.config_windows_launch_identity(&peer));
2110 if let Some(identity) = identity {
2111 self.config_desktop_stamps
2114 .lock()
2115 .unwrap_or_else(|p| p.into_inner())
2116 .insert(connection, identity);
2117 }
2118 self.captured_peers
2119 .lock()
2120 .unwrap_or_else(|p| p.into_inner())
2121 .insert(connection, peer.clone());
2122 #[cfg(windows)]
2123 if let Some(stamp) = self.stamp_windows_launch(&peer) {
2124 self.operator_apps
2125 .lock()
2126 .unwrap_or_else(|p| p.into_inner())
2127 .insert(connection, stamp);
2128 }
2129 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
2130 self.operator_apps
2131 .lock()
2132 .unwrap_or_else(|p| p.into_inner())
2133 .insert(connection, stamp);
2134 }
2135 }
2136
2137 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
2138 self.local_connections
2139 .lock()
2140 .unwrap_or_else(|p| p.into_inner())
2141 .insert(connection);
2142 }
2143
2144 fn connection_operator_app(
2147 &self,
2148 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
2149 ) -> Option<subc_protocol::peer::OperatorApp> {
2150 stamp
2151 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
2152 .map(|stamp| stamp.app.clone())
2153 }
2154
2155 fn handle_connection_describe(
2156 &self,
2157 ctx: &RouteCtx,
2158 frame: Frame,
2159 ) -> Result<Vec<Frame>, RouterError> {
2160 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
2161 return Ok(vec![control_error_frame(
2162 &frame,
2163 "invalid_control_body",
2164 "malformed connection.describe body",
2165 )?]);
2166 }
2167 let local = self
2168 .local_connections
2169 .lock()
2170 .unwrap_or_else(|p| p.into_inner())
2171 .contains(&ctx.connection_id);
2172 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
2173 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
2174 Ok(vec![control_response_body_frame(
2175 &frame,
2176 &subc_control::ConnectionDescribeReply::new(local, operator_app),
2177 ops::CONNECTION_DESCRIBE,
2178 )?])
2179 }
2180
2181 pub(crate) fn record_observed_peer(
2182 &self,
2183 connection: ConnectionId,
2184 peer: subc_os::local_peer::ObservedPeer,
2185 ) {
2186 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
2187 let peer = match peer {
2188 OsPeer::Unobserved => ObservedPeer::unobserved(),
2189 OsPeer::Observed(peer) => {
2190 let observed = match peer.user {
2191 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
2192 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
2193 };
2194 observed
2195 .with_exe(
2196 peer.exe
2197 .and_then(|path| path.into_os_string().into_string().ok()),
2198 )
2199 .with_signing_id(peer.signing_id)
2200 }
2201 };
2202 self.observed_peers
2203 .lock()
2204 .unwrap_or_else(|p| p.into_inner())
2205 .insert(connection, peer);
2206 }
2207
2208 pub(crate) fn record_caller_endpoints(
2209 &self,
2210 connection: ConnectionId,
2211 endpoints: Option<(SocketAddr, SocketAddr)>,
2212 ) {
2213 let mut callers = self
2214 .caller_endpoints
2215 .lock()
2216 .unwrap_or_else(|p| p.into_inner());
2217 callers.remove(&connection);
2218 if let Some(endpoints) = endpoints {
2219 callers.insert(connection, endpoints);
2220 }
2221 }
2222
2223 #[cfg(test)]
2224 pub(crate) fn with_caller_lookup_for_test(
2225 mut self,
2226 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2227 ) -> Self {
2228 self.caller_lookup = Some(Arc::new(lookup));
2229 self
2230 }
2231
2232 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2233 let endpoints = self
2234 .caller_endpoints
2235 .lock()
2236 .unwrap_or_else(|p| p.into_inner())
2237 .get(&connection)
2238 .copied();
2239 #[cfg(test)]
2240 let lookup = self.caller_lookup.clone();
2241 tokio::task::spawn_blocking(move || {
2242 #[cfg(test)]
2243 if let Some(lookup) = lookup {
2244 return lookup(endpoints);
2245 }
2246 endpoints
2247 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2248 .unwrap_or_default()
2249 })
2250 .await
2251 .unwrap_or_default()
2252 }
2253
2254 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2255 frame.header.channel == 0
2256 && frame.header.ty == FrameType::Request
2257 && self
2258 .forwarding
2259 .module_endpoint_for_connection(ctx.connection_id)
2260 .is_ok_and(|endpoint| endpoint.is_none())
2261 && parse_client_control_request(&frame.body)
2262 .ok()
2263 .is_some_and(|request| privileged_operation(&request).is_some())
2264 }
2265
2266 pub(crate) async fn handle_control_frame_timed(
2267 &self,
2268 ctx: &RouteCtx,
2269 frame: Frame,
2270 dispatch_started_at: Option<StdInstant>,
2271 ) -> Result<Vec<Frame>, RouterError> {
2272 let first = ctx.egress.first_ingress_frame();
2273 self.handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
2274 .await
2275 }
2276
2277 pub(crate) async fn handle_control_frame_ingress(
2278 &self,
2279 ctx: &RouteCtx,
2280 frame: Frame,
2281 dispatch_started_at: Option<StdInstant>,
2282 first: bool,
2283 ) -> Result<Vec<Frame>, RouterError> {
2284 if ctx.egress.ingress_terminal() {
2285 return Ok(Vec::new());
2286 }
2287 #[cfg(test)]
2288 self.claims.record_dispatch(frame.header.corr);
2289 let claim_binding = self.claims.binding(ctx.connection_id);
2290 let mut prefix_attempt = (frame.header.ty == FrameType::Hello
2291 && claim_binding
2292 .as_ref()
2293 .is_some_and(|binding| binding.prefix_module_id().is_some()))
2294 .then(|| self.prefix_hello_attempt(ctx));
2295 if claim_binding
2296 .as_ref()
2297 .is_some_and(|binding| binding.prefix_module_id().is_some())
2298 {
2299 let registered = self
2300 .forwarding
2301 .module_endpoint_for_connection(ctx.connection_id)
2302 .map_err(RouterError::Forwarding)?
2303 .is_some();
2304 if !registered && frame.header.ty != FrameType::Hello {
2305 return self
2306 .refuse_claim_connection(ctx, &frame, "claim_hello_required")
2307 .await;
2308 }
2309 if registered && frame.header.ty == FrameType::Goodbye {
2310 ctx.egress.terminate_ingress();
2311 let result = self.handle_goodbye(ctx.connection_id);
2312 self.claims.connection_closed(ctx.connection_id);
2313 self.request_refusal_close(
2314 ctx.connection_id,
2315 Some(&ctx.egress),
2316 CloseReason::new("claim_registration_ended", "prefix registration ended"),
2317 );
2318 return result;
2319 }
2320 if registered && frame.header.ty == FrameType::Request {
2321 let allowed =
2322 serde_json::from_slice::<ControlOpProbe>(&frame.body).is_ok_and(|probe| {
2323 matches!(
2324 probe.op.as_str(),
2325 "catalog.update"
2326 | "catalog.update_role_declarations"
2327 | "scope.describe"
2328 | "supervisor.live_roots"
2329 | "activity.record"
2330 )
2331 });
2332 if !allowed {
2333 return Ok(vec![control_error_frame(
2334 &frame,
2335 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2336 "prefix registrations have only brokered module operations",
2337 )?]);
2338 }
2339 }
2340 if registered
2341 && frame.header.ty == FrameType::Push
2342 && !serde_json::from_slice::<ControlOpProbe>(&frame.body)
2343 .is_ok_and(|probe| probe.op == "route.status")
2344 {
2345 return Ok(vec![control_error_frame(
2346 &frame,
2347 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2348 "prefix registrations may push only route.status for routes they serve",
2349 )?]);
2350 }
2351 }
2352 match frame.header.ty {
2353 FrameType::Ping => Ok(vec![pong(&frame)?]),
2354 FrameType::Hello => {
2355 if claim_binding
2356 .as_ref()
2357 .is_some_and(|binding| binding.prefix_module_id().is_none())
2358 {
2359 return self
2360 .refuse_claim_connection(ctx, &frame, "claim_hello_not_allowed")
2361 .await;
2362 }
2363 let channel_spawn = self.supervisor.registration_spawn(ctx.connection_id);
2367 if let Some(spawn) = &channel_spawn {
2368 spawn.wait_launch().await;
2369 if !spawn.launch_ready() {
2372 ctx.egress.terminate_ingress();
2373 return Ok(Vec::new());
2374 }
2375 #[cfg(test)]
2376 self.pause_hello_ingress_for_test().await;
2377 } else if claim_binding.is_none() {
2378 if let Ok(hello) = serde_json::from_slice::<serde_json::Value>(&frame.body) {
2379 if let (Some(module), Some(nonce)) = (
2380 hello
2381 .get("manifest")
2382 .and_then(|value| value.get("module_id"))
2383 .and_then(|value| value.as_str()),
2384 hello.get("launch_nonce").and_then(|value| value.as_str()),
2385 ) {
2386 if !self
2387 .config_runtime
2388 .as_ref()
2389 .is_some_and(|owner| owner.module_is_gated(module))
2390 && !self.supervisor.wait_program_binding(module, nonce).await
2391 {
2392 self.request_refusal_close(
2393 ctx.connection_id,
2394 Some(&ctx.egress),
2395 CloseReason::new(
2396 "program_refused",
2397 "the spawned program binding failed",
2398 ),
2399 );
2400 return Ok(vec![control_error_frame(
2401 &frame,
2402 if self
2403 .config_runtime
2404 .as_ref()
2405 .is_some_and(|owner| owner.is_recovering())
2406 {
2407 "config_gated"
2408 } else {
2409 "program_refused"
2410 },
2411 "the spawned program binding failed",
2412 )?]);
2413 }
2414 }
2415 }
2416 }
2417 let handler = self.clone();
2418 let connection = ctx.connection_id;
2419 let egress = ctx.egress.clone();
2420 let correlation = frame.header.corr;
2421 let dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
2422 let span = tracing::Span::current();
2423 let hello_binding = claim_binding.clone();
2424 let replies = tokio::task::spawn_blocking(move || {
2425 tracing::dispatcher::with_default(&dispatcher, || {
2426 span.in_scope(|| {
2427 handler.handle_hello_checked(
2428 connection,
2429 Some(egress),
2430 frame,
2431 true,
2432 hello_binding,
2433 channel_spawn,
2434 )
2435 })
2436 })
2437 })
2438 .await
2439 .map_err(|error| RouterError::backend(0, correlation, error.to_string()))??;
2440 #[cfg(test)]
2441 if prefix_attempt.is_some() {
2442 self.claims.pause_prefix_commit_for_test(false).await;
2443 }
2444 if let Some(attempt) = &mut prefix_attempt {
2445 let accepted = !replies
2446 .iter()
2447 .any(|reply| reply.header.ty == FrameType::Error)
2448 && self
2449 .registry
2450 .get_module_by_connection(ctx.connection_id)
2451 .map_err(|err| RouterError::backend(0, correlation, err.to_string()))?
2452 .is_some();
2453 if accepted {
2454 attempt.accepted = true;
2455 } else {
2456 ctx.egress.terminate_ingress();
2457 for reply in replies {
2458 ctx.egress.send_flushed(reply).await?;
2459 }
2460 return Ok(Vec::new());
2461 }
2462 }
2463 Ok(replies)
2464 }
2465 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2466 FrameType::Cancel => {
2467 if self
2473 .agent_declarations
2474 .cancel(ctx.connection_id, frame.header.corr)
2475 || self
2476 .forwarding
2477 .operator_confirms()
2478 .cancel(ctx.connection_id, frame.header.corr)
2479 || self.cancel_config_approval(ctx.connection_id, frame.header.corr)
2480 || self
2481 .supervisor
2482 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2483 || self.cancel_activity_subscription(ctx, &frame)?
2484 || self.cancel_catalog_subscription(ctx.connection_id, frame.header.corr)
2485 {
2486 Ok(Vec::new())
2487 } else {
2488 Ok(vec![control_error_frame(
2489 &frame,
2490 "unknown_subscription",
2491 "no subscription has this correlation id",
2492 )?])
2493 }
2494 }
2495 FrameType::Request => {
2496 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2499 let claim_present = op.as_ref().is_some_and(|probe| probe.op == "claim.present");
2500 if let Some(refusal) = self
2501 .recovery_request_refusal(
2502 ctx,
2503 &frame,
2504 op.as_ref().map(|probe| probe.op.as_str()),
2505 )
2506 .await?
2507 {
2508 if claim_present {
2509 self.reclaim_presented_expiry(&frame);
2510 return self
2511 .refuse_claim_connection(ctx, &frame, "config_gated")
2512 .await;
2513 }
2514 return Ok(vec![refusal]);
2515 }
2516 if claim_present {
2517 return self.handle_claim_present(ctx, &frame, first).await;
2518 }
2519 if op.as_ref().is_some_and(|probe| probe.op == "claim.mint") {
2520 return self.handle_claim_mint(ctx, &frame).await;
2521 }
2522 if let Some(refusal) = self.registration_operation_refusal(
2523 ctx,
2524 &frame,
2525 op.as_ref().map(|probe| probe.op.as_str()),
2526 )? {
2527 return Ok(vec![refusal]);
2528 }
2529 if op
2530 .as_ref()
2531 .is_some_and(|probe| probe.op == ops::CATALOG_WATCH)
2532 {
2533 return self.handle_catalog_watch(ctx, frame);
2534 }
2535 if op
2536 .as_ref()
2537 .is_some_and(|probe| probe.op == MODULE_TO_SUBC_OP_CATALOG_UPDATE)
2538 && subc_protocol::provider_declarations::catalog_update_has_provider_declarations(&frame.body)
2539 .unwrap_or(false)
2540 {
2541 return Ok(vec![control_error_frame(
2542 &frame,
2543 error_codes::CATALOG_UPDATE_FROZEN_DECLARATIONS,
2544 "provider_declarations are fixed at HELLO; restart the module to change them",
2545 )?]);
2546 }
2547 if let Some(probe) = op
2548 .as_ref()
2549 .filter(|probe| config_gate::is_module_config_op(&probe.op))
2550 {
2551 return self
2552 .handle_module_config_request(ctx, frame, &probe.op)
2553 .await;
2554 }
2555 if op
2556 .as_ref()
2557 .is_some_and(|probe| config_gate::is_config_op(&probe.op))
2558 {
2559 return self.handle_config_gate_request(ctx, frame).await;
2560 }
2561 if op
2562 .as_ref()
2563 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2564 {
2565 return self.handle_activity_record(ctx, frame).await;
2566 }
2567 if op.as_ref().is_some_and(|probe| {
2568 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2569 }) {
2570 return self.handle_activity_read(ctx, frame).await;
2571 }
2572 if op
2573 .as_ref()
2574 .is_some_and(|probe| probe.op == "operator.confirm")
2575 {
2576 return self.handle_operator_confirm(ctx, frame);
2577 }
2578 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2579 return self
2580 .observe_control_wait(
2581 ctx,
2582 None,
2583 ops::APP_OPEN,
2584 self.handle_app_open(ctx, frame),
2585 )
2586 .await;
2587 }
2588 if op
2589 .as_ref()
2590 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2591 {
2592 return self.handle_connection_describe(ctx, frame);
2593 }
2594 if op
2595 .as_ref()
2596 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2597 {
2598 return self.handle_scope_served_by(ctx.connection_id, frame);
2599 }
2600 if self
2601 .forwarding
2602 .module_endpoint_for_connection(ctx.connection_id)
2603 .map_err(RouterError::Forwarding)?
2604 .is_some()
2605 {
2606 if !is_known_module_request_op(&frame.body) {
2607 return Ok(vec![control_error_frame(
2608 &frame,
2609 "unsupported_control_frame",
2610 "module-originated channel-0 REQUEST is not supported",
2611 )?]);
2612 }
2613 let request = match parse_module_control_request_from_module(&frame.body) {
2614 Ok(request) => request,
2615 Err((err, ControlRequestBodyError::UnknownOp)) => {
2616 return Ok(vec![control_error_frame(
2617 &frame,
2618 "unsupported_control_frame",
2619 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2620 )?])
2621 }
2622 Err((err, ControlRequestBodyError::InvalidBody)) => {
2623 return Ok(vec![control_error_frame(
2624 &frame,
2625 "invalid_control_body",
2626 format!("malformed module control body: {err}"),
2627 )?])
2628 }
2629 };
2630 let op = module_control_request_op(&request);
2631 let corr = frame.header.corr;
2632 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2633 if let ModuleControlRequestFromModule::AgentCommands {
2634 started,
2635 finished,
2636 known,
2637 } = request
2638 {
2639 return self
2640 .agent_declarations
2641 .enqueue(
2642 self,
2643 ctx,
2644 frame,
2645 crate::resource_totals::Declaration {
2646 started,
2647 finished,
2648 known,
2649 },
2650 )
2651 .await;
2652 }
2653 let result =
2654 self.handle_module_control_request(ctx.connection_id, frame, request);
2655 log_slow_control_dispatch(
2656 dispatch_started_at,
2657 op,
2658 ctx.connection_id,
2659 corr,
2660 None,
2661 );
2662 return result;
2663 }
2664
2665 if is_known_module_request_op(&frame.body) {
2666 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2667 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2668 let code = if is_agents {
2669 "not_supervised"
2670 } else if op
2671 .as_ref()
2672 .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2673 {
2674 error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2675 } else {
2676 "not_registered"
2677 };
2678 return Ok(vec![control_error_frame(
2679 &frame,
2680 code,
2681 "module-originated control requests require the module's own registered connection",
2682 )?]);
2683 }
2684
2685 let request = match parse_client_control_request(&frame.body) {
2686 Ok(request) => request,
2687 Err((err, ControlRequestBodyError::UnknownOp)) => {
2688 return Ok(vec![control_error_frame(
2689 &frame,
2690 "unknown_control_op",
2691 format!("unknown client control op: {err}"),
2692 )?])
2693 }
2694 Err((err, ControlRequestBodyError::InvalidBody)) => {
2695 return Ok(vec![control_error_frame(
2696 &frame,
2697 "invalid_control_body",
2698 format!("malformed client control body: {err}"),
2699 )?])
2700 }
2701 };
2702 let op = client_control_request_op(&request);
2703 let corr = frame.header.corr;
2704 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2705 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2706 #[cfg(test)]
2707 if let Some(delay) = self.control_dispatch_delay {
2708 tokio::time::sleep(delay).await;
2709 }
2710 let result = self
2711 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2712 .await;
2713 if let Some(timings) = route_open_timings.as_mut() {
2714 timings.finish();
2715 }
2716 log_slow_control_dispatch(
2717 dispatch_started_at,
2718 op,
2719 ctx.connection_id,
2720 corr,
2721 route_open_timings.as_ref(),
2722 );
2723 result
2724 }
2725 FrameType::Push => {
2726 let Some(endpoint) = self
2727 .forwarding
2728 .module_endpoint_for_connection(ctx.connection_id)
2729 .map_err(RouterError::Forwarding)?
2730 else {
2731 return Ok(vec![control_error_frame(
2732 &frame,
2733 "unsupported_control_frame",
2734 "client-originated channel-0 PUSH is not supported",
2735 )?]);
2736 };
2737 self.handle_status_update(endpoint, frame)
2738 }
2739 FrameType::Response | FrameType::Error
2740 if self
2741 .forwarding
2742 .module_endpoint_for_connection(ctx.connection_id)
2743 .map_err(RouterError::Forwarding)?
2744 .is_some() =>
2745 {
2746 self.handle_module_relay_response(ctx.connection_id, frame)
2747 }
2748 ty => Ok(vec![control_error_frame(
2749 &frame,
2750 "unsupported_control_frame",
2751 format!("unsupported channel-0 frame {ty:?}"),
2752 )?]),
2753 }
2754 }
2755
2756 #[cfg(test)]
2757 fn pause_cleanup_for_test(&self, stage: u8) {
2758 let mut slot = self.cleanup_pause.lock().unwrap_or_else(|p| p.into_inner());
2759 if slot.as_ref().is_some_and(|(at, _)| *at == stage) {
2760 let (_, (reached, release)) = slot.take().unwrap();
2761 drop(slot);
2762 let _ = reached.send(());
2763 let _ = release.recv_timeout(Duration::from_secs(5));
2764 }
2765 }
2766
2767 #[cfg(all(test, unix))]
2770 pub(crate) fn after_nonce_attestation_for_test(&self, hook: impl FnOnce() + Send + 'static) {
2771 *self
2772 .after_nonce_attestation
2773 .lock()
2774 .unwrap_or_else(|p| p.into_inner()) = Some(Box::new(hook));
2775 }
2776
2777 #[cfg(test)]
2778 pub(super) fn pause_after_nonce_attestation_for_test(&self) {
2779 let hook = self
2780 .after_nonce_attestation
2781 .lock()
2782 .unwrap_or_else(|p| p.into_inner())
2783 .take();
2784 if let Some(hook) = hook {
2785 hook();
2786 }
2787 }
2788
2789 #[cfg(test)]
2790 async fn pause_hello_ingress_for_test(&self) {
2791 let pause = self
2792 .hello_ingress_pause
2793 .lock()
2794 .unwrap_or_else(|p| p.into_inner())
2795 .take();
2796 if let Some((reached, release)) = pause {
2797 let _ = reached.send(());
2798 let _ = release.await;
2799 }
2800 }
2801
2802 pub fn cleanup_connection(
2803 &self,
2804 connection_id: ConnectionId,
2805 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2806 self.agent_declarations.close(connection_id);
2820 self.close_catalog_subscriptions(connection_id);
2821 #[cfg(test)]
2822 self.pause_cleanup_for_test(0);
2823 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
2824 spawn.retire_registration_channel();
2825 }
2826 self.supervisor
2827 .forget_registration_connection(connection_id);
2828 #[cfg(test)]
2829 self.pause_cleanup_for_test(1);
2830 self.local_connections
2831 .lock()
2832 .unwrap_or_else(|p| p.into_inner())
2833 .remove(&connection_id);
2834 self.caller_endpoints
2835 .lock()
2836 .unwrap_or_else(|p| p.into_inner())
2837 .remove(&connection_id);
2838 self.build_class_warned
2839 .lock()
2840 .unwrap_or_else(|p| p.into_inner())
2841 .remove(&connection_id);
2842 self.observed_peers
2843 .lock()
2844 .unwrap_or_else(|p| p.into_inner())
2845 .remove(&connection_id);
2846 self.captured_peers
2847 .lock()
2848 .unwrap_or_else(|p| p.into_inner())
2849 .remove(&connection_id);
2850 self.config_desktop_stamps
2851 .lock()
2852 .unwrap_or_else(|p| p.into_inner())
2853 .remove(&connection_id);
2854 self.operator_apps
2855 .lock()
2856 .unwrap_or_else(|p| p.into_inner())
2857 .remove(&connection_id);
2858 self.forwarding
2859 .operator_confirms()
2860 .connection_closed(connection_id);
2861 let cleanup = self.cleanup_connection_with_end_reason(
2862 connection_id,
2863 RegistrationEndReason::ConnectionClosed,
2864 );
2865 self.claims.connection_closed(connection_id);
2868 cleanup
2869 }
2870
2871 fn cleanup_connection_with_end_reason(
2872 &self,
2873 connection_id: ConnectionId,
2874 requested_reason: RegistrationEndReason,
2875 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2876 #[cfg(test)]
2877 if let Some(pause) = self
2878 .activity_request_pause
2879 .lock()
2880 .unwrap()
2881 .as_ref()
2882 .filter(|pause| pause.connection == connection_id)
2883 {
2884 pause
2885 .cleanup_started
2886 .store(true, std::sync::atomic::Ordering::Release);
2887 }
2888 if let Some(runtime) = &self.config_runtime {
2889 runtime.disconnected(connection_id);
2890 }
2891 self.agent_declarations.close(connection_id);
2892 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2893 self.registry
2894 .get_module_by_connection(connection_id)?
2895 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2896 .and_then(|module| module.registration_end_reason().ok().flatten())
2897 .unwrap_or(requested_reason)
2898 } else {
2899 requested_reason
2900 };
2901 let crash_closed = self
2902 .registry
2903 .get_module_by_connection(connection_id)?
2904 .and_then(|registration| {
2905 self.forwarding
2906 .module_endpoint_for_connection(connection_id)
2907 .ok()
2908 .flatten()
2909 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2910 .map(|routes| (registration.manifest.module_id, routes))
2911 });
2912 let crash_closed = crash_closed.map(|(module_id, routes)| {
2913 let terminal = match self.supervisor.get(&module_id) {
2914 None => false,
2915 Some(module) => match module.will_recover_after_connection_loss() {
2916 Ok(will_recover) => !will_recover,
2917 Err(err) => {
2918 warn!(
2919 %module_id,
2920 error = %err,
2921 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2922 );
2923 false
2924 }
2925 },
2926 };
2927 let (reason, drained) = match self
2931 .forwarding
2932 .daemon_drain_result_for_connection(connection_id)
2933 {
2934 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2935 Ok(None) => (RouteCloseReason::Crash, false),
2936 Err(err) => {
2937 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2938 (RouteCloseReason::Crash, false)
2939 }
2940 };
2941 (module_id, routes, reason, drained, terminal)
2942 });
2943 let registrations = self.deregister_connection(connection_id, end_reason);
2944 let ended_authority = self
2947 .forwarding
2948 .target_authority_for_connection(connection_id)
2949 .ok()
2950 .flatten();
2951 let cleanup = if crash_closed.is_some() {
2952 self.forwarding.cleanup_connection_counted(connection_id)
2953 } else {
2954 self.forwarding
2960 .cleanup_connection(connection_id)
2961 .map(|released| crate::forwarding::ConnectionCleanup {
2962 released,
2963 abandoned_relays: 0,
2964 })
2965 };
2966 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2971 let abandoned = cleanup
2972 .as_ref()
2973 .map_or(0, |cleanup| cleanup.abandoned_relays);
2974 send_route_control_pushes(
2975 &self.forwarding,
2976 routes,
2977 ClientControlPush::RouteClosed {
2978 module_id,
2979 channels: Vec::new(),
2980 reason,
2981 drained,
2982 abandoned,
2983 excluded_subscriptions: 0,
2984 terminal: Some(terminal),
2985 },
2986 );
2987 }
2988 if let Ok(cleanup) = cleanup {
2989 self.emit_route_goodbyes(cleanup.released);
2990 }
2991 if let Some(record) = ended_authority {
2992 self.change_spawn_authority(&record);
2993 }
2994 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2998 crate::supervise::notify_registration_release();
2999 self.capability_evaluator.wake_deadline_loop();
3000 self.refresh_capability_requirements();
3001 }
3002 self.supervisor.remove_spawn_subscribers(connection_id);
3003 self.activity_streams
3004 .lock()
3005 .unwrap_or_else(|p| p.into_inner())
3006 .retain(|(id, _), task| {
3007 if *id == connection_id {
3008 task.abort();
3009 false
3010 } else {
3011 true
3012 }
3013 });
3014 self.hello_launch_nonces
3017 .lock()
3018 .unwrap_or_else(|poisoned| poisoned.into_inner())
3019 .forget(connection_id);
3020 self.scopes
3021 .write()
3022 .unwrap_or_else(|poisoned| poisoned.into_inner())
3023 .release_connection(connection_id);
3024 registrations
3025 }
3026
3027 pub(crate) fn handle_route_goodbye(
3028 &self,
3029 connection_id: ConnectionId,
3030 route_channel: u16,
3031 route_epoch: u32,
3032 ) -> Result<bool, RouterError> {
3033 debug!(
3034 connection_id = connection_id.get(),
3035 route_channel, route_epoch, "handling route GOODBYE"
3036 );
3037 let RouteRelease::Removed(released_route) = self
3038 .forwarding
3039 .release_client_route(connection_id, route_channel, route_epoch)
3040 .map_err(RouterError::Forwarding)?
3041 else {
3042 return Ok(false);
3043 };
3044 self.emit_route_goodbyes(vec![released_route]);
3045 Ok(true)
3046 }
3047
3048 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
3049 scope_expiry::ScopeExpiryContext::new(self).emit_route_goodbyes(released_routes);
3050 }
3051
3052 fn send_abandoned_route_bind_goodbye(
3064 &self,
3065 module_sink: &crate::FrameSink,
3066 negotiated_ver: u8,
3067 module_channel: u16,
3068 module_epoch: u32,
3069 ) {
3070 let frame = match Frame::build_with_version(
3071 negotiated_ver,
3072 FrameType::Goodbye,
3073 control_flags(),
3074 module_channel,
3075 module_epoch,
3076 0,
3077 Vec::new(),
3078 ) {
3079 Ok(frame) => frame,
3080 Err(err) => {
3081 warn!(
3082 route_channel = module_channel,
3083 error = %err,
3084 "failed to build GOODBYE for abandoned route.bind"
3085 );
3086 return;
3087 }
3088 };
3089 crate::forwarding::send_module_route_goodbye(
3090 &self.counters,
3091 module_sink,
3092 frame,
3093 None,
3094 "abandoned route.bind",
3095 );
3096 }
3097
3098 #[cfg(test)]
3099 fn handle_hello(
3100 &self,
3101 connection_id: ConnectionId,
3102 egress: Option<crate::router::FrameSink>,
3103 frame: Frame,
3104 ) -> Result<Vec<Frame>, RouterError> {
3105 let binding = self.claims.binding(connection_id);
3106 let channel_spawn = self.supervisor.registration_spawn(connection_id);
3107 self.handle_hello_checked(connection_id, egress, frame, false, binding, channel_spawn)
3108 }
3109
3110 fn handle_hello_checked(
3111 &self,
3112 connection_id: ConnectionId,
3113 sink: Option<crate::FrameSink>,
3114 frame: Frame,
3115 wait_for_gate: bool,
3116 prefix_binding: Option<claims::ClaimBinding>,
3117 channel_spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3118 ) -> Result<Vec<Frame>, RouterError> {
3119 let primary_sink = channel_spawn.as_ref().and_then(|_| sink.clone());
3120 #[cfg(test)]
3121 if prefix_binding.is_some() {
3122 self.claims.pause_prefix_hello_for_test(5);
3123 match self.claims.take_prefix_hello_failure_for_test() {
3124 Some("error") => {
3125 return Err(RouterError::backend(
3126 0,
3127 frame.header.corr,
3128 "injected HELLO failure",
3129 ))
3130 }
3131 Some("panic") => panic!("injected HELLO task panic"),
3132 _ => {}
3133 }
3134 }
3135 if let Some(binding) = &prefix_binding {
3136 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3137 let module = value
3138 .as_ref()
3139 .and_then(|value| value["manifest"]["module_id"].as_str());
3140 if !binding.admit_prefix_hello(module)
3141 || !self.prefix_binding_live(binding, connection_id)
3142 || !module.is_some_and(|module| {
3143 self.supervisor
3144 .prefix_claim_authorized(&binding.registration.module_id, module)
3145 })
3146 || binding.registration.check().is_err()
3147 {
3148 return Ok(vec![control_error_frame(
3149 &frame,
3150 "claim_prefix_refused",
3151 "HELLO must name the prefix claim's exact module id once",
3152 )?]);
3153 }
3154 }
3155 struct RefuseUnregisteredChannel {
3156 spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3157 sink: Option<crate::FrameSink>,
3158 }
3159 impl Drop for RefuseUnregisteredChannel {
3160 fn drop(&mut self) {
3161 if let Some(spawn) = &self.spawn {
3162 if !spawn.registered() {
3163 if let Some(sink) = &self.sink {
3164 sink.terminate_ingress();
3165 }
3166 spawn.retire_registration_channel();
3167 }
3168 }
3169 }
3170 }
3171 let _channel_attempt = RefuseUnregisteredChannel {
3172 spawn: channel_spawn.clone(),
3173 sink: sink.clone(),
3174 };
3175 if let Some(spawn) = &channel_spawn {
3176 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3177 if let Err(code) = spawn.admit_hello(
3178 value
3179 .as_ref()
3180 .and_then(|value| value["manifest"]["module_id"].as_str()),
3181 ) {
3182 return Ok(vec![control_error_frame(
3183 &frame,
3184 code,
3185 "registration channel HELLO refused",
3186 )?]);
3187 }
3188 }
3189 debug!(
3190 connection_id = connection_id.get(),
3191 corr = frame.header.corr,
3192 "handling HELLO"
3193 );
3194 if self
3198 .registry
3199 .get_module_by_connection(connection_id)
3200 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3201 .is_some()
3202 {
3203 return Ok(vec![control_error_frame(
3204 &frame,
3205 "invalid_hello",
3206 "connection is already registered as a module",
3207 )?]);
3208 }
3209 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
3210 Ok(value) => value,
3211 Err(err) => {
3212 return Ok(vec![control_error_frame(
3213 &frame,
3214 "invalid_hello",
3215 format!("malformed HELLO body: {err}"),
3216 )?])
3217 }
3218 };
3219 if let Err(detail) = catalog::validate_hello_declarations(&hello_value) {
3220 return Ok(vec![control_error_frame(
3221 &frame,
3222 error_codes::INVALID_PROVIDER_DECLARATIONS,
3223 detail,
3224 )?]);
3225 }
3226 if let Some(module) = hello_value
3227 .get("manifest")
3228 .and_then(|value| value.get("module_id"))
3229 .and_then(|value| value.as_str())
3230 {
3231 let recovering = self
3232 .config_runtime
3233 .as_ref()
3234 .is_some_and(|owner| owner.check_recovery().unwrap_or(true));
3235 if recovering
3239 && channel_spawn.as_ref().is_none_or(|spawn| {
3240 !self.supervisor.has_live_verified_channel_binding(spawn)
3241 || spawn.check().is_err()
3242 })
3243 && !(channel_spawn.is_none()
3244 && hello_value
3245 .get("launch_nonce")
3246 .and_then(|value| value.as_str())
3247 .is_some_and(|nonce| {
3248 self.supervisor.spawned_consumer_authorized(module, nonce)
3249 && self
3250 .supervisor
3251 .has_live_verified_program_binding(module, nonce)
3252 }))
3253 {
3254 return Ok(vec![control_error_frame(
3255 &frame,
3256 "config_gated",
3257 "recovery admits only previously bound supervised launches",
3258 )?]);
3259 }
3260 if self
3261 .config_runtime
3262 .as_ref()
3263 .is_some_and(|owner| owner.module_is_gated(module))
3264 && (channel_spawn
3265 .as_ref()
3266 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3267 || self.config_runtime.as_ref().is_some_and(|owner| {
3268 !recovering && owner.check_module_hold(module).is_err()
3269 }))
3270 {
3271 return Ok(vec![control_error_frame(
3272 &frame,
3273 "reserved_module",
3274 "a gated identity belongs only to its supervised child",
3275 )?]);
3276 }
3277 }
3278 if channel_spawn.is_none() {
3279 if let (Some(module), Some(nonce)) = (
3280 hello_value
3281 .get("manifest")
3282 .and_then(|value| value.get("module_id"))
3283 .and_then(|value| value.as_str()),
3284 hello_value
3285 .get("launch_nonce")
3286 .and_then(|value| value.as_str()),
3287 ) {
3288 let binding = self.supervisor.has_program_binding_record(module, nonce);
3289 let required = self
3290 .config_runtime
3291 .as_ref()
3292 .is_some_and(|owner| owner.module_is_gated(module));
3293 if let Some(error) = (binding || required)
3294 .then(|| {
3295 self.supervisor
3296 .check_current_program_binding(module, nonce)
3297 .err()
3298 })
3299 .flatten()
3300 {
3301 let code = if error.code == "program_refused" {
3302 "program_refused"
3303 } else {
3304 "config_delivery_deferred"
3305 };
3306 self.request_refusal_close(
3307 connection_id,
3308 sink.as_ref(),
3309 CloseReason::new(code, error.detail.clone()),
3310 );
3311 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3312 }
3313 }
3314 }
3315 if let Err(err) = validate_hello_display_name(&hello_value) {
3316 return Ok(vec![control_error_frame(
3317 &frame,
3318 "invalid_manifest",
3319 err.to_string(),
3320 )?]);
3321 }
3322 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
3323 return Ok(vec![control_error_frame(
3324 &frame,
3325 "invalid_capability_grammar",
3326 err.to_string(),
3327 )?]);
3328 }
3329 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
3330 return Ok(vec![control_error_frame(
3331 &frame,
3332 "invalid_manifest",
3333 err.to_string(),
3334 )?]);
3335 }
3336 if let Err(err) = validate_hello_event_declarations(&hello_value) {
3337 return Ok(vec![control_error_frame(
3338 &frame,
3339 "invalid_event_declaration",
3340 err.to_string(),
3341 )?]);
3342 }
3343 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
3344 return Ok(vec![control_error_frame(
3345 &frame,
3346 "invalid_manifest",
3347 err.to_string(),
3348 )?]);
3349 }
3350 if let Some(provenance) = hello_value
3351 .get("manifest")
3352 .and_then(|manifest| manifest.get("provenance"))
3353 {
3354 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
3355 return Ok(vec![control_error_frame(
3356 &frame,
3357 "invalid_manifest",
3358 format!("malformed manifest provenance: {err}"),
3359 )?]);
3360 }
3361 }
3362 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
3363 let module_id = hello_value["manifest"]["module_id"]
3364 .as_str()
3365 .unwrap_or("<unknown>");
3366 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
3367 }
3368 if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
3369 let module_id = hello_value
3370 .get("manifest")
3371 .and_then(|m| m.get("module_id"))
3372 .and_then(serde_json::Value::as_str)
3373 .unwrap_or("unknown");
3374 warn!(module_id, %reason, "dropping operation_deferral from HELLO");
3375 }
3376 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
3377 let module_id = hello_value["manifest"]["module_id"]
3378 .as_str()
3379 .unwrap_or("<unknown>");
3380 warn!(module_id, %reason, "dropping flow_operations from HELLO");
3381 }
3382 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
3383 let module_id = hello_value["manifest"]["module_id"]
3384 .as_str()
3385 .unwrap_or("<unknown>");
3386 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
3387 }
3388 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
3389 Ok(hello) => hello,
3390 Err(err) => {
3391 return Ok(vec![control_error_frame(
3392 &frame,
3393 "invalid_hello",
3394 format!("malformed HELLO body: {err}"),
3395 )?])
3396 }
3397 };
3398
3399 if hello.protocol_ver != hello.manifest.protocol_ver {
3400 return Ok(vec![control_error_frame(
3401 &frame,
3402 "invalid_manifest",
3403 format!(
3404 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
3405 hello.protocol_ver, hello.manifest.protocol_ver
3406 ),
3407 )?]);
3408 }
3409
3410 if hello.manifest.module_id.trim().is_empty() {
3411 return Ok(vec![control_error_frame(
3412 &frame,
3413 "invalid_manifest",
3414 "manifest module_id must not be empty",
3415 )?]);
3416 }
3417
3418 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
3419 Ok(negotiated_ver) => negotiated_ver,
3420 Err(message) => {
3421 return Ok(vec![control_error_frame(
3422 &frame,
3423 "version_unsupported",
3424 message,
3425 )?])
3426 }
3427 };
3428
3429 let _publication_gate = if let Some(owner) = &self.config_runtime {
3435 let gate = owner
3436 .module_gate(&hello.manifest.module_id)
3437 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3438 Some(if wait_for_gate {
3439 gate.blocking_lock_owned()
3440 } else {
3441 match gate.try_lock_owned() {
3442 Ok(guard) => guard,
3443 Err(_) => {
3444 return Ok(vec![control_error_frame(
3445 &frame,
3446 "config_delivery_deferred",
3447 "module admission is busy",
3448 )?])
3449 }
3450 }
3451 })
3452 } else {
3453 None
3454 };
3455 #[cfg(test)]
3456 if let Some(hold) = &self.hello_before_publication_hold {
3457 hold.wait();
3458 hold.wait();
3459 }
3460 let module = &hello.manifest.module_id;
3461 let required = self
3462 .config_runtime
3463 .as_ref()
3464 .is_some_and(|owner| owner.module_is_gated(module));
3465 let recovering = self
3466 .config_runtime
3467 .as_ref()
3468 .is_some_and(|owner| owner.is_recovering());
3469 if recovering
3477 && channel_spawn.as_ref().is_none_or(|spawn| {
3478 !self.supervisor.has_live_verified_channel_binding(spawn) || spawn.check().is_err()
3479 })
3480 && !(channel_spawn.is_none()
3481 && hello.launch_nonce.as_deref().is_some_and(|nonce| {
3482 self.supervisor.spawned_consumer_authorized(module, nonce)
3483 && self
3484 .supervisor
3485 .has_live_verified_program_binding(module, nonce)
3486 }))
3487 {
3488 return Ok(vec![control_error_frame(
3489 &frame,
3490 "config_gated",
3491 "recovery admits only previously bound supervised launches",
3492 )?]);
3493 }
3494 if required
3495 && (channel_spawn
3496 .as_ref()
3497 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3498 || self
3499 .config_runtime
3500 .as_ref()
3501 .is_some_and(|owner| !recovering && owner.check_module_hold(module).is_err()))
3502 {
3503 return Ok(vec![control_error_frame(
3504 &frame,
3505 "reserved_module",
3506 "gated admission requires the current supervised child's registration channel and verified image binding",
3507 )?]);
3508 }
3509 if let Some(spawn) = &channel_spawn {
3510 if let Err(code) = spawn.check() {
3511 return Ok(vec![control_error_frame(
3512 &frame,
3513 code,
3514 "registration launch is no longer live",
3515 )?]);
3516 }
3517 } else if let Some(nonce) = hello.launch_nonce.as_deref() {
3518 if required || self.supervisor.has_program_binding_record(module, nonce) {
3519 if let Err(error) = self.supervisor.check_current_program_binding(module, nonce) {
3520 let code = if error.code == "program_refused" {
3521 "program_refused"
3522 } else {
3523 "config_delivery_deferred"
3524 };
3525 self.request_refusal_close(
3526 connection_id,
3527 sink.as_ref(),
3528 CloseReason::new(code, error.detail.clone()),
3529 );
3530 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3531 }
3532 }
3533 }
3534
3535 let swap_admission = match &channel_spawn {
3543 Some(spawn) if spawn.is_candidate() => SwapHelloAdmission::Candidate,
3544 Some(_) => SwapHelloAdmission::NotSwapping,
3545 None if prefix_binding.is_some() => SwapHelloAdmission::NotSwapping,
3546 None => self
3547 .supervisor
3548 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref()),
3549 };
3550 if swap_admission == SwapHelloAdmission::ProtocolNone {
3551 warn!(
3552 module_id = %hello.manifest.module_id,
3553 connection_id = connection_id.get(),
3554 "HELLO refused: the module_id belongs to a protocol:none module, which never registers"
3555 );
3556 return Ok(vec![control_error_frame(
3559 &frame,
3560 "swap_token_invalid",
3561 format!(
3562 "module_id '{}' is a protocol:none module; a protocol:none module may not register",
3563 hello.manifest.module_id
3564 ),
3565 )?]);
3566 }
3567 if swap_admission == SwapHelloAdmission::ChannelRequired {
3568 warn!(
3569 module_id = %hello.manifest.module_id,
3570 connection_id = connection_id.get(),
3571 "HELLO refused: the module_id is gated and the HELLO did not arrive over its registration channel"
3572 );
3573 return Ok(vec![control_error_frame(
3576 &frame,
3577 "swap_token_invalid",
3578 format!(
3579 "module_id '{}' is gated; it must register over its supervised registration channel",
3580 hello.manifest.module_id
3581 ),
3582 )?]);
3583 }
3584 if swap_admission == SwapHelloAdmission::Refused {
3585 warn!(
3586 module_id = %hello.manifest.module_id,
3587 connection_id = connection_id.get(),
3588 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
3589 );
3590 return Ok(vec![control_error_frame(
3591 &frame,
3592 "swap_token_invalid",
3593 format!(
3594 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
3595 hello.manifest.module_id
3596 ),
3597 )?]);
3598 }
3599 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
3600
3601 let verification = if swap_candidate && channel_spawn.is_none() {
3606 ReservedHelloVerification {
3607 nonce_verified: true,
3608 prefix_owner: None,
3609 }
3610 } else {
3611 let verification = if let Some(binding) = &prefix_binding {
3612 if !self.supervisor.prefix_claim_authorized(
3613 &binding.registration.module_id,
3614 &hello.manifest.module_id,
3615 ) {
3616 return Ok(vec![control_error_frame(
3617 &frame,
3618 "claim_prefix_refused",
3619 "prefix ownership changed",
3620 )?]);
3621 }
3622 Ok(ReservedHelloVerification {
3623 nonce_verified: true,
3624 prefix_owner: Some(binding.registration.module_id.clone()),
3625 })
3626 } else if channel_spawn.is_some() {
3627 self.supervisor
3628 .channel_hello_verification(&hello.manifest.module_id)
3629 } else {
3630 self.supervisor.reserved_hello_verification(
3631 &hello.manifest.module_id,
3632 hello.launch_nonce.as_deref(),
3633 )
3634 };
3635 match verification {
3636 Ok(verified) => verified,
3637 Err(rejection) => {
3638 let message = match rejection {
3639 ReservedHelloRejection::Exact { module_id } => format!(
3640 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
3641 ),
3642 ReservedHelloRejection::Prefix {
3643 prefix,
3644 owner_module_id,
3645 } => format!(
3646 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
3647 hello.manifest.module_id
3648 ),
3649 };
3650 return Ok(vec![control_error_frame(
3651 &frame,
3652 "reserved_module",
3653 message,
3654 )?]);
3655 }
3656 }
3657 };
3658
3659 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
3660 &hello.manifest.module_id,
3661 hello.manifest.capabilities.as_ref(),
3662 );
3663 if let Some(refusal) = reserved_capability_refusals.first() {
3664 let capability = refusal.capability.clone();
3665 let bound_module = refusal.claimants[0].clone();
3666 log_duplicate_claim_events(reserved_capability_refusals);
3667 return Ok(vec![control_error_frame(
3668 &frame,
3669 "reserved_capability",
3670 format!(
3671 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3672 capability, bound_module, hello.manifest.module_id
3673 ),
3674 )?]);
3675 }
3676
3677 if self
3680 .forwarding
3681 .connection_has_client_routes(connection_id)
3682 .map_err(RouterError::Forwarding)?
3683 {
3684 return Ok(vec![control_error_frame(
3685 &frame,
3686 "invalid_hello",
3687 "connection has open client routes and cannot also register as a module",
3688 )?]);
3689 }
3690
3691 self.hello_launch_nonces
3699 .lock()
3700 .unwrap_or_else(|poisoned| poisoned.into_inner())
3701 .record(
3702 connection_id,
3703 if prefix_binding.is_some() || channel_spawn.is_some() {
3704 None
3705 } else {
3706 hello.launch_nonce.as_deref()
3707 },
3708 );
3709 let control_ops = effective_module_control_ops(hello.control_ops);
3710 let prefix_submodule = verification.prefix_owner.is_some();
3711 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
3714 #[cfg(test)]
3715 struct PublicationFinished(Arc<std::sync::Barrier>);
3716 #[cfg(test)]
3717 impl Drop for PublicationFinished {
3718 fn drop(&mut self) {
3719 self.0.wait();
3720 }
3721 }
3722 #[cfg(test)]
3723 let _publication_finished = self.hello_after_launch_check_hold.as_ref().map(|hold| {
3724 hold.wait();
3725 hold.wait();
3726 PublicationFinished(hold.clone())
3727 });
3728 let publication_spawn = channel_spawn
3729 .as_ref()
3730 .or_else(|| prefix_binding.as_ref().map(|binding| &binding.registration));
3731 #[cfg(test)]
3732 if prefix_binding.is_some() {
3733 self.claims.pause_prefix_hello_for_test(7);
3734 }
3735 let _registration_publication = match publication_spawn {
3736 Some(spawn) => Some(match spawn.begin_publication() {
3737 Ok(guard) => guard,
3738 Err(code) => {
3739 return Ok(vec![control_error_frame(
3740 &frame,
3741 code,
3742 "registration ended before HELLO publication",
3743 )?])
3744 }
3745 }),
3746 None => None,
3747 };
3748 let (_prefix_live, _prefix_ownership) = if let Some(binding) = &prefix_binding {
3749 let live = self.claims.lock_prefix_binding(binding, connection_id);
3750 let ownership = self.supervisor.prefix_claim_publication(
3751 &binding.registration.module_id,
3752 &hello.manifest.module_id,
3753 );
3754 if live.is_none()
3755 || ownership.is_none()
3756 || !self.claim_primary_registered(binding.spawn, &binding.registration)
3757 || binding.prefix_module_id() != Some(hello.manifest.module_id.as_str())
3758 {
3759 return Ok(vec![control_error_frame(
3760 &frame,
3761 "claim_prefix_refused",
3762 "prefix authority ended before publication",
3763 )?]);
3764 }
3765 (live, ownership)
3766 } else {
3767 (None, None)
3768 };
3769 if let Some(spawn) = publication_spawn {
3770 self.forwarding
3771 .bind_spawn(connection_id, spawn)
3772 .map_err(RouterError::Forwarding)?;
3773 }
3774 let hello_authority = match (&channel_spawn, &prefix_binding) {
3784 (Some(spawn), _) => Some(spawn.authority.clone()),
3785 (None, None) => hello.launch_nonce.as_deref().and_then(|nonce| {
3786 self.supervisor
3787 .spawned_consumer_record(&hello.manifest.module_id, nonce)
3788 }),
3789 (None, Some(_)) => None,
3790 }
3791 .map(|record| HelloAuthority {
3792 record,
3793 requested: hello
3794 .manifest
3795 .provider_declarations
3796 .clone()
3797 .unwrap_or_default(),
3798 froze: Default::default(),
3799 });
3800 #[cfg(test)]
3801 if prefix_binding.is_some() {
3802 self.claims.pause_prefix_hello_for_test(8);
3803 }
3804 if swap_candidate {
3805 let module_id = hello.manifest.module_id.clone();
3806 let mut differing_declarations = Default::default();
3807 let reply = self.register_swap_candidate(
3808 connection_id,
3809 sink,
3810 &frame,
3811 hello.manifest,
3812 negotiated_ver,
3813 control_ops,
3814 hello_ack,
3815 hello_authority.as_ref(),
3816 &mut differing_declarations,
3817 #[cfg(windows)]
3818 hello.launch_nonce.as_deref(),
3819 )?;
3820 if let Some(spawn) = &channel_spawn {
3821 if self
3822 .registry
3823 .get_module_by_connection(connection_id)
3824 .ok()
3825 .flatten()
3826 .is_some()
3827 {
3828 if let Some(sink) = primary_sink {
3829 spawn.bind_primary_sink(sink);
3830 }
3831 spawn.hello_accepted();
3832 }
3833 } else if self
3834 .registry
3835 .get_module_by_connection(connection_id)
3836 .ok()
3837 .flatten()
3838 .is_some()
3839 {
3840 self.supervisor
3841 .retire_unused_registration_channel(&module_id, true);
3842 }
3843 if self
3844 .registry
3845 .get_module_by_connection(connection_id)
3846 .ok()
3847 .flatten()
3848 .is_some()
3849 {
3850 self.file_development_widening(
3851 hello_authority.map(|authority| authority.record),
3852 differing_declarations,
3853 );
3854 }
3855 return Ok(reply);
3856 }
3857 let registration = match self
3864 .registry
3865 .register_unpublished_verified_with_control_ops(
3866 hello.manifest,
3867 negotiated_ver,
3868 connection_id,
3869 control_ops,
3870 verification.nonce_verified,
3871 ) {
3872 Ok(registration) => registration,
3873 Err(RegistryError::DuplicateModuleId { module_id }) => {
3874 return Ok(vec![control_error_frame(
3875 &frame,
3876 "duplicate_module_id",
3877 format!(
3878 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
3879 ),
3880 )?])
3881 }
3882 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3883 return Ok(vec![control_error_frame(
3884 &frame,
3885 "invalid_module_id",
3886 err.to_string(),
3887 )?])
3888 }
3889 Err(err) => {
3890 return Ok(vec![control_error_frame(
3891 &frame,
3892 "registry_error",
3893 err.to_string(),
3894 )?])
3895 }
3896 };
3897 let differing_declarations =
3898 match self.admit_hello_authority(connection_id, hello_authority.as_ref()) {
3899 Ok(differing) => differing,
3900 Err(err) => {
3901 if matches!(
3902 self.rollback_unpublished_hello(connection_id),
3903 Ok(r) if !r.is_empty()
3904 ) {
3905 crate::supervise::notify_registration_release();
3906 }
3907 return Err(RouterError::Forwarding(err));
3908 }
3909 };
3910 #[cfg(test)]
3911 self.hold_between_registry_and_forwarding();
3912
3913 let mut reply = Vec::new();
3914 let publication = self
3917 .registry
3918 .publish_registration_after(connection_id, || {
3919 if let Some(sink) = sink {
3920 let concurrency = manifest_concurrency(®istration.manifest);
3936 self.forwarding.register_module_connection_acked(
3937 connection_id,
3938 registration.manifest.module_id.clone(),
3939 negotiated_ver,
3940 concurrency,
3941 sink,
3942 crate::forwarding::ModuleHelloAck {
3943 frame: hello_ack,
3944 verified_prefix_owner: verification.prefix_owner,
3945 },
3946 )?;
3947 } else {
3948 reply.push(hello_ack);
3951 }
3952 #[cfg(test)]
3953 self.hold_between_ack_and_publication();
3954 Ok(())
3955 })
3956 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3957 let published = match publication {
3958 Ok(published) => published,
3959 Err(err) => {
3960 if matches!(
3967 self.rollback_unpublished_hello(connection_id),
3968 Ok(r) if !r.is_empty()
3969 ) {
3970 crate::supervise::notify_registration_release();
3971 }
3972 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
3973 return Ok(vec![control_error_frame(
3974 &frame,
3975 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
3976 "invalid_hello"
3977 } else {
3978 forwarding_error_code(&err)
3979 },
3980 err.to_string(),
3981 )?]);
3982 }
3983 };
3984 if !published {
3985 debug!(
3986 module_id = %registration.manifest.module_id,
3987 connection_id = connection_id.get(),
3988 "registration was removed before it could be published"
3989 );
3990 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
3991 return Ok(reply);
3992 }
3993
3994 drop(_prefix_ownership);
3995 drop(_prefix_live);
3996
3997 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
4006 info!(
4007 module_id = %registration.manifest.module_id,
4008 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
4009 );
4010 }
4011 if registration.manifest.registration_channel == Some(false)
4012 && self
4013 .supervisor
4014 .module_is_gated(®istration.manifest.module_id)
4015 && self
4016 .config_runtime
4017 .as_ref()
4018 .and_then(|owner| owner.registration_declaration(®istration.manifest.module_id))
4019 .as_deref()
4020 == Some("declared")
4021 {
4022 warn!(module_id = %registration.manifest.module_id,
4023 "HELLO registration_channel is false despite the approved channel declaration; registration remains admitted");
4024 }
4025
4026 self.apply_registration_capabilities(®istration);
4027
4028 self.supervisor
4029 .record_activity_registration(®istration.manifest.module_id);
4030
4031 info!(
4032 module_id = %registration.manifest.module_id,
4033 module_version = %registration.manifest.module_version,
4034 negotiated_ver,
4035 routable_provider = manifest_provides_routable_role(®istration.manifest),
4036 connection_id = connection_id.get(),
4037 "module registered"
4038 );
4039
4040 if let Some(spawn) = &channel_spawn {
4041 if let Some(sink) = primary_sink {
4042 spawn.bind_primary_sink(sink);
4043 }
4044 spawn.hello_accepted();
4045 } else if !prefix_submodule {
4046 self.supervisor
4047 .retire_unused_registration_channel(®istration.manifest.module_id, false);
4048 }
4049 self.file_development_widening(
4050 hello_authority.map(|authority| authority.record),
4051 differing_declarations,
4052 );
4053 #[cfg(windows)]
4054 if let Some(nonce) = hello.launch_nonce.as_deref() {
4055 self.supervisor
4056 .program_hello_accepted(®istration.manifest.module_id, nonce);
4057 }
4058
4059 #[cfg(test)]
4060 if let Some(hold) = &self.hello_inside_publication_hold {
4061 hold.wait();
4062 hold.wait();
4063 }
4064
4065 Ok(reply)
4066 }
4067
4068 #[allow(clippy::too_many_arguments)]
4085 fn register_swap_candidate(
4086 &self,
4087 connection_id: ConnectionId,
4088 sink: Option<crate::FrameSink>,
4089 frame: &Frame,
4090 manifest: ModuleManifest,
4091 negotiated_ver: u8,
4092 control_ops: Vec<String>,
4093 hello_ack: Frame,
4094 hello_authority: Option<&HelloAuthority>,
4095 differing_declarations: &mut ProviderDeclarations,
4096 #[cfg(windows)] launch_nonce: Option<&str>,
4097 ) -> Result<Vec<Frame>, RouterError> {
4098 let module_id = manifest.module_id.clone();
4099 self.supervisor
4102 .with_swap_candidate_admission(&module_id, || {
4103 let reply = self.publish_swap_candidate(
4104 connection_id,
4105 sink,
4106 frame,
4107 manifest,
4108 negotiated_ver,
4109 control_ops,
4110 hello_ack,
4111 hello_authority,
4112 differing_declarations,
4113 #[cfg(windows)]
4114 launch_nonce,
4115 );
4116 let admitted = self
4117 .registry
4118 .get_candidate(&module_id)
4119 .ok()
4120 .flatten()
4121 .is_some_and(|candidate| candidate.connection_id == connection_id);
4122 (reply, admitted)
4123 })
4124 .unwrap_or_else(|| {
4125 Ok(vec![control_error_frame(
4126 frame,
4127 "swap_token_invalid",
4128 "swap ended before candidate HELLO publication",
4129 )?])
4130 })
4131 }
4132
4133 #[allow(clippy::too_many_arguments)]
4134 fn publish_swap_candidate(
4135 &self,
4136 connection_id: ConnectionId,
4137 sink: Option<crate::FrameSink>,
4138 frame: &Frame,
4139 manifest: ModuleManifest,
4140 negotiated_ver: u8,
4141 control_ops: Vec<String>,
4142 hello_ack: Frame,
4143 hello_authority: Option<&HelloAuthority>,
4144 differing_declarations: &mut ProviderDeclarations,
4145 #[cfg(windows)] launch_nonce: Option<&str>,
4146 ) -> Result<Vec<Frame>, RouterError> {
4147 let module_id = manifest.module_id.clone();
4148 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
4149 manifest,
4150 negotiated_ver,
4151 connection_id,
4152 control_ops,
4153 ) {
4154 Ok(registration) => registration,
4155 Err(RegistryError::DuplicateModuleId { module_id }) => {
4156 return Ok(vec![control_error_frame(
4157 frame,
4158 "duplicate_module_id",
4159 format!(
4160 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
4161 ),
4162 )?])
4163 }
4164 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
4165 return Ok(vec![control_error_frame(
4166 frame,
4167 "invalid_module_id",
4168 err.to_string(),
4169 )?])
4170 }
4171 Err(err) => {
4172 return Ok(vec![control_error_frame(
4173 frame,
4174 "registry_error",
4175 err.to_string(),
4176 )?])
4177 }
4178 };
4179 *differing_declarations = match self.admit_hello_authority(connection_id, hello_authority) {
4180 Ok(differing) => differing,
4181 Err(err) => {
4182 if matches!(
4183 self.rollback_unpublished_hello(connection_id),
4184 Ok(r) if !r.is_empty()
4185 ) {
4186 crate::supervise::notify_registration_release();
4187 }
4188 return Err(RouterError::Forwarding(err));
4189 }
4190 };
4191 #[cfg(test)]
4192 self.hold_between_registry_and_forwarding();
4193 let mut reply = Vec::new();
4194 let publication = self
4195 .registry
4196 .publish_registration_after(connection_id, || {
4197 if let Some(sink) = sink {
4198 let concurrency = manifest_concurrency(®istration.manifest);
4202 self.forwarding.register_candidate_module_connection_acked(
4203 connection_id,
4204 module_id.clone(),
4205 negotiated_ver,
4206 concurrency,
4207 sink,
4208 hello_ack,
4209 )?;
4210 } else {
4211 reply.push(hello_ack);
4212 }
4213 #[cfg(test)]
4214 self.hold_between_ack_and_publication();
4215 Ok(())
4216 })
4217 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4218 let published = match publication {
4219 Ok(published) => published,
4220 Err(err) => {
4221 if matches!(
4222 self.rollback_unpublished_hello(connection_id),
4223 Ok(r) if !r.is_empty()
4224 ) {
4225 crate::supervise::notify_registration_release();
4226 }
4227 self.refuse_hello_authority(connection_id, hello_authority);
4228 return Ok(vec![control_error_frame(
4229 frame,
4230 forwarding_error_code(&err),
4231 err.to_string(),
4232 )?]);
4233 }
4234 };
4235 if !published {
4236 debug!(
4237 module_id = %module_id,
4238 connection_id = connection_id.get(),
4239 "swap candidate registration was removed before it could be published"
4240 );
4241 self.refuse_hello_authority(connection_id, hello_authority);
4242 return Ok(reply);
4243 }
4244 info!(
4245 module_id = %module_id,
4246 module_version = %registration.manifest.module_version,
4247 negotiated_ver,
4248 ready = registration.ready,
4249 connection_id = connection_id.get(),
4250 "swap candidate registered; not routable until cutover"
4251 );
4252 #[cfg(windows)]
4253 if let Some(nonce) = launch_nonce {
4254 self.supervisor.program_hello_accepted(&module_id, nonce);
4255 }
4256 Ok(reply)
4257 }
4258
4259 fn admit_hello_authority(
4265 &self,
4266 connection: ConnectionId,
4267 authority: Option<&HelloAuthority>,
4268 ) -> Result<ProviderDeclarations, ForwardingError> {
4269 let Some(authority) = authority else {
4270 return Ok(ProviderDeclarations::new());
4271 };
4272 let froze = authority
4273 .record
4274 .freeze_declarations(authority.requested.clone());
4275 authority
4276 .froze
4277 .store(froze.is_some(), std::sync::atomic::Ordering::Relaxed);
4278 self.forwarding
4279 .bind_target_authority(connection, authority.record.clone())
4280 .inspect_err(|_| self.refuse_hello_authority(connection, Some(authority)))?;
4281 Ok(froze.unwrap_or_default())
4282 }
4283
4284 fn refuse_hello_authority(&self, connection: ConnectionId, authority: Option<&HelloAuthority>) {
4289 let Some(authority) = authority else {
4290 return;
4291 };
4292 if let Err(err) = self.forwarding.unbind_target_authority(connection) {
4293 warn!(error = %err, "failed to unbind a refused HELLO from its spawn");
4294 }
4295 if authority
4296 .froze
4297 .swap(false, std::sync::atomic::Ordering::Relaxed)
4298 {
4299 authority.record.unfreeze_refused_declarations();
4300 }
4301 }
4302
4303 fn build_hello_ack(
4304 &self,
4305 frame: &Frame,
4306 negotiated_ver: u8,
4307 module_id: &str,
4308 ) -> Result<Frame, RouterError> {
4309 let ack = ModuleHelloAckBody {
4310 negotiated_ver,
4311 subc_ops: module_subc_ops(),
4312 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
4313 storage: self
4314 .storage_config
4315 .as_ref()
4316 .map(|cfg| cfg.descriptor_for(module_id)),
4317 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4318 };
4319 let body = serde_json::to_vec(&ack).map_err(|err| {
4320 RouterError::backend(
4321 0,
4322 frame.header.corr,
4323 format!("failed to encode HELLO_ACK: {err}"),
4324 )
4325 })?;
4326
4327 Frame::build_with_version(
4328 negotiated_ver,
4329 FrameType::HelloAck,
4330 control_flags(),
4331 0,
4332 0,
4333 frame.header.corr,
4334 body,
4335 )
4336 .map_err(RouterError::FrameBuild)
4337 }
4338
4339 async fn handle_client_control_request(
4340 &self,
4341 ctx: &RouteCtx,
4342 frame: Frame,
4343 request: ClientControlRequest,
4344 route_open_timings: &mut Option<RouteOpenTimings>,
4345 ) -> Result<Vec<Frame>, RouterError> {
4346 if let Some(owner) = &self.config_runtime {
4347 let module = match &request {
4348 ClientControlRequest::SupervisorRestart { module_id, .. }
4349 | ClientControlRequest::SupervisorReload { module_id }
4350 | ClientControlRequest::SupervisorSwap { module_id, .. }
4351 | ClientControlRequest::SupervisorSetEnabled {
4352 module_id,
4353 enabled: true,
4354 } => Some(module_id.as_str()),
4355 _ => None,
4356 };
4357 let rescan = matches!(
4358 &request,
4359 ClientControlRequest::SupervisorRescan { preview: false }
4360 | ClientControlRequest::SupervisorRescanChecked { preview: false, .. }
4361 );
4362 let held = if module.is_some() || rescan {
4363 let module = module.map(str::to_owned);
4364 owner
4365 .on_blocking(move |owner| owner.checked_admission_hold(module.as_deref()))
4366 .await
4367 } else {
4368 Ok(())
4369 };
4370 if let Err(error) = held {
4371 if error.code != "config_delivery_deferred" {
4372 let code = if error.code == "store_tampered" {
4373 "store_tampered"
4374 } else {
4375 "config_gated"
4376 };
4377 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
4378 }
4379 }
4380 }
4381 if let Some(operation) = privileged_operation(&request) {
4382 let caller = self.resolve_caller(ctx.connection_id).await;
4383 let op = client_control_request_op(&request);
4384 let text = prompt_text(operation, &caller);
4385 match self.privileged_operator_prompt {
4386 PrivilegedOperatorPrompt::Log => {
4387 info!(target: "subc_daemon::privileged_prompt", operation = op,
4388 target = ?operation, caller = %caller.description(), summary = %text,
4389 "privileged request would prompt the operator");
4390 }
4391 PrivilegedOperatorPrompt::Enforce => {
4392 let key =
4393 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
4394 let stamp = self
4395 .operator_apps
4396 .lock()
4397 .unwrap_or_else(|p| p.into_inner())
4398 .get(&ctx.connection_id)
4399 .cloned();
4400 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
4401 &self.operator_app_policy,
4402 &self.launch_table,
4403 stamp.as_ref(),
4404 )
4405 .then(|| self.shortcut_context(stamp, None, None));
4406 let outcome = self
4407 .forwarding
4408 .operator_confirms()
4409 .approve_privileged_with_shortcut(
4410 ctx,
4411 &frame,
4412 op,
4413 text,
4414 caller.description(),
4415 key,
4416 shortcut,
4417 )
4418 .await;
4419 if outcome.code != "confirmed" {
4420 return Ok(vec![outcome.frame(&frame)]);
4421 }
4422 }
4423 }
4424 }
4425 match request {
4426 ClientControlRequest::DaemonShutdownIntent { intent } => {
4427 if intent.as_str() != Some("restart") {
4428 return Ok(vec![control_error_frame(
4429 &frame,
4430 "invalid_params",
4431 "intent must be 'restart'",
4432 )?]);
4433 }
4434 if let Some(owner) = &self.config_runtime {
4439 match owner
4440 .on_blocking(|owner| owner.assign_daemon_restart())
4441 .await
4442 {
4443 Ok(Some(generation)) => {
4444 info!(
4445 generation,
4446 "daemon restart will warm up approved generation"
4447 )
4448 }
4449 Ok(None) => {}
4450 Err(error) => warn!(
4451 %error,
4452 "daemon restart intent not recorded; the next start boots the last active generation"
4453 ),
4454 }
4455 }
4456 let response = ClientControlResponse::DaemonShutdownIntent {
4457 intent: "restart".to_owned(),
4458 expires_at_ms: self.supervisor.announce_daemon_restart(),
4459 };
4460 Ok(vec![control_response_body_frame(
4461 &frame,
4462 &response,
4463 "ClientControlResponse::DaemonShutdownIntent",
4464 )?])
4465 }
4466 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
4467 ClientControlRequest::CatalogList { module_id } => {
4468 self.handle_catalog_list(frame, module_id)
4469 }
4470 ClientControlRequest::RouteOpen {
4471 target,
4472 identity,
4473 consumer_identity,
4474 consumer_capabilities,
4475 role_versions,
4476 busy_refusals,
4477 admission_facts,
4478 scope,
4479 ..
4480 } => {
4481 let extensions = match serde_json::from_slice::<subc_control::RouteOpenRequestExt>(
4482 &frame.body,
4483 ) {
4484 Ok(extensions) => extensions,
4485 Err(err) => {
4486 return Ok(vec![control_error_frame(
4487 &frame,
4488 "invalid_control_body",
4489 format!("malformed route.open extensions: {err}"),
4490 )?]);
4491 }
4492 };
4493 self.handle_route_open(
4494 ctx,
4495 frame,
4496 RouteOpenRequest {
4497 target,
4498 identity,
4499 consumer_identity,
4500 consumer_capabilities,
4501 role_versions,
4502 busy_refusals,
4503 admission_facts,
4504 scope,
4505 accepts_close_reasons: extensions.accepts_close_reasons,
4506 },
4507 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
4508 )
4509 .await
4510 }
4511 ClientControlRequest::RoutePoll {
4512 route_channel,
4513 route_epoch,
4514 kind,
4515 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
4516 ClientControlRequest::SupervisorList {} => {
4517 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
4518 self.agent_declarations.refuse(
4519 self,
4520 ctx,
4521 control_error_frame(
4522 &frame,
4523 "busy",
4524 "supervisor list work is at capacity; retry",
4525 )?,
4526 );
4527 return Ok(Vec::new());
4528 };
4529 let handler = self.clone();
4531 let sink = ctx.egress.clone();
4532 tokio::spawn(
4533 async move {
4534 let _permit = permit;
4535 let result = handler.handle_supervisor_list(frame.clone()).await;
4536 let replies = match result {
4537 Ok(replies) => replies,
4538 Err(error) => {
4539 control_error_frame(&frame, "backend_error", error.to_string())
4540 .into_iter()
4541 .collect()
4542 }
4543 };
4544 for reply in replies {
4545 if !matches!(
4546 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
4547 .await,
4548 Ok(Ok(()))
4549 ) {
4550 break;
4551 }
4552 }
4553 }
4554 .with_current_subscriber(),
4555 );
4556 Ok(Vec::new())
4557 }
4558 ClientControlRequest::SupervisorSpawnSnapshot {} => {
4559 self.handle_supervisor_spawn_snapshot(frame)
4560 }
4561 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
4562 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
4563 }
4564 ClientControlRequest::SupervisorRestart {
4565 module_id,
4566 drain_timeout_ms,
4567 } => {
4568 self.log_supervisor_request_received(
4569 ctx,
4570 frame.header.corr,
4571 ops::SUPERVISOR_RESTART,
4572 Some(&module_id),
4573 None,
4574 )?;
4575 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4576 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
4577 .await
4578 }
4579 ClientControlRequest::SupervisorSwap {
4580 module_id,
4581 ready_timeout_ms,
4582 } => {
4583 self.log_supervisor_request_received(
4584 ctx,
4585 frame.header.corr,
4586 ops::SUPERVISOR_SWAP,
4587 Some(&module_id),
4588 None,
4589 )?;
4590 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4591 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
4592 .await
4593 }
4594 ClientControlRequest::SupervisorReload { module_id } => {
4595 self.log_supervisor_request_received(
4596 ctx,
4597 frame.header.corr,
4598 ops::SUPERVISOR_RELOAD,
4599 Some(&module_id),
4600 None,
4601 )?;
4602 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4603 self.handle_supervisor_reload(frame, module_id, requester)
4604 .await
4605 }
4606 ClientControlRequest::SupervisorRescan { preview } => {
4607 if !preview {
4608 self.log_supervisor_request_received(
4609 ctx,
4610 frame.header.corr,
4611 ops::SUPERVISOR_RESCAN,
4612 None,
4613 None,
4614 )?;
4615 }
4616 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4617 let operation = self.handle_supervisor_rescan(
4618 frame,
4619 if preview {
4620 RescanMode::Preview
4621 } else {
4622 RescanMode::Execute
4623 },
4624 requester,
4625 );
4626 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
4627 .await
4628 }
4629 ClientControlRequest::SupervisorRescanChecked {
4630 preview,
4631 candidate,
4632 expect,
4633 } => {
4634 self.observe_control_wait(
4635 ctx,
4636 None,
4637 ops::SUPERVISOR_RESCAN,
4638 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
4639 )
4640 .await
4641 }
4642 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
4643 self.log_supervisor_request_received(
4644 ctx,
4645 frame.header.corr,
4646 ops::SUPERVISOR_RELEASE_RESERVED,
4647 Some(&module_id),
4648 None,
4649 )?;
4650 self.handle_supervisor_release_reserved(frame, module_id)
4651 .await
4652 }
4653 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
4654 self.log_supervisor_request_received(
4655 ctx,
4656 frame.header.corr,
4657 ops::SUPERVISOR_SET_ENABLED,
4658 Some(&module_id),
4659 Some(enabled),
4660 )?;
4661 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4662 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
4663 .await
4664 }
4665 ClientControlRequest::SupervisorHealthProbe { module_id } => {
4666 self.handle_supervisor_health_probe(ctx, frame, module_id)
4667 .await
4668 }
4669 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
4670 ClientControlRequest::SupervisorRoutes { module_id } => {
4671 self.handle_supervisor_routes(frame, module_id)
4672 }
4673 ClientControlRequest::SupervisorProvenance { module_id } => {
4674 self.handle_supervisor_provenance(ctx, frame, module_id)
4675 .await
4676 }
4677 ClientControlRequest::SupervisorStderrTail {
4678 module_id,
4679 max_lines,
4680 max_bytes,
4681 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
4682 ClientControlRequest::SupervisorTerminals { module_id } => {
4683 self.handle_supervisor_terminals(ctx, frame, module_id)
4684 .await
4685 }
4686 }
4687 }
4688
4689 fn log_supervisor_request_received(
4690 &self,
4691 ctx: &RouteCtx,
4692 corr: u64,
4693 op: &'static str,
4694 module_id: Option<&str>,
4695 enabled: Option<bool>,
4696 ) -> Result<(), RouterError> {
4697 let caller = self.supervisor_requester(ctx, corr)?;
4698 let caller = principal_label(&caller);
4699 info!(
4702 target: "control",
4703 op,
4704 module_id,
4705 enabled,
4706 connection_id = ctx.connection_id.get(),
4707 caller = %caller,
4708 "supervisor request received"
4709 );
4710 Ok(())
4711 }
4712
4713 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
4714 Ok(self
4715 .registry
4716 .get_module_by_connection(ctx.connection_id)
4717 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4718 .map(|registration| Principal::Reserved {
4719 module_id: registration.manifest.module_id,
4720 })
4721 .unwrap_or(Principal::Direct))
4722 }
4723
4724 fn handle_module_control_request(
4725 &self,
4726 connection_id: ConnectionId,
4727 frame: Frame,
4728 request: ModuleControlRequestFromModule,
4729 ) -> Result<Vec<Frame>, RouterError> {
4730 match request {
4731 ModuleControlRequestFromModule::RouteAuthorize {
4732 principal,
4733 scope,
4734 target,
4735 expected,
4736 } => self.handle_route_authorize(
4737 connection_id,
4738 frame,
4739 principal,
4740 scope,
4741 target,
4742 expected,
4743 ),
4744 ModuleControlRequestFromModule::CatalogUpdate {
4745 provides,
4746 capabilities,
4747 ready,
4748 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
4749 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
4750 self.handle_catalog_update_role_declarations(
4751 connection_id,
4752 frame,
4753 role_declarations,
4754 )
4755 }
4756 ModuleControlRequestFromModule::LiveRoots {} => {
4757 let registered = self
4758 .registry
4759 .get_module_by_connection(connection_id)
4760 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4761 let Some(registration) = registered else {
4762 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
4763 };
4764 let response = self
4765 .forwarding
4766 .live_roots(®istration.manifest.module_id)
4767 .map_err(RouterError::Forwarding)?;
4768 Ok(vec![control_response_body_frame(
4769 &frame,
4770 &response,
4771 "ModuleControlResponseToModule::LiveRoots",
4772 )?])
4773 }
4774 ModuleControlRequestFromModule::AgentCommands {
4775 started,
4776 finished,
4777 known,
4778 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
4779 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
4780 self.handle_scope_sync(connection_id, frame, generation, scopes)
4781 }
4782 ModuleControlRequestFromModule::ScopeApply {
4783 generation,
4784 upsert,
4785 end,
4786 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
4787 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
4788 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
4789 }
4790 _ => Ok(vec![control_error_frame(
4791 &frame,
4792 "unsupported_control_frame",
4793 "unsupported module control operation",
4794 )?]),
4795 }
4796 }
4797
4798 fn handle_scope_served_by(
4799 &self,
4800 connection_id: ConnectionId,
4801 frame: Frame,
4802 ) -> Result<Vec<Frame>, RouterError> {
4803 let registered = self
4804 .registry
4805 .get_module_by_connection(connection_id)
4806 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4807 let Some(registered) = registered else {
4808 return Ok(vec![control_error_frame(
4809 &frame,
4810 "not_registered",
4811 "scope.served_by requires an active module registration owned by this connection",
4812 )?]);
4813 };
4814 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
4815 {
4816 Ok(request) => request,
4817 Err(err) => {
4818 return Ok(vec![control_error_frame(
4819 &frame,
4820 "invalid_control_body",
4821 format!("malformed scope.served_by body: {err}"),
4822 )?])
4823 }
4824 };
4825 let answer = self
4826 .scopes
4827 .read()
4828 .unwrap_or_else(|poisoned| poisoned.into_inner())
4829 .served_by(
4830 ®istered.manifest.module_id,
4831 &request.scope_ref,
4832 request.scope_epoch,
4833 &request.module_id,
4834 );
4835 let reply = subc_control::ScopeServedByReply::new(
4836 answer,
4837 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4838 );
4839 Ok(vec![control_response_body_frame(
4840 &frame,
4841 &reply,
4842 "ScopeServedByReply",
4843 )?])
4844 }
4845
4846 fn handle_operator_confirm(
4847 &self,
4848 ctx: &RouteCtx,
4849 frame: Frame,
4850 ) -> Result<Vec<Frame>, RouterError> {
4851 use crate::operator_confirm::{audit, Outcome};
4852 let registration = self
4853 .registry
4854 .get_module_by_connection(ctx.connection_id)
4855 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4856 let Some(registration) = registration else {
4857 let outcome = Outcome::refusal("not_registered");
4858 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4859 return Ok(vec![outcome.frame(&frame)]);
4860 };
4861 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
4862 Ok(request) => request,
4863 Err(_) => {
4864 let outcome = Outcome::refusal("invalid_control_body");
4865 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4866 return Ok(vec![outcome.frame(&frame)]);
4867 }
4868 };
4869 let declared = matches!(
4870 request.kind,
4871 Some(subc_protocol::session::OperatorConfirmKind::Operation)
4872 ) && request.operation.as_ref().is_some_and(|operation| {
4873 registration
4874 .manifest
4875 .confirm_operations
4876 .as_ref()
4877 .is_some_and(|operations| operations.contains(operation))
4878 });
4879 let module_id = registration.manifest.module_id;
4880 let nonce = self
4884 .hello_launch_nonces
4885 .lock()
4886 .unwrap_or_else(|p| p.into_inner())
4887 .nonce(ctx.connection_id)
4888 .map(str::to_owned);
4889 let nonce_proven = self
4892 .confirm_channel_proven(ctx.connection_id, &module_id)
4893 .unwrap_or_else(|| {
4894 nonce.as_deref().is_some_and(|nonce| {
4895 self.supervisor
4896 .spawned_consumer_authorized(&module_id, nonce)
4897 })
4898 });
4899 let confirms = self.forwarding.operator_confirms();
4900 self.forwarding
4901 .with_operator_route(
4902 ctx.connection_id,
4903 request.route_channel,
4904 request.route_epoch,
4905 |binding| {
4906 let shortcut = binding
4907 .filter(|binding| {
4908 nonce_proven
4909 && crate::operator_confirm::valid_summary(&request.summary)
4910 && declared
4911 && crate::operator_confirm::ShortcutContext::route_eligible(
4912 &self.operator_app_policy,
4913 &self.launch_table,
4914 binding,
4915 )
4916 })
4917 .map(|binding| {
4918 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
4919 &self.operator_app_policy,
4920 binding,
4921 );
4922 let stamp = if phone.is_some() {
4923 None
4924 } else {
4925 binding.operator_app.clone()
4926 };
4927 self.shortcut_context(
4928 stamp,
4929 phone,
4930 Some((request.route_channel, request.route_epoch)),
4931 )
4932 });
4933 confirms.admit_with_shortcut(
4934 ctx,
4935 frame,
4936 module_id,
4937 nonce_proven,
4938 request,
4939 binding,
4940 shortcut,
4941 )
4942 },
4943 )
4944 .map_err(RouterError::Forwarding)
4945 }
4946
4947 fn shortcut_context(
4948 &self,
4949 stamp: Option<crate::operator_apps::OperatorAppStamp>,
4950 phone_fingerprint: Option<String>,
4951 route: Option<(u16, u32)>,
4952 ) -> crate::operator_confirm::ShortcutContext {
4953 crate::operator_confirm::ShortcutContext {
4954 ledger: self.activity.clone(),
4955 forwarding: self.forwarding.clone(),
4956 launches: self.launch_table.clone(),
4957 stamp,
4958 phone_fingerprint,
4959 route,
4960 connections: self.operator_apps.clone(),
4961 #[cfg(test)]
4962 hooks: Default::default(),
4963 }
4964 }
4965
4966 #[cfg(any(test, feature = "test-support"))]
4968 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
4969 self.operator_app_policy.phone_enabled = enabled;
4970 self
4971 }
4972
4973 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
4976 let mut scopes = ScopeRecords::from(scopes);
4977 for (index, record) in scopes.records.iter_mut().enumerate() {
4978 if let Some(refusal) = Self::canonicalize_workdir(record) {
4979 scopes.refusals.insert(index, refusal);
4980 }
4981 }
4982 scopes
4983 }
4984
4985 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
4986 let workdir = record.workdir.as_mut()?;
4987 let path = std::path::Path::new(workdir);
4988 if !path.is_absolute() {
4989 return Some((
4990 error_codes::INVALID_WORKDIR,
4991 "workdir must be absolute".to_string(),
4992 ));
4993 }
4994 match ProjectRootId::from_path_allowing_missing(path) {
4995 Ok(root) => match root.as_path().to_str() {
4996 Some(canonical) => {
4997 *workdir = canonical.to_string();
4998 None
4999 }
5000 None => Some((
5001 error_codes::INVALID_WORKDIR,
5002 "canonical workdir is not UTF-8".to_string(),
5003 )),
5004 },
5005 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
5006 }
5007 }
5008
5009 fn handle_scope_sync(
5013 &self,
5014 connection_id: ConnectionId,
5015 frame: Frame,
5016 generation: u64,
5017 scopes: Vec<ScopeRecord>,
5018 ) -> Result<Vec<Frame>, RouterError> {
5019 self.handle_scope_change(connection_id, frame, generation, scopes, None)
5020 }
5021
5022 fn handle_scope_change(
5023 &self,
5024 connection_id: ConnectionId,
5025 frame: Frame,
5026 generation: u64,
5027 scopes: Vec<ScopeRecord>,
5028 end: Option<Vec<ScopeEnd>>,
5029 ) -> Result<Vec<Frame>, RouterError> {
5030 let is_apply = end.is_some();
5031 let Some(registration) = self
5032 .registry
5033 .get_module_by_connection(connection_id)
5034 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5035 else {
5036 return Ok(vec![control_error_frame(
5037 &frame,
5038 "not_registered",
5039 "scope.sync and scope.apply require an active module registration owned by this connection",
5040 )?]);
5041 };
5042 let owner = registration.manifest.module_id;
5043 let scopes = Self::prepare_scope_records(scopes);
5044 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
5045 let is_current_launch = |connection: ConnectionId| {
5046 self.connection_is_current_launch(&owner, connection, current_nonce.as_deref())
5047 };
5048 let mut table = self
5053 .scopes
5054 .write()
5055 .unwrap_or_else(|poisoned| poisoned.into_inner());
5056 let outcome = match end {
5057 Some(end) => table.apply(
5058 &owner,
5059 connection_id,
5060 is_current_launch,
5061 generation,
5062 scopes,
5063 end,
5064 ),
5065 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
5066 };
5067 let drained = match &outcome {
5068 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
5069 Err(_) => Vec::new(),
5070 };
5071 let has_deadlines = outcome.is_ok() && table.has_deadlines();
5072 drop(table);
5073 if has_deadlines {
5074 self.scope_deadline_added.notify_one();
5075 }
5076 match outcome {
5077 Ok(applied) => {
5078 let counts = ScopeOutcomeCounts::of(&applied.results);
5079 info!(
5080 owner = %owner,
5081 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5082 generation,
5083 records = applied.results.len(),
5084 created = counts.created,
5085 replaced = counts.replaced,
5086 updated = counts.updated,
5087 unchanged = counts.unchanged,
5088 refused = counts.refused,
5089 ended = applied.ended.len(),
5090 tag_changes = applied.tag_changes.len(),
5091 routes_closed = drained.len(),
5092 "scope change accepted"
5093 );
5094 for refused in applied
5100 .results
5101 .iter()
5102 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
5103 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
5104 {
5105 warn!(
5106 owner = %owner,
5107 generation,
5108 scope_ref = %refused.scope_ref,
5109 scope_epoch = refused.scope_epoch,
5110 code = refused.code.as_deref().unwrap_or(""),
5111 "scope record refused"
5112 );
5113 }
5114 self.close_scope_drained_routes(drained);
5115 let response = if is_apply {
5116 ModuleControlResponseToModule::ScopeApply {
5117 generation,
5118 results: applied.results,
5119 end_results: applied.end_results,
5120 ended: applied.ended,
5121 }
5122 } else {
5123 ModuleControlResponseToModule::ScopeSync {
5124 generation,
5125 results: applied.results,
5126 ended: applied.ended,
5127 }
5128 };
5129 Ok(vec![control_response_body_frame(
5130 &frame,
5131 &response,
5132 "scope change response",
5133 )?])
5134 }
5135 Err(refusal) => {
5136 info!(
5137 owner = %owner,
5138 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5139 generation,
5140 code = refusal.code,
5141 "scope change refused"
5142 );
5143 Ok(vec![control_error_frame(
5144 &frame,
5145 refusal.code,
5146 refusal.message,
5147 )?])
5148 }
5149 }
5150 }
5151
5152 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
5154 scope_expiry::ScopeExpiryContext::new(self).close_scope_drained_routes(drained);
5155 }
5156
5157 fn handle_route_authorize(
5163 &self,
5164 connection_id: ConnectionId,
5165 frame: Frame,
5166 principal: Principal,
5167 scope: subc_protocol::scope::RouteAuthorizeScope,
5168 target: String,
5169 expected: subc_protocol::scope::RouteAuthorizeExpected,
5170 ) -> Result<Vec<Frame>, RouterError> {
5171 use crate::scopes::admission::{admit_route, AdmissionMode};
5172 use subc_protocol::scope::{
5173 RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
5174 };
5175 let registry = self
5176 .registry
5177 .route_admission_snapshot()
5178 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5179 let owner_configured = match &scope.owner {
5180 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5181 _ => false,
5182 };
5183 let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
5184 let answer = {
5185 let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
5186 let permitted = registry
5187 .iter()
5188 .find(|r| r.connection_id == connection_id)
5189 .is_some_and(|r| {
5190 scopes.is_federated_owner(&Principal::Reserved {
5191 module_id: r.manifest.module_id.clone(),
5192 }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
5193 caps.provides
5194 .iter()
5195 .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
5196 })
5197 });
5198 if !permitted {
5199 drop(scopes);
5200 return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
5201 "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
5202 }
5203 admit_route(
5204 &scopes,
5205 ®istry,
5206 &principal,
5207 &target,
5208 Some(&selector),
5209 owner_configured,
5210 AdmissionMode::Authorize(&expected),
5211 )
5212 };
5213 let response = match answer {
5214 Ok(admission) => {
5215 let mut response = RouteAuthorizeResponse::new(
5216 RouteAuthorizeOutcome::Authorized,
5217 "route admission authorized",
5218 );
5219 response.content_version = admission.content_version;
5220 response
5221 }
5222 Err(refusal) => refusal.response(),
5223 };
5224 Ok(vec![control_response_body_frame(
5225 &frame,
5226 &response,
5227 "RouteAuthorizeResponse",
5228 )?])
5229 }
5230
5231 fn handle_scope_describe(
5234 &self,
5235 connection_id: ConnectionId,
5236 frame: Frame,
5237 owner: Principal,
5238 scope_ref: String,
5239 ) -> Result<Vec<Frame>, RouterError> {
5240 let registered = self
5241 .registry
5242 .get_module_by_connection(connection_id)
5243 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5244 let Some(registered) = registered else {
5245 return Ok(vec![control_error_frame(
5246 &frame,
5247 "not_registered",
5248 "scope.describe requires an active module registration owned by this connection",
5249 )?]);
5250 };
5251 let scopes = self
5252 .scopes
5253 .read()
5254 .unwrap_or_else(|poisoned| poisoned.into_inner());
5255 if scopes.is_federated_owner(&owner)
5256 && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
5257 {
5258 return Ok(vec![control_error_frame(
5259 &frame,
5260 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
5261 "scope.describe of the federated owner requires scope-federated/v1",
5262 )?]);
5263 }
5264 let mut description = scopes.describe(&owner, &scope_ref);
5265 drop(scopes);
5266 if owner
5267 != (Principal::Reserved {
5268 module_id: registered.manifest.module_id.clone(),
5269 })
5270 {
5271 if let Some(facts) = description
5272 .stamp
5273 .as_mut()
5274 .and_then(|stamp| stamp.attributes.federated.as_mut())
5275 {
5276 facts.stamp_required = false;
5277 }
5278 }
5279 if !registered
5282 .manifest
5283 .capabilities
5284 .as_ref()
5285 .is_some_and(|caps| {
5286 caps.provides
5287 .iter()
5288 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5289 })
5290 {
5291 if let Some(stamp) = description.stamp.as_mut() {
5292 stamp.attributes.on_behalf_of_agent = None;
5293 }
5294 }
5295 if !registered
5296 .manifest
5297 .capabilities
5298 .as_ref()
5299 .is_some_and(|caps| {
5300 caps.provides
5301 .iter()
5302 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5303 })
5304 {
5305 if let Some(stamp) = description.stamp.as_mut() {
5306 stamp.attributes.flow_version = None;
5307 stamp.attributes.flow_code_hash = None;
5308 stamp.attributes.flow_install_id = None;
5309 }
5310 }
5311 let owner_configured = match &owner {
5312 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5317 _ => false,
5318 };
5319 let response = ModuleControlResponseToModule::ScopeDescribe {
5320 status: description.status,
5321 scope_epoch: description.scope_epoch,
5322 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
5323 owner_synced: description.owner_synced,
5324 owner_configured,
5325 scope: description.stamp,
5326 };
5327 Ok(vec![control_response_body_frame(
5328 &frame,
5329 &response,
5330 "ModuleControlResponseToModule::ScopeDescribe",
5331 )?])
5332 }
5333
5334 fn handle_catalog_update_role_declarations(
5335 &self,
5336 connection_id: ConnectionId,
5337 frame: Frame,
5338 role_declarations: serde_json::Value,
5339 ) -> Result<Vec<Frame>, RouterError> {
5340 if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
5341 return Ok(vec![control_error_frame(
5342 &frame,
5343 "invalid_role_declarations",
5344 err.to_string(),
5345 )?]);
5346 }
5347 let declarations = serde_json::from_value(role_declarations)
5348 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5349 if self
5350 .registry
5351 .replace_role_declarations_for_connection(connection_id, declarations)
5352 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5353 .is_none()
5354 {
5355 return Ok(vec![control_error_frame(
5356 &frame,
5357 "not_registered",
5358 "catalog.update_role_declarations requires an active module registration owned by this connection",
5359 )?]);
5360 }
5361 control_response_body_frame(
5362 &frame,
5363 &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
5364 "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
5365 )
5366 .map(|frame| vec![frame])
5367 }
5368
5369 fn handle_catalog_update(
5370 &self,
5371 connection_id: ConnectionId,
5372 frame: Frame,
5373 provides: Vec<ProviderRole>,
5374 capabilities: Option<CapabilityDeclarations>,
5375 ready: Option<bool>,
5376 ) -> Result<Vec<Frame>, RouterError> {
5377 self.refresh_capability_requirements();
5378 let Some(registration) = self
5379 .registry
5380 .get_module_by_connection(connection_id)
5381 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5382 else {
5383 return Ok(vec![control_error_frame(
5384 &frame,
5385 "not_registered",
5386 "catalog.update requires an active module registration owned by this connection",
5387 )?]);
5388 };
5389
5390 if let Some(message) =
5391 catalog_update_frozen_field_message(®istration.manifest, &provides)
5392 {
5393 return Ok(vec![control_error_frame(
5394 &frame,
5395 "catalog_update_frozen_field",
5396 message,
5397 )?]);
5398 }
5399
5400 let mut candidate = registration.manifest.clone();
5401 candidate.provides = provides.clone();
5402 candidate.capabilities = capabilities
5403 .clone()
5404 .or_else(|| registration.manifest.capabilities.clone());
5405 if let Some(rich_ui) = &candidate.rich_ui {
5406 if let Err(err) = rich_ui.validate() {
5407 return Ok(vec![control_error_frame(
5408 &frame,
5409 "invalid_manifest",
5410 err.to_string(),
5411 )?]);
5412 }
5413 }
5414 if let Err(err) = candidate.validate_capability_grammar() {
5415 return Ok(vec![control_error_frame(
5416 &frame,
5417 "invalid_capability_grammar",
5418 err.to_string(),
5419 )?]);
5420 }
5421
5422 let mut conflicts = self
5425 .capability_evaluator
5426 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
5427 if let Some(conflict) = conflicts.first() {
5428 let message = format!(
5429 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
5430 conflict.capability, conflict.claimants[0], candidate.module_id
5431 );
5432 for conflict in &mut conflicts {
5433 conflict.source = DuplicateClaimSource::CatalogUpdate;
5434 }
5435 log_duplicate_claim_events(conflicts);
5436 return Ok(vec![control_error_frame(
5437 &frame,
5438 "reserved_capability",
5439 message,
5440 )?]);
5441 }
5442
5443 let updated = match self.registry.replace_catalog_for_connection(
5444 connection_id,
5445 provides,
5446 capabilities,
5447 ready,
5448 ) {
5449 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => {
5450 return Ok(vec![control_error_frame(
5451 &frame,
5452 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5453 "scope-federated/v1 is fixed at HELLO; restart the module to change it",
5454 )?])
5455 }
5456 Err(crate::registry::RegistryError::CallerAttestationCapabilityChanged) => {
5457 return Ok(vec![control_error_frame(
5458 &frame,
5459 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5460 "caller-attestation/v1 is fixed at HELLO; restart the module to change it",
5461 )?])
5462 }
5463 result => {
5464 result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5465 }
5466 };
5467 if updated.is_none() {
5468 return Ok(vec![control_error_frame(
5469 &frame,
5470 "not_registered",
5471 "catalog.update requires an active module registration owned by this connection",
5472 )?]);
5473 }
5474 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
5475 log_duplicate_claim_events(
5476 self.capability_evaluator
5477 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
5478 );
5479 }
5480 if capability_census_trigger(
5481 registration.manifest.capabilities.as_ref(),
5482 updated
5483 .as_ref()
5484 .and_then(|entry| entry.manifest.capabilities.as_ref()),
5485 ) {
5486 self.enforce_capability_denies();
5487 }
5488 self.refresh_capability_requirements();
5489
5490 let response = ModuleControlResponseToModule::CatalogUpdate {};
5491 control_response_body_frame(
5492 &frame,
5493 &response,
5494 "ModuleControlResponseToModule::CatalogUpdate",
5495 )
5496 .map(|frame| vec![frame])
5497 }
5498
5499 pub(crate) fn with_activity_ledger_config(
5500 mut self,
5501 run_dir: &Path,
5502 incarnation: String,
5503 config: crate::activity::ActivityConfig,
5504 ) -> Self {
5505 self.activity = Some(crate::activity::ActivityLedger::open_config(
5506 run_dir.join("activity.jsonl"),
5507 incarnation,
5508 config,
5509 ));
5510 self.supervisor.set_activity(self.activity.clone());
5511 self
5512 }
5513
5514 pub(crate) async fn configure_activity(
5515 &self,
5516 config: crate::activity::ActivityConfig,
5517 ) -> Result<(), &'static str> {
5518 match &self.activity {
5519 Some(ledger) => ledger.configure(config).await,
5520 None => Ok(()),
5521 }
5522 }
5523
5524 fn cancel_activity_subscription(
5525 &self,
5526 ctx: &RouteCtx,
5527 frame: &Frame,
5528 ) -> Result<bool, RouterError> {
5529 let task = self
5530 .activity_streams
5531 .lock()
5532 .unwrap_or_else(|p| p.into_inner())
5533 .remove(&(ctx.connection_id, frame.header.corr));
5534 let Some(task) = task else {
5535 return Ok(false);
5536 };
5537 task.abort();
5538 let end = Frame::build_with_version(
5539 response_version(frame),
5540 FrameType::StreamEnd,
5541 control_flags(),
5542 0,
5543 0,
5544 frame.header.corr,
5545 Vec::new(),
5546 )
5547 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
5548 let sink = ctx.egress.clone();
5549 tokio::spawn(async move {
5550 let _ = sink.send(end).await;
5551 });
5552 Ok(true)
5553 }
5554
5555 async fn handle_activity_read(
5556 &self,
5557 ctx: &RouteCtx,
5558 frame: Frame,
5559 ) -> Result<Vec<Frame>, RouterError> {
5560 #[cfg(test)]
5561 {
5562 let pause = self
5563 .activity_request_pause
5564 .lock()
5565 .unwrap()
5566 .clone()
5567 .filter(|pause| pause.connection == ctx.connection_id);
5568 if let Some(pause) = pause {
5569 let _cancelled = ActivityRequestCancellation(pause.clone());
5570 pause.started.notify_one();
5571 std::future::pending::<()>().await;
5572 }
5573 }
5574 use subc_control::{
5575 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
5576 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
5577 };
5578 #[derive(Deserialize)]
5579 struct Identity {
5580 #[serde(default)]
5581 consumer_identity: Option<ConsumerIdentity>,
5582 }
5583 let mut bound_spawn = None;
5587 let legacy = match self.activity_reader(ctx.connection_id) {
5588 channel_operations::ActivityReader::Module { module_id, spawn } => {
5589 bound_spawn = Some(spawn);
5590 Some(Ok(Principal::Reserved { module_id }))
5591 }
5592 channel_operations::ActivityReader::Refused => {
5594 return Ok(vec![control_error_frame(
5595 &frame,
5596 ACTIVITY_READ_FORBIDDEN,
5597 "helper, relay and retired connections cannot read activity",
5598 )?]);
5599 }
5600 channel_operations::ActivityReader::Legacy => None,
5601 };
5602 let principal = match legacy {
5606 Some(principal) => principal,
5607 None => match self
5608 .forwarding
5609 .module_id_for_connection(ctx.connection_id)
5610 .map_err(RouterError::Forwarding)?
5611 {
5612 Some(module_id) => Ok(Principal::Reserved { module_id }),
5613 None => {
5614 let identity: Identity = match serde_json::from_slice(&frame.body) {
5615 Ok(identity) => identity,
5616 Err(_) => {
5617 return Ok(vec![control_error_frame(
5618 &frame,
5619 ACTIVITY_MALFORMED,
5620 "malformed activity read",
5621 )?])
5622 }
5623 };
5624 self.route_open_principal(
5625 ctx.connection_id,
5626 &frame,
5627 identity.consumer_identity,
5628 )?
5629 }
5630 },
5631 };
5632 let admitted = matches!(principal, Ok(Principal::Direct))
5633 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
5634 if !admitted {
5635 return Ok(vec![control_error_frame(
5636 &frame,
5637 ACTIVITY_READ_FORBIDDEN,
5638 "activity reads require an operator or trusted mirror principal",
5639 )?]);
5640 }
5641 let Some(ledger) = &self.activity else {
5642 return Ok(vec![control_error_frame(
5643 &frame,
5644 ACTIVITY_UNAVAILABLE,
5645 "activity ledger unavailable",
5646 )?]);
5647 };
5648 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
5649 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
5650 {
5651 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
5652 Ok(request) => ledger.query(request).await,
5653 Err(_) => Err(ACTIVITY_MALFORMED),
5654 };
5655 return match result {
5656 Ok(reply) => Ok(vec![control_response_body_frame(
5657 &frame,
5658 &reply,
5659 ops::ACTIVITY_QUERY,
5660 )?]),
5661 Err(code) => Ok(vec![control_error_frame(
5662 &frame,
5663 code,
5664 "activity query refused",
5665 )?]),
5666 };
5667 }
5668 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
5669 Ok(request) => request,
5670 Err(_) => {
5671 return Ok(vec![control_error_frame(
5672 &frame,
5673 ACTIVITY_MALFORMED,
5674 "malformed activity subscription",
5675 )?])
5676 }
5677 };
5678 let mut subscription = match ledger.subscribe(request.since).await {
5679 Ok(subscription) => subscription,
5680 Err(code) => {
5681 return Ok(vec![control_error_frame(
5682 &frame,
5683 code,
5684 "activity subscription refused",
5685 )?])
5686 }
5687 };
5688 let key = (ctx.connection_id, frame.header.corr);
5689 let sink = ctx.egress.clone();
5690 let mut streams = self
5691 .activity_streams
5692 .lock()
5693 .unwrap_or_else(|p| p.into_inner());
5694 streams.retain(|_, task| !task.is_finished());
5695 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
5696 return Ok(vec![control_error_frame(
5697 &frame,
5698 "duplicate_subscription",
5699 "activity subscription correlation id already in use",
5700 )?]);
5701 }
5702 let stream = async move {
5703 loop {
5704 let entry = if let Some(entry) = subscription.replay.pop_front() {
5705 Some(entry)
5706 } else {
5707 subscription.receiver.recv().await
5708 };
5709 let Some(entry) = entry else {
5710 break;
5711 };
5712 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
5713 return;
5714 };
5715 let Ok(next) = Frame::build_with_version(
5716 response_version(&frame),
5717 FrameType::StreamData,
5718 control_flags(),
5719 0,
5720 0,
5721 frame.header.corr,
5722 body,
5723 ) else {
5724 return;
5725 };
5726 if sink.send(next).await.is_err() {
5727 return;
5728 }
5729 }
5730 if let Some(code) = subscription.terminal_code() {
5733 if let Ok(end) =
5734 control_error_frame(&frame, code, "activity subscriber fell behind")
5735 {
5736 let _ = sink.send(end).await;
5737 }
5738 }
5739 };
5740 let task = tokio::spawn(async move {
5744 match bound_spawn {
5745 Some(spawn) => {
5746 tokio::select! {
5747 () = stream => {}
5748 () = spawn.wait_closed() => {}
5749 }
5750 }
5751 None => stream.await,
5752 }
5753 });
5754 streams.insert(key, task.abort_handle());
5755 Ok(Vec::new())
5756 }
5757
5758 async fn handle_activity_record(
5759 &self,
5760 ctx: &RouteCtx,
5761 frame: Frame,
5762 ) -> Result<Vec<Frame>, RouterError> {
5763 use subc_control::{
5764 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
5765 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
5766 };
5767 let result = async {
5768 let module = self
5769 .forwarding
5770 .module_id_for_connection(ctx.connection_id)
5771 .ok()
5772 .flatten()
5773 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
5774 let request: ActivityRecordRequest =
5775 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
5776 crate::activity::validate(&module, &request)?;
5777 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
5778 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
5779 let at_ms = std::time::SystemTime::now()
5780 .duration_since(std::time::UNIX_EPOCH)
5781 .unwrap_or_default()
5782 .as_millis() as u64;
5783 ledger.record(module, request, actor, at_ms).await
5784 }
5785 .await;
5786 match result {
5787 Ok(entry) => Ok(vec![control_response_body_frame(
5788 &frame,
5789 &ActivityRecordResponse { seq: entry.seq },
5790 "activity.record",
5791 )?]),
5792 Err(code) => Ok(vec![control_error_frame(
5793 &frame,
5794 code,
5795 "activity recording refused",
5796 )?]),
5797 }
5798 }
5799
5800 fn activity_actor(
5801 &self,
5802 connection: ConnectionId,
5803 module: &str,
5804 on: Option<&subc_control::ActivityOn>,
5805 ) -> Result<subc_control::ActivityActor, &'static str> {
5806 use crate::forwarding::{DataRoute, DataRouteState};
5807 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
5808 let principal = |id: &str| Principal::Reserved {
5809 module_id: id.to_owned(),
5810 };
5811 let mut actor = ActivityActor {
5812 principal: principal(module),
5813 scope_owner: None,
5814 agent_id: None,
5815 run_id: None,
5816 flow_id: None,
5817 project_root: None,
5818 workdir: None,
5819 project_id: None,
5820 };
5821 let scope = match on {
5822 None => return Ok(actor),
5823 Some(ActivityOn::Scope(scope)) => {
5824 let selector = subc_protocol::scope::ScopeSelector::new(
5825 principal(module),
5826 scope.scope_ref.clone(),
5827 Some(scope.epoch),
5828 );
5829 self.scopes
5830 .read()
5831 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5832 .attribute(&principal(module), module, &selector, true)
5833 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5834 }
5835 Some(ActivityOn::Route(route)) => {
5836 let DataRoute::Module(DataRouteState::Bound(binding)) = self
5837 .forwarding
5838 .lookup_data_route(connection, route.channel, route.epoch)
5839 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5840 else {
5841 return Err(ACTIVITY_ACTOR_UNKNOWN);
5842 };
5843 if binding.module_id != module {
5844 return Err(ACTIVITY_ACTOR_UNKNOWN);
5845 }
5846 actor.principal = binding.principal.clone();
5847 actor.project_root = binding
5848 .project_root
5849 .as_ref()
5850 .map(|root| root.as_path().to_path_buf());
5851 let Some(bound) = &binding.scope else {
5852 return Ok(actor);
5853 };
5854 let owner = principal(&bound.owner);
5855 let selector = subc_protocol::scope::ScopeSelector::new(
5856 owner.clone(),
5857 bound.scope_ref.clone(),
5858 Some(bound.tag.scope_epoch),
5859 );
5860 let admitted = self
5861 .scopes
5862 .read()
5863 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5864 .attribute(&owner, module, &selector, true)
5865 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
5866 if admitted.tag != bound.tag {
5867 return Err(ACTIVITY_ACTOR_UNKNOWN);
5868 }
5869 admitted
5870 }
5871 };
5872 actor.scope_owner = Some(scope.stamp.owner);
5873 actor.agent_id = scope.stamp.attributes.agent_id;
5874 actor.run_id = scope.stamp.attributes.run_id;
5875 actor.flow_id = scope.stamp.attributes.flow_id;
5876 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
5877 actor.project_id = scope.stamp.project_id;
5878 Ok(actor)
5879 }
5880
5881 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5882 self.refresh_capability_requirements();
5883 let mut counters = self.counters.snapshot();
5889 let losses = self.resource_tracker.loss_diagnostics();
5890 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
5891 if let Some(obj) = counters.as_object_mut() {
5892 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
5893 }
5894 }
5895 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
5896 obj.insert(
5897 "activity_unparseable_lines".into(),
5898 ledger.read_errors().into(),
5899 );
5900 obj.insert(
5901 "activity_lifecycle_append_failures".into(),
5902 ledger.lifecycle_failures().into(),
5903 );
5904 obj.insert(
5905 "activity_started_digest_skips".into(),
5906 ledger.lifecycle_digest_skips().into(),
5907 );
5908 }
5909 if let (Ok((connections_with_routes, max)), Some(obj)) = (
5910 self.forwarding.client_route_concentration(),
5911 counters.as_object_mut(),
5912 ) {
5913 obj.insert(
5914 "client_connections_with_routes".into(),
5915 connections_with_routes.into(),
5916 );
5917 obj.insert("max_routes_on_one_connection".into(), max.into());
5918 }
5919 if let (Some(open_breakers), Some(obj)) = (
5925 self.route_bind_breakers.open_snapshot(),
5926 counters.as_object_mut(),
5927 ) {
5928 obj.insert("route_bind_breakers_open".into(), open_breakers);
5929 }
5930 if let (Some(reason), Some(obj)) = (
5935 self.local_endpoint_unavailable.clone(),
5936 counters.as_object_mut(),
5937 ) {
5938 obj.insert("local_endpoint_unavailable".into(), reason.into());
5939 }
5940 let response = ClientControlResponse::ServerDescribe {
5941 protocol_ver: PROTOCOL_VERSION,
5942 subc_ops: subc_ops(),
5943 capabilities: self.describe_capabilities(),
5944 connected_clients: self.connected_clients.count(),
5945 counters: Some(counters),
5946 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
5947 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
5948 capability_requirements: self.capability_requirement_statuses(),
5949 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
5950 config_path: self
5951 .rescan
5952 .as_ref()
5953 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
5954 };
5955 Ok(vec![control_response_body_frame(
5956 &frame,
5957 &response,
5958 "ClientControlResponse::ServerDescribe",
5959 )?])
5960 }
5961
5962 fn handle_catalog_list(
5963 &self,
5964 frame: Frame,
5965 module_id: Option<String>,
5966 ) -> Result<Vec<Frame>, RouterError> {
5967 let (generation, modules) = self
5968 .registry
5969 .list_modules_with_nonce_verification()
5970 .map_err(|err| {
5971 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
5972 })?;
5973 let entries = modules
5974 .into_iter()
5975 .filter(|(registration, _)| {
5976 module_id
5977 .as_deref()
5978 .map(|wanted| registration.manifest.module_id == wanted)
5979 .unwrap_or(true)
5980 })
5981 .map(|(registration, nonce_verified)| {
5982 let entry = self
5983 .catalog_discovery_entry(®istration, module_id.is_some())
5984 .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
5985 let not_ready = self.not_ready_reason(®istration);
5986 let mut entry = entry;
5987 entry.roles = registration.manifest.provides;
5988 entry.control_ops = registration.control_ops;
5989 Ok(entry
5990 .with_ready(not_ready.is_none())
5991 .with_display_name(registration.manifest.display_name)
5992 .with_not_ready(not_ready)
5993 .with_module_version(Some(registration.manifest.module_version))
5994 .with_provenance(registration.manifest.provenance)
5995 .with_capabilities(registration.manifest.capabilities)
5996 .with_self_signals(registration.manifest.self_signals)
5997 .with_events(registration.manifest.events)
5998 .with_operation_delivery(registration.manifest.operation_delivery)
5999 .with_operation_payload(registration.manifest.operation_payload)
6000 .with_flow_operations(registration.manifest.flow_operations)
6001 .with_operation_deferral(registration.manifest.operation_deferral)
6002 .with_confirm_operations(registration.manifest.confirm_operations)
6003 .with_rich_ui(registration.manifest.rich_ui)
6004 .with_role_declarations(registration.manifest.role_declarations)
6005 .with_nonce_verified(nonce_verified))
6006 })
6007 .collect::<Result<Vec<_>, RouterError>>()?;
6008 let response = ClientControlResponse::CatalogList {
6009 generation,
6010 modules: entries,
6011 subc_ops: subc_ops(),
6012 };
6013 Ok(vec![control_response_body_frame(
6014 &frame,
6015 &response,
6016 "ClientControlResponse::CatalogList",
6017 )?])
6018 }
6019
6020 fn route_open_caller_from_frame(&self, ctx: &RouteCtx, frame: &Frame) -> RouteOpenCaller {
6024 let Ok(ClientControlRequest::RouteOpen {
6025 identity,
6026 consumer_identity,
6027 scope,
6028 ..
6029 }) = parse_client_control_request(&frame.body)
6030 else {
6031 return RouteOpenCaller::unknown();
6032 };
6033 let principal = self
6034 .attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref())
6035 .map(|attested| attested.principal);
6036 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
6037 }
6038
6039 fn route_open_principal(
6040 &self,
6041 connection_id: ConnectionId,
6042 frame: &Frame,
6043 consumer_identity: Option<ConsumerIdentity>,
6044 ) -> Result<Result<Principal, Frame>, RouterError> {
6045 let attested = self
6046 .attest_route_open_consumer(connection_id, consumer_identity.as_ref())
6047 .map(|attested| attested.principal);
6048 self.route_open_principal_attested(frame, consumer_identity, attested)
6049 }
6050
6051 fn route_open_principal_attested(
6055 &self,
6056 frame: &Frame,
6057 consumer_identity: Option<ConsumerIdentity>,
6058 attested: Option<Principal>,
6059 ) -> Result<Result<Principal, Frame>, RouterError> {
6060 if let Some(principal) = attested {
6061 return Ok(Ok(principal));
6062 }
6063 Ok(Err(control_error_frame(
6064 frame,
6065 "bad_consumer_identity",
6066 consumer_identity.map_or_else(
6067 || "connection claim is no longer authorized for routes".to_string(),
6068 |identity| format!(
6069 "consumer_identity for module_id '{}' conflicts with its connection claim or did not match a supervised launch nonce",
6070 identity.module_id
6071 ),
6072 ),
6073 )?))
6074 }
6075
6076 #[allow(clippy::too_many_arguments)]
6083 fn route_open_refusal_frame(
6084 &self,
6085 ctx: &RouteCtx,
6086 caller: &RouteOpenCaller,
6087 frame: &Frame,
6088 module_id: &str,
6089 reason: &'static str,
6090 code: &'static str,
6091 message: impl Into<String>,
6092 ) -> Result<Frame, RouterError> {
6093 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
6094 let detail = if matches!(
6095 reason,
6096 "registration_not_active"
6097 | "supervisor_not_live"
6098 | "no_forwarding_connection"
6099 | "reloading"
6100 ) {
6101 let expected_back = (reason == "reloading").then_some(true);
6104 self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
6105 } else {
6106 None
6107 };
6108 control_error_body_frame(
6109 frame,
6110 ErrorBody {
6111 code: code.to_string(),
6112 message: message.into(),
6113 detail,
6114 },
6115 )
6116 }
6117
6118 fn route_open_recovery_detail(
6119 &self,
6120 module_id: &str,
6121 corr: u64,
6122 detail: Option<serde_json::Value>,
6123 expected_back: Option<bool>,
6124 ) -> Result<Option<serde_json::Value>, RouterError> {
6125 let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
6126 return Ok(detail);
6127 };
6128 let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
6129 detail["state"] = serde_json::json!(status.state.to_string());
6130 if let Some(expected_back) = expected_back {
6131 detail["expected_back"] = serde_json::json!(expected_back);
6132 } else if let Some(module) = self.supervisor.get(module_id) {
6133 match module.will_recover_after_connection_loss() {
6134 Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
6135 Err(err) => warn!(
6136 %module_id,
6137 error = %err,
6138 "failed to read route.open recovery verdict; omitting expected_back"
6139 ),
6140 }
6141 }
6142 Ok(Some(detail))
6143 }
6144
6145 #[allow(clippy::too_many_arguments)]
6154 fn route_bind_rejection(
6155 &self,
6156 ctx: &RouteCtx,
6157 caller: &RouteOpenCaller,
6158 frame: &Frame,
6159 target_module_id: &str,
6160 body: ErrorBody,
6161 daemon_authored: bool,
6162 ) -> Result<Vec<Frame>, RouterError> {
6163 if daemon_authored {
6169 let scope_code = match body.code.as_str() {
6170 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
6171 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
6172 _ => None,
6173 };
6174 if let Some(code) = scope_code {
6175 self.observe_route_open_refusal(
6176 ctx,
6177 caller,
6178 target_module_id,
6179 "scope_changed_before_commit",
6180 code,
6181 );
6182 return Ok(vec![control_error_body_frame(frame, body)?]);
6183 }
6184 }
6185 self.counters
6186 .increment_route_open_refused("module_rejected");
6187 info!(
6188 target: "control",
6189 code = "module_rejected",
6190 module_code = ?body.code,
6191 module_id = ?target_module_id,
6192 connection_id = ctx.connection_id.get(),
6193 principal = %caller.principal,
6194 scope = %caller.scope,
6195 harness = %caller.harness,
6196 "route.open refused"
6197 );
6198 let refusal = if daemon_authored {
6199 control_error_body_frame(frame, body)?
6200 } else {
6201 control_relayed_error_body_frame(frame, body)?
6202 };
6203 Ok(vec![refusal])
6204 }
6205
6206 #[allow(clippy::too_many_arguments)]
6224 fn route_open_breaker_refusal_frame(
6225 &self,
6226 ctx: &RouteCtx,
6227 caller: &RouteOpenCaller,
6228 frame: &Frame,
6229 module_id: &str,
6230 consecutive_timeouts: u32,
6231 retry_in: Duration,
6232 probe_in_flight: bool,
6233 ) -> Result<Frame, RouterError> {
6234 self.counters
6235 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
6236 debug!(
6237 target: "control",
6238 code = "module_timeout",
6239 module_id = ?module_id,
6240 connection_id = ctx.connection_id.get(),
6241 principal = %caller.principal,
6242 scope = %caller.scope,
6243 harness = %caller.harness,
6244 consecutive_timeouts,
6245 retry_in_ms = retry_in.as_millis() as u64,
6246 probe_in_flight,
6247 "route.open refused by open bind-relay breaker"
6248 );
6249 let detail = if probe_in_flight {
6256 "one test route is already being tried; retry once it settles".to_string()
6257 } else {
6258 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
6259 };
6260 control_error_frame(
6261 frame,
6262 "module_timeout",
6263 format!(
6264 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
6265 timed out in a row); {detail}; its established routes are unaffected"
6266 ),
6267 )
6268 }
6269
6270 fn observe_route_open_refusal(
6283 &self,
6284 ctx: &RouteCtx,
6285 caller: &RouteOpenCaller,
6286 module_id: &str,
6287 reason: &'static str,
6288 code: &'static str,
6289 ) {
6290 self.counters.increment_route_open_refused(code);
6291 info!(
6292 target: "control",
6293 code,
6294 reason,
6295 module_id = ?module_id,
6296 connection_id = ctx.connection_id.get(),
6297 principal = %caller.principal,
6298 scope = %caller.scope,
6299 harness = %caller.harness,
6300 "route.open refused"
6301 );
6302 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
6303 self.route_outages.record_not_serving(module_id, reason);
6304 }
6305 }
6306
6307 fn observe_route_open_accept(
6352 &self,
6353 ctx: &RouteCtx,
6354 module_id: &str,
6355 principal: &str,
6356 scope: &str,
6357 ) {
6358 self.route_outages.record_accepted(module_id);
6359 self.counters.increment_route_open_accepted(principal);
6360 info!(
6361 target: "control",
6362 principal,
6363 module_id,
6364 connection_id = ctx.connection_id.get(),
6365 scope = %scope,
6366 "route.open accepted"
6367 );
6368 }
6369
6370 fn supervised_absent_route_open_refusal_frame(
6371 &self,
6372 ctx: &RouteCtx,
6373 caller: &RouteOpenCaller,
6374 frame: &Frame,
6375 module_id: &str,
6376 code: &'static str,
6377 status: &crate::supervise::ModuleStatus,
6378 ) -> Result<Frame, RouterError> {
6379 self.counters.increment_route_open_refused(code);
6380 info!(
6381 target: "control",
6382 code,
6383 reason = "supervised_not_registered",
6384 module_id = ?module_id,
6385 connection_id = ctx.connection_id.get(),
6386 principal = %caller.principal,
6387 scope = %caller.scope,
6388 harness = %caller.harness,
6389 state = %status.state,
6390 enabled = status.enabled,
6391 live = status.live,
6392 "route.open refused"
6393 );
6394 self.route_outages
6398 .record_not_serving(module_id, "supervised_not_registered");
6399 let message = if let Some(pid) = status.waiting_for_previous_pid {
6400 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
6401 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
6402 })
6403 } else {
6404 format!(
6405 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
6406 status.state, status.enabled, status.live
6407 )
6408 };
6409 let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
6413 control_error_body_frame(
6414 frame,
6415 ErrorBody {
6416 code: code.to_string(),
6417 message,
6418 detail: self.route_open_recovery_detail(
6419 module_id,
6420 frame.header.corr,
6421 None,
6422 expected_back,
6423 )?,
6424 },
6425 )
6426 }
6427
6428 async fn handle_route_open(
6429 &self,
6430 ctx: &RouteCtx,
6431 frame: Frame,
6432 request: RouteOpenRequest,
6433 timings: &mut RouteOpenTimings,
6434 ) -> Result<Vec<Frame>, RouterError> {
6435 let RouteOpenRequest {
6436 target,
6437 mut identity,
6438 consumer_identity,
6439 consumer_capabilities,
6440 role_versions,
6441 busy_refusals,
6442 admission_facts,
6443 scope,
6444 accepts_close_reasons,
6445 } = request;
6446 let target_module_id = target_module_id(&target).to_string();
6447 timings.module_id.clone_from(&target_module_id);
6448 if self
6449 .registry
6450 .get_module_by_connection(ctx.connection_id)
6451 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6452 .is_some()
6453 {
6454 return Ok(vec![control_error_frame(
6455 &frame,
6456 "invalid_request",
6457 "module connections cannot open client routes",
6458 )?]);
6459 }
6460 debug!(
6461 connection_id = ctx.connection_id.get(),
6462 corr = frame.header.corr,
6463 module_id = %target_module_id,
6464 "handling route.open"
6465 );
6466
6467 let attested =
6473 self.attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref());
6474 let mut caller = RouteOpenCaller::new(
6475 attested.as_ref().map(|attested| &attested.principal),
6476 scope.as_ref(),
6477 &identity,
6478 );
6479
6480 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
6486 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
6487 self.observe_route_open_refusal(
6488 ctx,
6489 &caller,
6490 &target_module_id,
6491 "invalid_role_versions",
6492 error_codes::INVALID_REQUEST,
6493 );
6494 return Ok(vec![control_error_body_frame(
6495 &frame,
6496 ErrorBody {
6497 code: error_codes::INVALID_REQUEST.to_string(),
6498 message: error.to_string(),
6499 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
6500 },
6501 )?]);
6502 }
6503
6504 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
6505 if status.waiting_for_previous_pid.is_some() {
6506 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6507 ctx,
6508 &caller,
6509 &frame,
6510 &target_module_id,
6511 "module_warming",
6512 &status,
6513 )?]);
6514 }
6515 }
6516
6517 let admission_registry = self
6536 .registry
6537 .route_admission_snapshot()
6538 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6539 let Some(registration) =
6540 crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
6541 else {
6542 if let Some((status, warming)) =
6543 self.supervisor_status(&target_module_id, frame.header.corr)?
6544 {
6545 if status.protocol == ModuleProtocol::None {
6552 return Ok(vec![self.route_open_refusal_frame(
6553 ctx,
6554 &caller,
6555 &frame,
6556 &target_module_id,
6557 "protocol_none",
6558 error_codes::MODULE_NO_PROTOCOL,
6559 format!(
6560 "module_id '{target_module_id}' is declared protocol: none; \
6561 it speaks no subc wire and serves no routes"
6562 ),
6563 )?]);
6564 }
6565 let code = if warming {
6566 "module_warming"
6567 } else {
6568 "target_unavailable"
6569 };
6570 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6571 ctx,
6572 &caller,
6573 &frame,
6574 &target_module_id,
6575 code,
6576 &status,
6577 )?]);
6578 }
6579 if let Some(removed_ago_ms) =
6580 self.supervisor.removal_tombstone_age_ms(&target_module_id)
6581 {
6582 return Ok(vec![self.route_open_refusal_frame(
6583 ctx,
6584 &caller,
6585 &frame,
6586 &target_module_id,
6587 "removed",
6588 error_codes::MODULE_REMOVED,
6589 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
6590 )?]);
6591 }
6592 return Ok(vec![self.route_open_refusal_frame(
6593 ctx,
6594 &caller,
6595 &frame,
6596 &target_module_id,
6597 "not_registered",
6598 error_codes::UNKNOWN_MODULE,
6599 format!("module_id '{target_module_id}' is not registered"),
6600 )?]);
6601 };
6602
6603 if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
6607 .is_err()
6608 {
6609 self.counters
6610 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
6611 info!(
6612 target: "control",
6613 code = error_codes::MODULE_WARMING,
6614 module_id = ?target_module_id,
6615 connection_id = ctx.connection_id.get(),
6616 principal = %caller.principal,
6617 scope = %caller.scope,
6618 harness = %caller.harness,
6619 reason = "declared_not_ready",
6620 "route.open refused"
6621 );
6622 self.route_outages
6625 .record_not_serving(&target_module_id, "declared_not_ready");
6626 return Ok(vec![control_error_body_frame(
6629 &frame,
6630 ErrorBody {
6631 code: error_codes::MODULE_WARMING.to_string(),
6632 message: format!(
6633 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
6634 ),
6635 detail: self.route_open_recovery_detail(
6636 &target_module_id,
6637 frame.header.corr,
6638 Some(serde_json::json!({ "reason": "declared_not_ready" })),
6639 Some(true),
6640 )?,
6641 },
6642 )?]);
6643 }
6644
6645 if let Some(capability) = self.first_unprovided_required_capability(registration) {
6664 self.counters
6665 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
6666 info!(
6667 target: "control",
6668 code = error_codes::MODULE_WARMING,
6669 module_id = ?target_module_id,
6670 connection_id = ctx.connection_id.get(),
6671 principal = %caller.principal,
6672 scope = %caller.scope,
6673 harness = %caller.harness,
6674 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6675 capability = %capability,
6676 "route.open refused"
6677 );
6678 return Ok(vec![control_error_body_frame(
6679 &frame,
6680 ErrorBody {
6681 code: error_codes::MODULE_WARMING.to_string(),
6682 message: format!(
6683 "module_id '{target_module_id}' requires capability '{capability}', \
6684 which no registered module provides; retry"
6685 ),
6686 detail: Some(serde_json::json!({
6687 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6688 "capability": capability,
6689 })),
6690 },
6691 )?]);
6692 }
6693
6694 if !target_has_required_role(&target, ®istration.manifest.provides) {
6695 return Ok(vec![self.route_open_refusal_frame(
6696 ctx,
6697 &caller,
6698 &frame,
6699 &target_module_id,
6700 "role_not_provided",
6701 "target_unavailable",
6702 format!("module_id '{target_module_id}' does not provide the requested target"),
6703 )?]);
6704 }
6705
6706 if registration.state != ChannelState::Active {
6707 return Ok(vec![self.route_open_refusal_frame(
6708 ctx,
6709 &caller,
6710 &frame,
6711 &target_module_id,
6712 "registration_not_active",
6713 "target_unavailable",
6714 format!("module_id '{target_module_id}' is not active"),
6715 )?]);
6716 }
6717
6718 if self
6719 .forwarding
6720 .module_is_draining(&target_module_id)
6721 .map_err(RouterError::Forwarding)?
6722 {
6723 return Ok(vec![self.route_open_refusal_frame(
6724 ctx,
6725 &caller,
6726 &frame,
6727 &target_module_id,
6728 "reloading",
6729 "module_reloading",
6730 format!("module_id '{target_module_id}' is reloading"),
6731 )?]);
6732 }
6733
6734 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
6735 process_liveness.process_live(&target_module_id) == Some(false)
6736 }) {
6737 if process_liveness.process_replacing(&target_module_id) {
6744 return Ok(vec![self.route_open_refusal_frame(
6745 ctx,
6746 &caller,
6747 &frame,
6748 &target_module_id,
6749 "reloading",
6750 "module_reloading",
6751 format!("module_id '{target_module_id}' is reloading"),
6752 )?]);
6753 }
6754 return Ok(vec![self.route_open_refusal_frame(
6755 ctx,
6756 &caller,
6757 &frame,
6758 &target_module_id,
6759 "supervisor_not_live",
6760 "target_unavailable",
6761 format!("module_id '{target_module_id}' is not live"),
6762 )?]);
6763 }
6764
6765 if !self
6766 .forwarding
6767 .has_live_module_connection(&target_module_id)
6768 .map_err(RouterError::Forwarding)?
6769 {
6770 return Ok(vec![self.route_open_refusal_frame(
6771 ctx,
6772 &caller,
6773 &frame,
6774 &target_module_id,
6775 "no_forwarding_connection",
6776 "target_unavailable",
6777 format!("module_id '{target_module_id}' has no live forwarding connection"),
6778 )?]);
6779 }
6780
6781 if let Some(error) =
6782 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
6783 {
6784 self.observe_route_open_refusal(
6785 ctx,
6786 &caller,
6787 &target_module_id,
6788 "op_not_allowed",
6789 "op_not_allowed",
6790 );
6791 return Ok(vec![error]);
6792 }
6793
6794 let principal = match (&attested, consumer_identity.is_some()) {
6795 (None, false) => {
6798 self.observe_route_open_refusal(
6799 ctx,
6800 &caller,
6801 &target_module_id,
6802 "registration_channel_closed",
6803 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6804 );
6805 return Ok(vec![control_error_frame(
6806 &frame,
6807 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6808 "the claim binding's launch has ended",
6809 )?]);
6810 }
6811 _ => match self.route_open_principal_attested(
6812 &frame,
6813 consumer_identity,
6814 attested.as_ref().map(|attested| attested.principal.clone()),
6815 )? {
6816 Ok(principal) => principal,
6817 Err(error) => {
6818 self.observe_route_open_refusal(
6819 ctx,
6820 &caller,
6821 &target_module_id,
6822 "bad_consumer_identity",
6823 "bad_consumer_identity",
6824 );
6825 return Ok(vec![error]);
6826 }
6827 },
6828 };
6829 let attested = attested.expect("an attested principal was resolved above");
6830 timings.principal = principal_label(&principal);
6831
6832 let Ok(mut caller_capture) = attested.capture() else {
6835 self.observe_route_open_refusal(
6836 ctx,
6837 &caller,
6838 &target_module_id,
6839 "registration_channel_closed",
6840 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6841 );
6842 return Ok(vec![control_error_frame(
6843 &frame,
6844 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6845 "the opener's launch has ended",
6846 )?]);
6847 };
6848 let caller_module_id = match &attested.spawn {
6852 Some((record, _)) => Some(record.module_id().to_owned()),
6853 None => match &principal {
6854 Principal::Reserved { module_id } => Some(module_id.clone()),
6855 _ => None,
6856 },
6857 };
6858
6859 let build_class = identity.build_class.take().filter(|build_class| {
6862 if caller_module_id.as_deref() == Some("callosum") {
6863 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
6864 } else {
6865 if self
6866 .build_class_warned
6867 .lock()
6868 .unwrap_or_else(|p| p.into_inner())
6869 .insert(ctx.connection_id)
6870 {
6871 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
6872 "ignoring build_class from a non-Callosum opener");
6873 }
6874 false
6875 }
6876 });
6877
6878 if let Some(opening_registration) = self.caller_registration(&attested, &admission_registry)
6882 {
6883 let opening_module_id = opening_registration.manifest.module_id.as_str();
6884 if let Some(capability) =
6885 denied_capability(&opening_registration.manifest, ®istration.manifest)
6886 {
6887 warn!(
6888 opening_module_id,
6889 target_module_id,
6890 capability,
6891 "refusing route.open because an attested capability deny edge matches"
6892 );
6893 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6894 &frame,
6895 &target_module_id,
6896 "capability_deny_edge",
6897 "capability_forbidden",
6898 format!(
6899 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
6900 ),
6901 )?]);
6902 }
6903 }
6904
6905 if admission_facts.is_some() {
6906 let carrier_matches = caller_module_id.is_some()
6907 && self.admission_facts_carrier_module_id.as_deref() == caller_module_id.as_deref();
6908 if !carrier_matches {
6909 return Ok(vec![self.route_open_refusal_frame(
6910 ctx,
6911 &caller,
6912 &frame,
6913 &target_module_id,
6914 "admission_facts_carrier_not_permitted",
6915 "admission_facts_not_permitted",
6916 "admission facts may only be carried by the configured reserved module",
6917 )?]);
6918 }
6919
6920 let target_allowed = self
6921 .admission_facts_targets
6922 .as_ref()
6923 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
6924 if !target_allowed {
6925 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6926 &frame,
6927 &target_module_id,
6928 "admission_facts_target_not_listed",
6929 "admission_facts_target_not_allowed",
6930 format!(
6931 "admission facts are not permitted for target module_id '{target_module_id}'"
6932 ),
6933 )?]);
6934 }
6935
6936 }
6940
6941 let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
6947 None => (None, None, false),
6948 Some(selector) => {
6949 let owner_configured = match &selector.owner {
6950 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
6956 _ => false,
6957 };
6958 let mut scopes = self
6959 .scopes
6960 .write()
6961 .unwrap_or_else(|poisoned| poisoned.into_inner());
6962 let admitted = crate::scopes::admission::admit_route(
6963 &scopes,
6964 &admission_registry,
6965 &principal,
6966 &target_module_id,
6967 Some(selector),
6968 owner_configured,
6969 crate::scopes::admission::AdmissionMode::Open,
6970 )
6971 .map(|route| {
6972 let admission = route.scope.expect("scoped admission");
6973 let bound = scopes.reserve_bind(
6974 &admission,
6975 &principal,
6976 ctx.connection_id,
6977 selector.hold_presence,
6978 );
6979 (admission, bound)
6980 });
6981 match admitted {
6982 Ok((admission, bound)) => {
6983 (Some(bound), Some(admission.stamp), admission.stamp_required)
6984 }
6985 Err(refusal) => {
6986 return Ok(vec![self.route_open_refusal_frame(
6987 ctx,
6988 &caller,
6989 &frame,
6990 &target_module_id,
6991 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
6992 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
6993 refusal.message,
6994 )?]);
6995 }
6996 }
6997 }
6998 };
6999 if let Some(stamp) = &scope_stamp {
7002 caller.scope = route_open_scope_label(stamp);
7003 }
7004 if scope_stamp.as_ref().is_some_and(scope_acts_for_another) {
7007 caller_capture = caller_capture.map(crate::spawn_authority::CallerCapture::relayed);
7008 }
7009
7010 timings.enter(RouteOpenPhase::ProjectRoot);
7018 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
7019 Ok(project) => project,
7020 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
7021 };
7022 timings.enter(RouteOpenPhase::Admission);
7023 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
7024 Ok(root) => root,
7025 Err(err) => {
7026 return Ok(vec![control_error_frame(
7027 &frame,
7028 "invalid_project_root",
7029 err.to_string(),
7030 )?])
7031 }
7032 };
7033 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
7039
7040 let _concurrency_guard = match self
7051 .route_bind_concurrency
7052 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
7053 {
7054 Ok(guard) => guard,
7055 Err(in_flight) => {
7056 return Ok(vec![self.route_open_target_capacity_refusal(
7057 ctx,
7058 &caller,
7059 &frame,
7060 &target_module_id,
7061 in_flight,
7062 )?]);
7063 }
7064 };
7065
7066 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
7069 RouteBindAdmission::Admitted { guard, probe } => {
7070 if probe {
7071 info!(
7072 module_id = %target_module_id,
7073 connection_id = ctx.connection_id.get(),
7074 "route.bind breaker half-open: admitting one probe"
7075 );
7076 }
7077 guard
7078 }
7079 RouteBindAdmission::Refused {
7080 consecutive_timeouts,
7081 retry_in,
7082 probe_in_flight,
7083 } => {
7084 return Ok(vec![self.route_open_breaker_refusal_frame(
7085 ctx,
7086 &caller,
7087 &frame,
7088 &target_module_id,
7089 consecutive_timeouts,
7090 retry_in,
7091 probe_in_flight,
7092 )?]);
7093 }
7094 };
7095
7096 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
7101 let relay_deadline = Instant::now() + route_bind_relay_timeout;
7102 let operator_app = self
7103 .operator_apps
7104 .lock()
7105 .unwrap_or_else(|p| p.into_inner())
7106 .get(&ctx.connection_id)
7107 .cloned();
7108 timings.enter(RouteOpenPhase::EgressReservation);
7109 let pending = match self
7110 .forwarding
7111 .begin_route_bind_relay_for(
7112 ctx.connection_id,
7113 ctx.egress.clone(),
7114 response_version(&frame),
7115 frame.header.corr,
7116 &target_module_id,
7117 principal.clone(),
7118 bound_scope,
7119 project_root,
7120 identity.harness.clone(),
7121 build_class,
7122 operator_app.clone(),
7123 crate::forwarding::RouteAuthority {
7124 incarnation: self.supervisor.daemon_incarnation(),
7125 caller: caller_capture
7126 .as_ref()
7127 .map(|caller| caller.authority.pair()),
7128 },
7129 relay_deadline,
7130 busy_refusals == Some(true),
7131 accepts_close_reasons,
7132 )
7133 .await
7134 {
7135 Ok(pending) => pending,
7136 Err(err) => {
7137 return Ok(vec![self.route_open_refusal_frame(
7138 ctx,
7139 &caller,
7140 &frame,
7141 &target_module_id,
7142 "relay_reservation_failed",
7143 forwarding_error_code(&err),
7144 err.to_string(),
7145 )?])
7146 }
7147 };
7148 timings.enter(RouteOpenPhase::Admission);
7149 let crate::forwarding::PendingRouteBindRelay {
7150 endpoint,
7151 module_sink,
7152 negotiated_ver,
7153 client_channel,
7154 client_epoch,
7155 module_channel,
7156 module_epoch,
7157 corr: relay_corr,
7158 receiver,
7159 } = pending;
7160 let mut reservation =
7161 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
7162
7163 let relay_registration = self
7168 .registry
7169 .get_module_by_connection(endpoint.connection_id)
7170 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7171 let relay_capabilities = relay_registration
7172 .as_ref()
7173 .and_then(|registration| registration.manifest.capabilities.as_ref());
7174 if let Some(selector) = &scope {
7175 if let Err(refusal) = self
7178 .scopes
7179 .read()
7180 .unwrap_or_else(|poisoned| poisoned.into_inner())
7181 .check_bind_proof(&principal, selector)
7182 {
7183 reservation.release_and_disarm();
7184 return Ok(vec![self.route_open_refusal_frame(
7185 ctx,
7186 &caller,
7187 &frame,
7188 &target_module_id,
7189 refusal.code,
7190 refusal.code,
7191 refusal.message,
7192 )?]);
7193 }
7194 }
7195 if let Some(stamp) = scope_stamp.as_ref() {
7196 if let Err(refusal) = crate::scopes::check_target_required_stamp(
7197 stamp_required,
7198 &target_module_id,
7199 relay_capabilities,
7200 )
7201 .and_then(|()| {
7202 crate::scopes::check_target_flow_support(
7203 stamp,
7204 &target_module_id,
7205 relay_capabilities,
7206 )
7207 })
7208 .and_then(|()| {
7209 crate::scopes::check_target_agent_run_support(
7210 stamp,
7211 &target_module_id,
7212 relay_capabilities,
7213 )
7214 }) {
7215 reservation.release_and_disarm();
7216 return Ok(vec![self.route_open_refusal_frame(
7217 ctx,
7218 &caller,
7219 &frame,
7220 &target_module_id,
7221 refusal.code,
7222 refusal.code,
7223 refusal.message,
7224 )?]);
7225 }
7226 }
7227
7228 if scope_stamp
7231 .as_ref()
7232 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
7233 && !crate::scopes::target_supports_federated(relay_capabilities)
7234 {
7235 scope_stamp = None;
7237 }
7238 if !relay_capabilities.is_some_and(|caps| {
7239 caps.provides
7240 .iter()
7241 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
7242 }) {
7243 if let Some(stamp) = scope_stamp.as_mut() {
7244 stamp.workdir = None;
7245 stamp.project_id = None;
7246 }
7247 }
7248 let worker_run = scope_stamp
7253 .as_ref()
7254 .is_some_and(crate::scopes::is_worker_run)
7255 && relay_capabilities.is_some_and(|caps| {
7256 caps.provides
7257 .iter()
7258 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
7259 });
7260 if !worker_run
7261 && !relay_capabilities.is_some_and(|caps| {
7262 caps.provides
7263 .iter()
7264 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
7265 })
7266 {
7267 if let Some(stamp) = scope_stamp.as_mut() {
7268 stamp.attributes.on_behalf_of_agent = None;
7269 }
7270 }
7271 if !relay_capabilities.is_some_and(|caps| {
7272 caps.provides
7273 .iter()
7274 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
7275 }) {
7276 if let Some(stamp) = scope_stamp.as_mut() {
7277 stamp.attributes.flow_version = None;
7278 stamp.attributes.flow_code_hash = None;
7279 stamp.attributes.flow_install_id = None;
7280 }
7281 }
7282 let call_mode = if worker_run
7283 || relay_capabilities
7284 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
7285 {
7286 Some(derive_call_mode(scope_stamp.as_ref()))
7287 } else {
7288 if let Some(stamp) = scope_stamp.as_mut() {
7289 stamp.expires_at_ms = None;
7290 }
7291 None
7292 };
7293
7294 debug!(
7295 connection_id = ctx.connection_id.get(),
7296 client_channel,
7297 client_epoch,
7298 module_channel,
7299 module_epoch,
7300 "reserved route handle pair"
7301 );
7302 let route_principal_label = principal_label(&principal);
7305 let relay = ModuleControlRequest::RouteBind {
7306 route_channel: module_channel,
7307 epoch: module_epoch,
7308 target,
7309 identity,
7310 principal: Some(principal),
7311 consumer_capabilities,
7312 role_versions,
7313 admission_facts,
7314 scope: scope_stamp,
7315 call_mode,
7316 };
7317 let peer = RouteBindPeer::new()
7318 .with_observed_peer(
7319 relay_capabilities
7320 .is_some_and(|caps| {
7321 caps.provides
7322 .iter()
7323 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
7324 })
7325 .then(|| {
7326 self.observed_peers
7327 .lock()
7328 .unwrap_or_else(|p| p.into_inner())
7329 .get(&ctx.connection_id)
7330 .cloned()
7331 .unwrap_or_else(ObservedPeer::unobserved)
7332 }),
7333 )
7334 .with_operator_app(
7335 self.connection_operator_app(operator_app.as_ref())
7336 .filter(|_| {
7337 relay_capabilities.is_some_and(|caps| {
7338 caps.provides
7339 .iter()
7340 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
7341 })
7342 }),
7343 )
7344 .with_caller(caller_for_target(
7345 caller_capture.as_ref(),
7346 relay_capabilities,
7347 &target_module_id,
7348 ));
7349 let relay_body = crate::bind_compat::encode_bind(
7350 &relay,
7351 &peer,
7352 &legacy_bind_project,
7353 relay_capabilities,
7354 )
7355 .map_err(|err| {
7356 RouterError::backend(
7357 0,
7358 frame.header.corr,
7359 format!("failed to encode route.bind request: {err}"),
7360 )
7361 })?;
7362 let relay_frame = Frame::build_with_version(
7363 negotiated_ver,
7364 FrameType::Request,
7365 control_flags(),
7366 0,
7367 0,
7368 relay_corr,
7369 relay_body,
7370 )
7371 .map_err(RouterError::FrameBuild)?;
7372
7373 timings.enter(RouteOpenPhase::BindSend);
7374 let sent = module_sink.send(relay_frame).await;
7375 timings.enter(RouteOpenPhase::Admission);
7376 if let Err(err) = sent {
7377 reservation.release_and_disarm();
7378 return Ok(vec![self.route_open_refusal_frame(
7379 ctx,
7380 &caller,
7381 &frame,
7382 &target_module_id,
7383 "relay_send_failed",
7384 "target_unavailable",
7385 err.to_string(),
7386 )?]);
7387 }
7388
7389 #[cfg(test)]
7390 if let Some(hold) = &self.route_bind_relay_sent_hold {
7391 hold.wait().await;
7395 hold.wait().await;
7396 }
7397
7398 if self
7403 .forwarding
7404 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
7405 .map_err(RouterError::Forwarding)?
7406 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
7407 {
7408 self.send_abandoned_route_bind_goodbye(
7409 &module_sink,
7410 negotiated_ver,
7411 module_channel,
7412 module_epoch,
7413 );
7414 }
7415
7416 timings.enter(RouteOpenPhase::BindAckWait);
7417 let outcome = timeout_at(relay_deadline, receiver).await;
7418 timings.enter(RouteOpenPhase::Admission);
7419 match outcome {
7420 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
7421 reservation.disarm();
7422 if breaker.record_accepted() {
7423 info!(
7424 module_id = %target_module_id,
7425 "route.bind breaker closed: the probe was accepted"
7426 );
7427 }
7428 self.observe_route_open_accept(
7429 ctx,
7430 &target_module_id,
7431 &route_principal_label,
7432 &caller.scope,
7433 );
7434 Ok(Vec::new())
7435 }
7436 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
7437 reservation.release_and_disarm();
7438 breaker.record_inconclusive();
7442 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
7443 }
7444 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
7445 reservation.release_and_disarm();
7446 breaker.record_inconclusive();
7447 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
7448 }
7449 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
7450 reservation.release_and_disarm();
7451 breaker.record_inconclusive();
7452 tracing::warn!(
7459 module_id = %target_module_id,
7460 "route.bind relay abandoned: {message}"
7461 );
7462 Ok(vec![self.route_open_refusal_frame(
7463 ctx,
7464 &caller,
7465 &frame,
7466 &target_module_id,
7467 "relay_abandoned",
7468 "target_unavailable",
7469 message,
7470 )?])
7471 }
7472 Ok(Err(_)) => {
7473 reservation.release_and_disarm();
7474 breaker.record_inconclusive();
7475 Ok(vec![self.route_open_refusal_frame(
7476 ctx,
7477 &caller,
7478 &frame,
7479 &target_module_id,
7480 "relay_waiter_canceled",
7481 "target_unavailable",
7482 "route.bind relay waiter was canceled before the module responded",
7483 )?])
7484 }
7485 Err(_) => {
7486 reservation.release_and_disarm();
7487 if let Some(opened) = breaker.record_timeout(
7491 self.route_bind_breaker_threshold,
7492 self.route_bind_breaker_cooldown,
7493 ) {
7494 warn!(
7495 module_id = %target_module_id,
7496 consecutive_timeouts = opened.consecutive_timeouts,
7497 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
7498 reopened_after_probe = opened.reopened_after_probe,
7499 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
7500 );
7501 }
7502 tracing::warn!(
7510 module_id = %target_module_id,
7511 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
7512 "route.bind relay timed out: module did not ack within budget"
7513 );
7514 Ok(vec![self.route_open_refusal_frame(
7515 ctx,
7516 &caller,
7517 &frame,
7518 &target_module_id,
7519 "relay_timed_out",
7520 "module_timeout",
7521 format!(
7522 "module_id '{target_module_id}' did not answer route.bind within {:?}",
7523 route_bind_relay_timeout
7524 ),
7525 )?])
7526 }
7527 }
7528 }
7529
7530 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7531 let response = ClientControlResponse::SupervisorSpawnSnapshot {
7532 snapshot: self.supervisor.spawn_snapshot(),
7533 };
7534 Ok(vec![control_response_body_frame(
7535 &frame,
7536 &response,
7537 "ClientControlResponse::SupervisorSpawnSnapshot",
7538 )?])
7539 }
7540
7541 fn handle_supervisor_spawn_subscribe(
7542 &self,
7543 ctx: &RouteCtx,
7544 frame: Frame,
7545 since: Option<SpawnCursor>,
7546 ) -> Result<Vec<Frame>, RouterError> {
7547 match self.supervisor.subscribe_spawns(
7548 ctx.connection_id,
7549 frame.header.corr,
7550 response_version(&frame),
7551 since,
7552 ctx.egress.clone(),
7553 ) {
7554 Ok(()) => Ok(Vec::new()),
7555 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
7556 Ok(vec![control_error_body_frame(
7557 &frame,
7558 ErrorBody {
7559 code: "spawn_cursor_incarnation_mismatch".to_string(),
7560 message: "spawn cursor belongs to a different daemon incarnation"
7561 .to_string(),
7562 detail: Some(serde_json::json!({
7563 "current_daemon_incarnation": current
7564 })),
7565 },
7566 )?])
7567 }
7568 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
7569 &frame,
7570 ErrorBody {
7571 code: "spawn_cursor_too_old".to_string(),
7572 message: "spawn cursor predates the retained event ring".to_string(),
7573 detail: Some(serde_json::json!({
7574 "oldest_retained_cursor": oldest
7575 })),
7576 },
7577 )?]),
7578 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
7579 0,
7580 frame.header.corr,
7581 format!("failed to open supervisor spawn subscription: {error}"),
7582 )),
7583 }
7584 }
7585
7586 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7587 let handler = self.clone();
7590 let runtime = tokio::runtime::Handle::current();
7591 let subscriber = tracing::dispatcher::get_default(Clone::clone);
7594 let corr = frame.header.corr;
7595 tokio::task::spawn_blocking(move || {
7596 runtime.block_on(
7597 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
7598 .with_subscriber(subscriber),
7599 )
7600 })
7601 .await
7602 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
7603 }
7604
7605 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7606 use crate::config_gate::runtime::ApprovedDaemonView;
7607 let publication = self
7608 .config_runtime
7609 .as_ref()
7610 .and_then(|owner| owner.polled_snapshot().ok());
7611 let approved = publication
7612 .as_ref()
7613 .map(|view| view.approved_daemon.clone())
7614 .unwrap_or_else(|| {
7615 if self.config_runtime.is_some() {
7616 ApprovedDaemonView::Unavailable
7617 } else {
7618 ApprovedDaemonView::NotStamped
7619 }
7620 });
7621 let generation = self
7622 .registry
7623 .generation()
7624 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7625 let supervised = self.supervisor.list();
7626 let mut statuses = BTreeMap::new();
7627 let mut not_running = BTreeSet::new();
7628 for module in &supervised {
7629 let status = module.status_for_control("list").map_err(|err| {
7630 RouterError::backend(
7631 0,
7632 frame.header.corr,
7633 format!("failed to read supervisor status: {err}"),
7634 )
7635 })?;
7636 if !status.enabled || !status.process_alive {
7637 not_running.insert(module.module_id().to_owned());
7638 }
7639 statuses.insert(module.module_id().to_owned(), status);
7640 }
7641 let mut process_resources = BTreeMap::new();
7644 for module in &supervised {
7645 process_resources.insert(
7646 module.module_id().to_owned(),
7647 crate::subtree_resources::read_process(module.clone()).await,
7648 );
7649 }
7650 let mut image_readings = BTreeMap::new();
7653 for module in &supervised {
7654 image_readings.insert(
7655 module.module_id().to_owned(),
7656 module.running_image_agreement().await,
7657 );
7658 }
7659 let readings = self.resource_tracker.list(supervised.clone());
7660 let mut modules = Vec::new();
7661 for module in supervised {
7662 let status = statuses
7663 .remove(module.module_id())
7664 .expect("each supervised module has a status snapshot");
7665 let (configured, _) = module.configuration().map_err(|err| {
7666 RouterError::backend(
7667 0,
7668 frame.header.corr,
7669 format!("failed to read module configuration: {err}"),
7670 )
7671 })?;
7672 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
7673 RouterError::backend(
7674 0,
7675 frame.header.corr,
7676 format!("failed to read retry time: {err}"),
7677 )
7678 })?;
7679 let image = image_readings
7680 .remove(module.module_id())
7681 .expect("each supervised module has an image snapshot");
7682 let resources = process_resources.remove(module.module_id());
7683 let mut verdict =
7684 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
7685 verdict.spec = Some(launch_spec_agreement(
7686 &configured,
7687 status.process_alive,
7688 status.launch_spec_digest.as_ref(),
7689 ));
7690 let pending_reload = Some(verdict);
7691 let configured_enabled = match &approved {
7692 ApprovedDaemonView::Approved {
7693 config: approved, ..
7694 } => approved
7695 .modules
7696 .iter()
7697 .find(|module| module.module_id == status.module_id)
7698 .map(|module| module.enabled),
7699 ApprovedDaemonView::NotStamped => self
7700 .capability_evaluator
7701 .configured_enabled(&status.module_id),
7702 ApprovedDaemonView::Unavailable => None,
7703 };
7704 let admission_refused = status
7705 .admission_refused
7706 .clone()
7707 .filter(|_| status.enabled && !status.process_alive)
7708 .filter(|_| {
7709 publication
7710 .as_ref()
7711 .and_then(|view| view.registration_declaration(&status.module_id))
7712 .as_deref()
7713 == Some("undeclared")
7714 });
7715 modules.push(
7718 SupervisorEntry::new(
7719 status.module_id,
7720 status.state.to_string(),
7721 status.enabled,
7722 status.live,
7723 status.health.status,
7724 )
7725 .with_configured_enabled(configured_enabled)
7726 .with_admission_refused(admission_refused)
7727 .with_next_retry_at_ms(next_retry_at_ms)
7728 .with_launch_nonce_env(Some(!cfg!(unix)))
7729 .with_launch_nonce_source(status.launch_nonce_source)
7730 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
7731 .with_protocol(status.protocol)
7732 .with_pending_reload(pending_reload)
7733 .with_last_probe_ms(status.health.last_probe_ms)
7734 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
7735 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
7736 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
7737 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
7738 .with_restart_count(Some(status.restart_count))
7739 .with_max_restarts(Some(status.max_restarts))
7740 .with_lifetime_restarts(Some(status.lifetime_restarts))
7741 .with_spawn_generation(Some(status.spawn_generation))
7742 .with_restart_window_secs(Some(status.restart_window.as_secs()))
7743 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
7744 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
7745 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
7746 .with_resources(resources),
7747 );
7748 }
7749 for module in &mut modules {
7750 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
7751 module.tree = tree.clone();
7752 module.cpu_last_hour = hour.clone();
7753 }
7754 module.tree_unavailable = readings
7755 .reasons
7756 .get(&module.module_id)
7757 .map(|reason| (*reason).to_owned());
7758 if module.tree.is_none() {
7759 module.tree_unavailable = Some(
7760 if not_running.contains(&module.module_id) {
7761 "not_running"
7762 } else if let Some(reason) = readings.reasons.get(&module.module_id) {
7763 reason
7764 } else {
7765 "not_yet_measured"
7766 }
7767 .to_owned(),
7768 );
7769 }
7770 }
7771 let response = ClientControlResponse::SupervisorList {
7772 generation,
7773 modules,
7774 daemon: readings.daemon,
7775 agent_commands: readings.agent_commands,
7776 };
7777 Ok(vec![control_response_body_frame(
7778 &frame,
7779 &response,
7780 "ClientControlResponse::SupervisorList",
7781 )?])
7782 }
7783
7784 fn handle_supervisor_stderr_tail(
7785 &self,
7786 frame: Frame,
7787 module_id: String,
7788 max_lines: Option<u32>,
7789 max_bytes: Option<u32>,
7790 ) -> Result<Vec<Frame>, RouterError> {
7791 let Some(module) = self.supervisor.get(&module_id) else {
7792 return Ok(vec![control_error_frame(
7793 &frame,
7794 "unknown_module",
7795 format!("module_id '{module_id}' is not supervised"),
7796 )?]);
7797 };
7798
7799 let snapshot = module.stderr_tail(
7800 max_lines.map(|value| value as usize),
7801 max_bytes.map(|value| value as usize),
7802 );
7803
7804 let response = ClientControlResponse::SupervisorStderrTail {
7805 module_id,
7806 tail: StderrTail {
7807 capture: match snapshot.capture {
7808 CaptureState::Captured => StderrCaptureState::Captured,
7809 CaptureState::Incomplete { reason } => {
7810 StderrCaptureState::Incomplete { reason }
7811 }
7812 CaptureState::NotCaptured { reason } => {
7813 StderrCaptureState::NotCaptured { reason }
7814 }
7815 },
7816 entries: snapshot
7817 .entries
7818 .into_iter()
7819 .map(|entry| match entry {
7820 TailEntry::Line {
7821 text,
7822 truncated,
7823 at_ms,
7824 } => StderrTailEntry::Line {
7825 text,
7826 truncated,
7827 at_ms,
7828 },
7829 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
7830 })
7831 .collect(),
7832 dropped_lines: snapshot.dropped_lines,
7833 },
7834 };
7835 Ok(vec![control_response_body_frame(
7836 &frame,
7837 &response,
7838 "ClientControlResponse::SupervisorStderrTail",
7839 )?])
7840 }
7841
7842 async fn observe_control_wait<F: std::future::Future>(
7843 &self,
7844 ctx: &RouteCtx,
7845 target: Option<&str>,
7846 kind: &'static str,
7847 future: F,
7848 ) -> F::Output {
7849 self.counters
7850 .observe_reader_wait(
7851 || {
7852 let module = self
7853 .forwarding
7854 .module_id_for_connection(ctx.connection_id)
7855 .ok()
7856 .flatten();
7857 let principal = if module.is_some() { "module" } else { "direct" };
7858 (
7859 self.resolve_reader_wait_target(kind, target)
7860 .or(module)
7861 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
7862 principal.into(),
7863 )
7864 },
7865 kind,
7866 future,
7867 )
7868 .await
7869 }
7870
7871 pub(crate) fn resolve_reader_wait_target(
7874 &self,
7875 op: &str,
7876 selector: Option<&str>,
7877 ) -> Option<String> {
7878 let selector = selector?;
7879 match op {
7880 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
7881 .supervisor
7882 .get(selector)
7883 .map(|module| module.module_id().to_owned()),
7884 ops::SUPERVISOR_HEALTH_PROBE => self
7885 .registry
7886 .get_module(selector)
7887 .ok()
7888 .flatten()
7889 .map(|registration| registration.manifest.module_id),
7890 _ => None,
7891 }
7892 }
7893
7894 async fn handle_supervisor_terminals(
7895 &self,
7896 ctx: &RouteCtx,
7897 frame: Frame,
7898 module_id: String,
7899 ) -> Result<Vec<Frame>, RouterError> {
7900 let Some(module) = self.supervisor.get(&module_id) else {
7901 return Ok(vec![control_error_frame(
7902 &frame,
7903 "unknown_module",
7904 format!("module_id '{module_id}' is not supervised"),
7905 )?]);
7906 };
7907
7908 let terminals = self
7911 .observe_control_wait(
7912 ctx,
7913 Some(module.module_id()),
7914 ops::SUPERVISOR_TERMINALS,
7915 module.read_durable_terminal_history(),
7916 )
7917 .await
7918 .map_err(|error| {
7919 RouterError::backend(
7920 0,
7921 frame.header.corr,
7922 format!("failed to read terminal history: {error}"),
7923 )
7924 })?;
7925 let response = ClientControlResponse::SupervisorTerminals {
7926 module_id,
7927 terminals,
7928 };
7929 Ok(vec![control_response_body_frame(
7930 &frame,
7931 &response,
7932 "ClientControlResponse::SupervisorTerminals",
7933 )?])
7934 }
7935
7936 fn handle_supervisor_routes(
7937 &self,
7938 frame: Frame,
7939 module_id: Option<String>,
7940 ) -> Result<Vec<Frame>, RouterError> {
7941 let modules = self
7942 .forwarding
7943 .route_census(module_id.as_deref())
7944 .map_err(RouterError::Forwarding)?
7945 .into_iter()
7946 .map(|(module_id, routes)| SupervisorRouteModule {
7947 module_id,
7948 routes: routes
7949 .into_iter()
7950 .map(|route| SupervisorRoute {
7951 consumer: match route.principal {
7952 Principal::Reserved { module_id } => {
7953 SupervisorRouteConsumer::Reserved { module_id }
7954 }
7955 Principal::Direct | Principal::Unverified => {
7956 SupervisorRouteConsumer::Direct {
7957 connection_id: route.goodbye_target.connection_id.get(),
7958 }
7959 }
7960 },
7961 age_ms: Instant::now()
7962 .saturating_duration_since(route.bound_at)
7963 .as_millis()
7964 .try_into()
7965 .unwrap_or(u64::MAX),
7966 draining: route.draining,
7967 drain_reason: route.drain_reason,
7968 })
7969 .collect(),
7970 })
7971 .collect();
7972 let response = ClientControlResponse::SupervisorRoutes { modules };
7973 Ok(vec![control_response_body_frame(
7974 &frame,
7975 &response,
7976 "ClientControlResponse::SupervisorRoutes",
7977 )?])
7978 }
7979
7980 fn running_registration_method(
7981 &self,
7982 module: &str,
7983 ) -> Option<subc_control::RegistrationMethod> {
7984 let registration = self.registry.get_module(module).ok()??;
7985 let connection = registration.connection_id;
7986 let channel = self.supervisor.registration_spawn(connection).is_some();
7987 let nonce = if channel {
7988 None
7989 } else {
7990 self.hello_launch_nonces
7991 .lock()
7992 .ok()?
7993 .nonce(connection)
7994 .map(str::to_owned)
7995 };
7996 if !channel && nonce.is_none() {
7997 return None;
7998 }
7999 self.supervisor
8000 .registration_snapshot_is_current_live(
8001 &self.registry,
8002 module,
8003 connection,
8004 nonce.as_deref(),
8005 )
8006 .then_some(if channel {
8007 subc_control::RegistrationMethod::Channel
8008 } else {
8009 subc_control::RegistrationMethod::Nonce
8010 })
8011 }
8012
8013 async fn handle_supervisor_provenance(
8014 &self,
8015 ctx: &RouteCtx,
8016 frame: Frame,
8017 module_id: Option<String>,
8018 ) -> Result<Vec<Frame>, RouterError> {
8019 let targeted = module_id.is_some();
8020 let selected = if let Some(module_id) = module_id {
8021 let Some(module) = self.supervisor.get(&module_id) else {
8022 return Ok(vec![control_error_frame(
8023 &frame,
8024 "unknown_module",
8025 format!("module_id '{module_id}' is not supervised"),
8026 )?]);
8027 };
8028 vec![module]
8029 } else {
8030 self.supervisor.list()
8031 };
8032
8033 let target = targeted.then(|| selected[0].module_id());
8034 let declarations = if let Some(owner) = &self.config_runtime {
8035 owner
8036 .on_blocking(|owner| owner.registration_declarations())
8037 .await
8038 .ok()
8039 } else {
8040 None
8041 };
8042 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
8043 let mut modules = Vec::with_capacity(selected.len());
8044 let mut extensions = Vec::with_capacity(selected.len());
8045 for module in &selected {
8046 let (status, observed_image) = module
8047 .status_and_running_image_agreement()
8048 .await
8049 .map_err(|err| {
8050 RouterError::backend(
8051 0,
8052 frame.header.corr,
8053 format!("failed to read supervisor status: {err}"),
8054 )
8055 })?;
8056 let module_declared = self
8057 .registry
8058 .get_module(&status.module_id)
8059 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
8060 .and_then(|registration| registration.manifest.provenance)
8061 .map(|build| ModuleDeclaredProvenance::Reported { build })
8062 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
8063 #[cfg(test)]
8064 let running_image = match &self.provenance_probe_override {
8065 Some(result) => result.clone(),
8066 None => observed_image,
8067 };
8068 #[cfg(not(test))]
8069 let running_image = observed_image;
8070 let registration = self.running_registration_method(&status.module_id);
8071 let mut extension = subc_control::SupervisorModuleProvenanceExt::new()
8072 .with_registration(registration);
8073 extension.registration_declaration = declarations
8074 .as_ref()
8075 .and_then(|entries| entries.get(&status.module_id))
8076 .cloned();
8077 if status.process_alive && status.registration_declaration.is_some() {
8078 extension.registration_declaration = status.registration_declaration.clone();
8079 }
8080 extensions.push(extension);
8081 modules.push(SupervisorModuleProvenance {
8082 module_id: status.module_id,
8083 module_declared,
8084 daemon_observed: SupervisorObservedProcess {
8085 launch_nonce_source: status.launch_nonce_source,
8086 pid: status.pid,
8087 spawned_at_ms: status.spawned_at_ms,
8088 spawned_from: status.spawned_from,
8089 running_image,
8090 },
8091 });
8092 }
8093 let daemon = SupervisorDaemonProvenance {
8094 daemon_build: self.daemon_provenance.build.clone(),
8095 daemon_observed: DaemonObservedProcess {
8096 pid: self.daemon_provenance.pid,
8097 started_at_ms: self
8098 .daemon_provenance
8099 .start_clock
8100 .map(|clock| clock.started_at_ms())
8101 .or(self.daemon_provenance.started_at_ms),
8102 running_image: self
8103 .daemon_provenance
8104 .probe
8105 .observe(
8106 self.daemon_provenance.pid,
8107 self.daemon_provenance.executable_path.as_deref(),
8108 self.daemon_provenance.executable_identity.clone(),
8109 self.daemon_provenance.process_start_time,
8110 )
8111 .await,
8112 },
8113 };
8114 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
8115 let mut response = serde_json::to_value(response)
8116 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
8117 for (entry, extension) in response["modules"]
8118 .as_array_mut()
8119 .expect("provenance modules are an array")
8120 .iter_mut()
8121 .zip(extensions)
8122 {
8123 extension.write_to(entry).map_err(|error| {
8124 RouterError::backend(0, frame.header.corr, error.to_string())
8125 })?;
8126 }
8127 Ok(vec![control_response_body_frame(
8128 &frame,
8129 &response,
8130 "ClientControlResponse::SupervisorProvenance",
8131 )?])
8132 })
8133 .await
8134 }
8135
8136 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
8137 self.refresh_capability_requirements();
8138 let publication = self
8139 .config_runtime
8140 .as_ref()
8141 .and_then(|owner| owner.polled_snapshot().ok());
8142 let generation = self
8143 .registry
8144 .generation()
8145 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
8146 let modules = self
8147 .supervisor
8148 .list()
8149 .into_iter()
8150 .map(|module| {
8151 let status = module.status_for_control("health").map_err(|err| {
8152 RouterError::backend(
8153 0,
8154 frame.header.corr,
8155 format!("failed to read supervisor health: {err}"),
8156 )
8157 })?;
8158 let module_id = status.module_id;
8159 let capability_detail = self
8160 .capability_evaluator
8161 .required_problem_detail(&module_id);
8162 let declaration_warning = self.registry.get_module(&module_id).ok().flatten()
8163 .filter(|registration| registration.manifest.registration_channel == Some(false))
8164 .filter(|_| publication.as_ref().and_then(|view| view.registration_declaration(&module_id))
8165 .as_deref().map(|next| if status.process_alive { status.registration_declaration.as_deref().unwrap_or(next) } else { next }) == Some("declared"))
8166 .map(|_| format!("{module_id}: HELLO registration_channel is false despite the approved channel declaration"));
8167 let capability_detail = append_capability_problem_detail(capability_detail, declaration_warning);
8168 Ok(SupervisorHealthEntry {
8169 module_id,
8170 status: status.health.status,
8171 detail: append_capability_problem_detail(
8172 status.health.detail,
8173 capability_detail,
8174 ),
8175 metrics: status.health.metrics,
8176 consecutive_failures: status.health.consecutive_failures,
8177 late_answer_count: status.health.late_answer_count,
8178 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
8179 last_action: status.health.last_action,
8180 last_action_ms: status.health.last_action_ms,
8181 last_probe_ms: status.health.last_probe_ms,
8182 })
8183 })
8184 .collect::<Result<Vec<_>, RouterError>>()?;
8185 let response = ClientControlResponse::SupervisorHealth {
8186 generation,
8187 modules,
8188 };
8189 Ok(vec![control_response_body_frame(
8190 &frame,
8191 &response,
8192 "ClientControlResponse::SupervisorHealth",
8193 )?])
8194 }
8195
8196 async fn handle_supervisor_restart(
8197 &self,
8198 frame: Frame,
8199 module_id: String,
8200 drain_timeout_ms: Option<u64>,
8201 requester: Principal,
8202 ) -> Result<Vec<Frame>, RouterError> {
8203 let operation_lock = self.supervisor.operation_lock();
8204 let _operation_guard = operation_lock.lock().await;
8205 let Some(module) = self.supervisor.get(&module_id) else {
8206 return Ok(vec![control_error_frame(
8207 &frame,
8208 "unknown_module",
8209 format!("module_id '{module_id}' is not supervised"),
8210 )?]);
8211 };
8212
8213 self.route_outages.mark_operator_action(&module_id);
8214 if let Err(err) = module
8215 .restart_requested(drain_timeout_ms, Some(requester))
8216 .await
8217 {
8218 self.route_outages
8219 .operator_action_ended_unrefused(&module_id);
8220 let (code, message) = match err {
8221 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8222 crate::supervise::SuperviseError::Disabled { .. } => {
8223 ("module_disabled", err.to_string())
8224 }
8225 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8226 ("swap_in_progress", err.to_string())
8227 }
8228 _ => (
8229 "target_unavailable",
8230 format!("failed to restart module_id '{module_id}': {err}"),
8231 ),
8232 };
8233 return Ok(vec![control_error_frame(&frame, code, message)?]);
8234 }
8235
8236 let response = ClientControlResponse::SupervisorAck {
8237 module_id,
8238 applied: true,
8239 };
8240 Ok(vec![control_response_body_frame(
8241 &frame,
8242 &response,
8243 "ClientControlResponse::SupervisorAck",
8244 )?])
8245 }
8246
8247 async fn handle_supervisor_swap(
8251 &self,
8252 frame: Frame,
8253 module_id: String,
8254 ready_timeout_ms: Option<u64>,
8255 requester: Principal,
8256 ) -> Result<Vec<Frame>, RouterError> {
8257 let module = {
8264 let operation_lock = self.supervisor.operation_lock();
8265 let _operation_guard = operation_lock.lock().await;
8266 self.supervisor.get(&module_id)
8267 };
8268 let Some(module) = module else {
8269 return Ok(vec![control_error_frame(
8270 &frame,
8271 "unknown_module",
8272 format!("module_id '{module_id}' is not supervised"),
8273 )?]);
8274 };
8275
8276 self.route_outages.mark_operator_action(&module_id);
8277 if let Err(err) = module
8278 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
8279 .await
8280 {
8281 self.route_outages
8282 .operator_action_ended_unrefused(&module_id);
8283 use crate::supervise::SuperviseError;
8284 let message = err.to_string();
8285 let error = match err {
8286 SuperviseError::ConfigHeld { .. } => ErrorBody::new("config_gated", message),
8287 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
8288 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
8289 code: "swap_refused".to_string(),
8290 message,
8291 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
8292 },
8293 SuperviseError::SwapFailed {
8294 arm,
8295 candidate_exit,
8296 ..
8297 } => ErrorBody {
8298 code: "swap_failed".to_string(),
8299 message,
8300 detail: Some(serde_json::json!({
8301 "arm": arm.as_str(),
8302 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
8303 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
8304 })),
8305 },
8306 _ => ErrorBody::new(
8307 "target_unavailable",
8308 format!("failed to swap module_id '{module_id}': {message}"),
8309 ),
8310 };
8311 return Ok(vec![control_error_body_frame(&frame, error)?]);
8312 }
8313 self.route_outages
8317 .operator_action_ended_unrefused(&module_id);
8318
8319 let response = ClientControlResponse::SupervisorAck {
8320 module_id,
8321 applied: true,
8322 };
8323 Ok(vec![control_response_body_frame(
8324 &frame,
8325 &response,
8326 "ClientControlResponse::SupervisorAck",
8327 )?])
8328 }
8329
8330 async fn handle_supervisor_reload(
8331 &self,
8332 frame: Frame,
8333 module_id: String,
8334 requester: Principal,
8335 ) -> Result<Vec<Frame>, RouterError> {
8336 let operation_lock = self.supervisor.operation_lock();
8337 let _operation_guard = operation_lock.lock().await;
8338 let Some(module) = self.supervisor.get(&module_id) else {
8339 return Ok(vec![control_error_frame(
8340 &frame,
8341 "unknown_module",
8342 format!("module_id '{module_id}' is not supervised"),
8343 )?]);
8344 };
8345
8346 self.route_outages.mark_operator_action(&module_id);
8347 if let Err(err) = module.reload_requested(Some(requester)).await {
8348 self.route_outages
8349 .operator_action_ended_unrefused(&module_id);
8350 let (code, message) = match err {
8351 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8352 crate::supervise::SuperviseError::Disabled { .. } => {
8353 ("module_disabled", err.to_string())
8354 }
8355 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8356 ("swap_in_progress", err.to_string())
8357 }
8358 _ => (
8359 "reload_failed",
8360 format!("failed to reload module_id '{module_id}': {err}"),
8361 ),
8362 };
8363 return Ok(vec![control_error_frame(&frame, code, message)?]);
8364 }
8365
8366 let response = ClientControlResponse::SupervisorAck {
8367 module_id,
8368 applied: true,
8369 };
8370 Ok(vec![control_response_body_frame(
8371 &frame,
8372 &response,
8373 "ClientControlResponse::SupervisorAck",
8374 )?])
8375 }
8376
8377 async fn handle_supervisor_rescan(
8381 &self,
8382 frame: Frame,
8383 mode: RescanMode,
8384 requester: Principal,
8385 ) -> Result<Vec<Frame>, RouterError> {
8386 let preview = mode.previews();
8387 let mut approved_preview = None;
8388 if matches!(mode, RescanMode::Preview) {
8389 if let Some(owner) = &self.config_runtime {
8390 match owner
8391 .on_blocking(|owner| {
8392 owner.observe_file_proposal()?;
8393 owner.approved_daemon_configuration()
8394 })
8395 .await
8396 {
8397 Ok(config) => approved_preview = config,
8398 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8399 }
8400 }
8401 }
8402 let Some(context) = self.rescan.clone() else {
8403 return Ok(vec![control_error_frame(
8404 &frame,
8405 "rescan_unavailable",
8406 "the daemon was not started with a reloadable config path".to_string(),
8407 )?]);
8408 };
8409
8410 let operation_lock = self.supervisor.operation_lock();
8411 let _operation_guard = operation_lock.lock().await;
8412 let loaded = match self.load_rescan_config(&mode, &context) {
8416 Ok(config) => config,
8417 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
8418 };
8419 let Some((config, activity_config)) = loaded else {
8427 return Ok(vec![control_error_frame(
8428 &frame,
8429 "invalid_daemon_config",
8430 format!(
8431 "daemon config not found at {}; refusing to rescan (an absent config would \
8432 retire every supervised module)",
8433 context.config_path.display()
8434 ),
8435 )?]);
8436 };
8437 let (
8438 configured_port,
8439 storage_config,
8440 admission_facts_carrier_module_id,
8441 admission_facts_targets,
8442 scope_authority_owners,
8443 federated_scope_owner,
8444 privileged_operator_prompt,
8445 modules,
8446 reserved_capabilities,
8447 ) = (
8448 config.port,
8449 config.storage,
8450 config.admission_facts_carrier_module_id,
8451 config.admission_facts_targets,
8452 config.scope_authority_owners,
8453 config.federated_scope_owner,
8454 config.privileged_operator_prompt,
8455 config.modules,
8456 config.reserved_capabilities,
8457 );
8458
8459 let mut restart_required = Vec::new();
8468 for section in RestartRequiredSection::ALL {
8469 let changed = match section {
8470 RestartRequiredSection::Port => configured_port != context.configured_port,
8471 RestartRequiredSection::Storage => storage_config != context.storage_config,
8472 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
8473 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
8474 }
8475 RestartRequiredSection::AdmissionFactsTargets => {
8476 admission_facts_targets != context.admission_facts_targets
8477 }
8478 RestartRequiredSection::ScopeAuthorityOwners => {
8479 scope_authority_owners != context.scope_authority_owners
8480 }
8481 RestartRequiredSection::FederatedScopeOwner => {
8482 federated_scope_owner != context.federated_scope_owner
8483 }
8484 RestartRequiredSection::PrivilegedOperatorPrompt => {
8485 privileged_operator_prompt != context.privileged_operator_prompt
8486 }
8487 };
8488 if changed {
8489 restart_required.push(section.label().to_string());
8490 }
8491 }
8492 if !restart_required.is_empty() {
8493 warn!(
8494 config_path = %context.config_path.display(),
8495 sections = %restart_required.join(", "),
8496 "daemon config changed outside the modules section; restart the daemon to apply those changes"
8497 );
8498 }
8499
8500 for configured in &modules {
8501 if let Err(err) = validate_spec(&configured.module_spec()) {
8502 return Ok(vec![control_error_frame(
8503 &frame,
8504 "invalid_daemon_config",
8505 format!("supervisor rescan rejected daemon config: {err}"),
8506 )?]);
8507 }
8508 }
8509
8510 let configured_capabilities = modules
8511 .iter()
8512 .map(|module| (module.module_id.clone(), module.enabled))
8513 .collect::<Vec<_>>();
8514 let preview_capability_warnings = if preview {
8515 let (_, registrations) = self.runtime_capability_snapshot()?;
8516 let current_modules = self
8517 .supervisor
8518 .list()
8519 .into_iter()
8520 .map(|module| module.module_id().to_string())
8521 .collect::<BTreeSet<_>>();
8522 let resulting_modules = configured_capabilities.clone();
8523 let removed = current_modules
8524 .into_iter()
8525 .filter(|module_id| {
8526 !resulting_modules
8527 .iter()
8528 .any(|(configured_id, _)| configured_id == module_id)
8529 })
8530 .collect::<Vec<_>>();
8531 let disabled = match &mode {
8534 RescanMode::Candidate(_) => resulting_modules
8535 .iter()
8536 .filter(|(_, enabled)| !enabled)
8537 .map(|(module_id, _)| module_id.clone())
8538 .collect::<Vec<_>>(),
8539 _ => Vec::new(),
8540 };
8541 self.capability_evaluator.preview_warnings(
8542 resulting_modules,
8543 &removed,
8544 &disabled,
8545 ®istrations,
8546 )
8547 } else {
8548 Vec::new()
8549 };
8550 let reconciled = match (&mode, approved_preview.as_ref()) {
8551 (RescanMode::Preview, Some(approved)) => Ok((
8552 config_gate::preview_approved_daemon(approved, &modules),
8553 Vec::new(),
8554 )),
8555 (RescanMode::Candidate(candidate), _) => self
8556 .preview_candidate(&context, modules, candidate)
8557 .map(|result| (result, Vec::new())),
8558 _ => {
8559 self.reconcile_supervised_modules(
8560 &context.supervisor,
8561 modules,
8562 preview,
8563 requester.clone(),
8564 )
8565 .await
8566 }
8567 };
8568 let (result, changed) = match reconciled {
8569 Ok(result) => result,
8570 Err(message) => {
8571 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
8572 }
8573 };
8574 if !preview {
8575 if let Some(ledger) = &self.activity {
8576 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
8577 ledger.config_changed(&result.added, &result.removed, &changed, requester);
8578 }
8579 if let Err(code) = self.configure_activity(activity_config).await {
8580 warn!(
8581 code,
8582 "activity retention could not be applied during rescan"
8583 );
8584 }
8585 self.capability_evaluator
8586 .configure(configured_capabilities, reserved_capabilities);
8587 self.capability_evaluator.wake_deadline_loop();
8588 self.refresh_capability_requirements();
8589 }
8590 let mut result = result;
8591 result.restart_required = restart_required;
8592 result.capability_warnings = preview_capability_warnings;
8593 let response = ClientControlResponse::SupervisorRescan { result };
8594 Ok(vec![control_response_body_frame(
8595 &frame,
8596 &response,
8597 "ClientControlResponse::SupervisorRescan",
8598 )?])
8599 }
8600
8601 async fn handle_supervisor_release_reserved(
8602 &self,
8603 frame: Frame,
8604 module_id: String,
8605 ) -> Result<Vec<Frame>, RouterError> {
8606 let Some(context) = self.rescan.clone() else {
8607 return Ok(vec![control_error_frame(
8608 &frame,
8609 "release_unavailable",
8610 "reserved-id release requires a daemon started with a reloadable config path",
8611 )?]);
8612 };
8613 let operation_lock = self.supervisor.operation_lock();
8614 let _operation_guard = operation_lock.lock().await;
8615 let approved = if let Some(owner) = &self.config_runtime {
8616 match owner
8619 .on_blocking(|owner| {
8620 owner.observe_file_proposal()?;
8621 owner.approved_daemon_configuration()
8622 })
8623 .await
8624 {
8625 Ok(config) => config,
8626 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8627 }
8628 } else {
8629 None
8630 };
8631 let loaded = if let Some(approved) = approved {
8632 Ok(Some(approved))
8633 } else {
8634 crate::daemon_config::load(&context.config_path)
8635 };
8636 let loaded = match loaded {
8637 Ok(Some(config)) => config,
8638 Ok(None) => {
8639 return Ok(vec![control_error_frame(
8640 &frame,
8641 "invalid_daemon_config",
8642 format!(
8643 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
8644 context.config_path.display()
8645 ),
8646 )?])
8647 }
8648 Err(err) => {
8649 return Ok(vec![control_error_frame(
8650 &frame,
8651 "invalid_daemon_config",
8652 format!("unable to verify reserved-id release against daemon config: {err}"),
8653 )?])
8654 }
8655 };
8656 if loaded
8657 .modules
8658 .iter()
8659 .any(|configured| configured.module_id == module_id)
8660 {
8661 return Ok(vec![control_error_frame(
8662 &frame,
8663 "reserved_module_configured",
8664 format!(
8665 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
8666 ),
8667 )?]);
8668 }
8669 if !self.supervisor.release_retained_reserved_gate(&module_id) {
8670 return Ok(vec![control_error_frame(
8671 &frame,
8672 "reserved_gate_not_retained",
8673 format!(
8674 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
8675 ),
8676 )?]);
8677 }
8678
8679 let response = ClientControlResponse::SupervisorAck {
8680 module_id,
8681 applied: true,
8682 };
8683 Ok(vec![control_response_body_frame(
8684 &frame,
8685 &response,
8686 "ClientControlResponse::SupervisorAck",
8687 )?])
8688 }
8689
8690 fn plan_rescan(
8694 &self,
8695 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8696 ) -> Result<RescanPlan, String> {
8697 let mut current = BTreeMap::new();
8698 for module in self.supervisor.list() {
8699 let (spec, health) = module.configuration().map_err(|err| {
8700 format!(
8701 "failed to read configuration for module_id '{}': {err}",
8702 module.module_id()
8703 )
8704 })?;
8705 let enabled = module
8706 .status()
8707 .map_err(|err| {
8708 format!(
8709 "failed to read status for module_id '{}': {err}",
8710 module.module_id()
8711 )
8712 })?
8713 .enabled;
8714 current.insert(
8715 module.module_id().to_string(),
8716 (module, spec, health, enabled),
8717 );
8718 }
8719 let configured = configured_modules
8720 .into_iter()
8721 .map(|module| (module.module_id.clone(), module))
8722 .collect::<BTreeMap<_, _>>();
8723
8724 let added = configured
8725 .keys()
8726 .filter(|module_id| !current.contains_key(*module_id))
8727 .cloned()
8728 .collect::<Vec<_>>();
8729 let removed = current
8730 .keys()
8731 .filter(|module_id| !configured.contains_key(*module_id))
8732 .cloned()
8733 .collect::<Vec<_>>();
8734 let mut changed_pending_reload = Vec::new();
8735 let mut configuration_changes = BTreeSet::new();
8736 let mut enabled_changes = BTreeSet::new();
8737 let mut unchanged = 0_u32;
8738
8739 for (module_id, configured_module) in &configured {
8740 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
8741 else {
8742 continue;
8743 };
8744 let launch_changed = *current_spec != configured_module.module_spec();
8748 let configuration_changed =
8749 launch_changed || *current_health != configured_module.health;
8750 let enabled_changed = *current_enabled != configured_module.enabled
8753 || self
8754 .capability_evaluator
8755 .configured_enabled(module_id)
8756 .is_some_and(|enabled| enabled != configured_module.enabled);
8757 if configuration_changed {
8758 configuration_changes.insert(module_id.clone());
8759 }
8760 if launch_changed {
8761 changed_pending_reload.push(module_id.clone());
8762 }
8763 if enabled_changed {
8764 enabled_changes.insert(module_id.clone());
8765 }
8766 if !configuration_changed && !enabled_changed {
8767 unchanged = unchanged.saturating_add(1);
8768 }
8769 }
8770 Ok(RescanPlan {
8771 current,
8772 configured,
8773 added,
8774 removed,
8775 changed_pending_reload,
8776 configuration_changes,
8777 enabled_changes,
8778 unchanged,
8779 })
8780 }
8781
8782 async fn reconcile_supervised_modules(
8791 &self,
8792 supervisor: &Supervisor,
8793 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8794 preview: bool,
8795 requester: Principal,
8796 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
8797 let plan = self.plan_rescan(configured_modules)?;
8798
8799 if preview {
8805 return Ok((plan.preview_result(), Vec::new()));
8806 }
8807 let changed = plan
8808 .configuration_changes
8809 .union(&plan.enabled_changes)
8810 .cloned()
8811 .collect::<Vec<_>>();
8812 let RescanPlan {
8813 current,
8814 configured,
8815 added,
8816 removed,
8817 changed_pending_reload,
8818 configuration_changes,
8819 enabled_changes,
8820 unchanged,
8821 } = plan;
8822 if !(added.is_empty()
8827 && removed.is_empty()
8828 && configuration_changes.is_empty()
8829 && enabled_changes.is_empty())
8830 {
8831 if let Some(context) = &self.rescan {
8832 context
8833 .generation
8834 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8835 }
8836 }
8837
8838 for module_id in &removed {
8839 let module = ¤t
8840 .get(module_id)
8841 .expect("removed module came from current supervisor state")
8842 .0;
8843 module
8844 .retire_requested(Some(requester.clone()))
8845 .await
8846 .map_err(|err| {
8847 format!("failed to retire module_id '{module_id}' during rescan: {err}")
8848 })?;
8849 self.supervisor.record_rescan_removal(module_id);
8883 self.supervisor.retire(module_id);
8884 self.route_outages.forget(module_id);
8885 }
8886
8887 for module_id in configured.keys() {
8888 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
8889 continue;
8890 };
8891 let configured_module = configured
8892 .get(module_id)
8893 .expect("configured module id came from configured map");
8894 if configuration_changes.contains(module_id) {
8895 module
8896 .update_configuration(
8897 configured_module.module_spec(),
8898 configured_module.health.clone(),
8899 configured_module.drain_timeout_ms,
8900 )
8901 .await
8902 .map_err(|err| {
8903 format!(
8904 "failed to update module_id '{module_id}' configuration during rescan: {err}"
8905 )
8906 })?;
8907 }
8908 if *current_enabled != configured_module.enabled {
8911 self.route_outages.mark_operator_action(module_id);
8914 module
8915 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
8916 .await
8917 .map_err(|err| {
8918 self.route_outages.operator_action_ended_unrefused(module_id);
8919 format!(
8920 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
8921 configured_module.enabled
8922 )
8923 })?;
8924 }
8925 }
8926
8927 for module_id in &added {
8928 let configured_module = configured
8929 .get(module_id)
8930 .expect("added module id came from configured map");
8931 supervisor
8932 .supervise_configured_requested_async(
8933 configured_module.module_spec(),
8934 configured_module.enabled,
8935 configured_module.health.clone(),
8936 configured_module.drain_timeout_ms,
8937 configured_module.restart,
8938 configured_module.register_timeout,
8939 Some(requester.clone()),
8940 )
8941 .await
8942 .map_err(|err| {
8943 format!("failed to add module_id '{module_id}' during rescan: {err}")
8944 })?;
8945 }
8946
8947 Ok((
8948 SupervisorRescanResult {
8949 added,
8950 removed,
8951 changed_pending_reload,
8952 enabled_changes: enabled_changes.iter().cloned().collect(),
8953 unchanged,
8954 preview: false,
8955 restart_required: Vec::new(),
8958 capability_warnings: Vec::new(),
8959 module_actions: Vec::new(),
8960 earlier_pending: Vec::new(),
8961 expect: None,
8962 },
8963 changed,
8964 ))
8965 }
8966
8967 async fn handle_supervisor_set_enabled(
8968 &self,
8969 frame: Frame,
8970 module_id: String,
8971 enabled: bool,
8972 requester: Principal,
8973 ) -> Result<Vec<Frame>, RouterError> {
8974 let operation_lock = self.supervisor.operation_lock();
8975 let _operation_guard = operation_lock.lock().await;
8976 let Some(module) = self.supervisor.get(&module_id) else {
8977 return Ok(vec![control_error_frame(
8978 &frame,
8979 "unknown_module",
8980 format!("module_id '{module_id}' is not supervised"),
8981 )?]);
8982 };
8983
8984 self.route_outages.mark_operator_action(&module_id);
8987 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
8988 Ok(applied) => applied,
8989 Err(err) => {
8990 self.route_outages
8991 .operator_action_ended_unrefused(&module_id);
8992 return Ok(vec![control_error_frame(
8993 &frame,
8994 if err.is_config_gate_error() {
8995 "config_gated"
8996 } else {
8997 "target_unavailable"
8998 },
8999 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
9000 )?]);
9001 }
9002 };
9003 if !applied {
9004 self.route_outages
9007 .operator_action_ended_unrefused(&module_id);
9008 }
9009
9010 self.capability_evaluator.wake_deadline_loop();
9011 self.refresh_capability_requirements();
9012 let response = ClientControlResponse::SupervisorAck { module_id, applied };
9013 Ok(vec![control_response_body_frame(
9014 &frame,
9015 &response,
9016 "ClientControlResponse::SupervisorAck",
9017 )?])
9018 }
9019
9020 async fn handle_supervisor_health_probe(
9021 &self,
9022 ctx: &RouteCtx,
9023 frame: Frame,
9024 module_id: String,
9025 ) -> Result<Vec<Frame>, RouterError> {
9026 self.refresh_capability_requirements();
9027 let Some(registration) = self
9028 .registry
9029 .get_module(&module_id)
9030 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
9031 else {
9032 return Ok(vec![control_error_frame(
9033 &frame,
9034 "unknown_module",
9035 format!("module_id '{module_id}' is not registered"),
9036 )?]);
9037 };
9038
9039 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
9052 {
9053 return Ok(vec![control_error_frame(
9054 &frame,
9055 "health_not_advertised",
9056 format!("module_id '{module_id}' did not advertise health.check"),
9057 )?]);
9058 }
9059
9060 self.observe_control_wait(
9061 ctx,
9062 Some(®istration.manifest.module_id),
9063 ops::SUPERVISOR_HEALTH_PROBE,
9064 async {
9065 let deadline = Instant::now() + self.health_probe_timeout;
9066 let pending = match self.forwarding.begin_module_control_rpc_for(
9067 &module_id,
9068 MODULE_CONTROL_OP_HEALTH_CHECK,
9069 deadline,
9070 ) {
9071 Ok(pending) => pending,
9072 Err(err) => {
9073 return Ok(vec![control_error_frame(
9074 &frame,
9075 forwarding_error_code(&err),
9076 err.to_string(),
9077 )?])
9078 }
9079 };
9080
9081 let PendingModuleControlRpc {
9082 endpoint,
9083 module_sink,
9084 negotiated_ver,
9085 corr: probe_corr,
9086 receiver,
9087 } = pending;
9088 let mut guard =
9089 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
9090 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
9091 .map_err(|err| {
9092 RouterError::backend(
9093 0,
9094 frame.header.corr,
9095 format!("failed to encode health.check request: {err}"),
9096 )
9097 })?;
9098 let probe_frame = Frame::build_with_version(
9099 negotiated_ver,
9100 FrameType::Request,
9101 control_flags(),
9102 0,
9103 0,
9104 probe_corr,
9105 probe_body,
9106 )
9107 .map_err(RouterError::FrameBuild)?;
9108
9109 match timeout_at(deadline, module_sink.send(probe_frame)).await {
9112 Ok(Ok(())) => {}
9113 Ok(Err(err)) => {
9114 return Ok(vec![control_error_frame(
9115 &frame,
9116 "target_unavailable",
9117 err.to_string(),
9118 )?]);
9119 }
9120 Err(_) => {
9121 return Ok(vec![control_error_frame(
9122 &frame,
9123 "module_timeout",
9124 format!(
9125 "module_id '{module_id}' did not answer health.check within {:?}",
9126 self.health_probe_timeout
9127 ),
9128 )?]);
9129 }
9130 }
9131
9132 match timeout_at(deadline, receiver).await {
9133 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => {
9134 Ok(vec![control_error_frame(
9135 &frame,
9136 "invalid_control_body",
9137 "health.check returned an unexpected config response",
9138 )?])
9139 }
9140 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
9141 guard.disarm();
9142 let Some(report) = response.health_report() else {
9143 return Ok(vec![control_error_frame(
9144 &frame,
9145 "invalid_control_body",
9146 "health.check RPC returned a non-health response",
9147 )?]);
9148 };
9149 let HealthReport {
9154 status,
9155 detail,
9156 metrics,
9157 } = report;
9158 let capability_detail = self
9159 .capability_evaluator
9160 .required_problem_detail(&module_id);
9161 let response = ClientControlResponse::SupervisorHealthProbe {
9162 module_id,
9163 status,
9164 detail: append_capability_problem_detail(detail, capability_detail),
9165 metrics,
9166 };
9167 Ok(vec![control_response_body_frame(
9168 &frame,
9169 &response,
9170 "ClientControlResponse::SupervisorHealthProbe",
9171 )?])
9172 }
9173 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
9174 guard.disarm();
9175 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
9180 }
9181 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
9182 guard.disarm();
9183 Ok(vec![control_error_frame(
9184 &frame,
9185 "target_unavailable",
9186 message,
9187 )?])
9188 }
9189 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
9190 guard.disarm();
9191 Ok(vec![control_error_frame(
9192 &frame,
9193 "invalid_control_body",
9194 message,
9195 )?])
9196 }
9197 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
9198 guard.disarm();
9199 Ok(vec![control_error_frame(
9200 &frame,
9201 "invalid_control_body",
9202 format!("expected module-control op '{expected}', got '{actual}'"),
9203 )?])
9204 }
9205 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
9206 guard.disarm();
9207 Ok(vec![control_error_frame(
9208 &frame,
9209 "module_timeout",
9210 format!(
9211 "module_id '{module_id}' answered health.check after {:?}",
9212 self.health_probe_timeout
9213 ),
9214 )?])
9215 }
9216 Ok(Err(_)) => Ok(vec![control_error_frame(
9217 &frame,
9218 "target_unavailable",
9219 "health.check waiter was canceled before the module responded",
9220 )?]),
9221 Err(_) => Ok(vec![control_error_frame(
9222 &frame,
9223 "module_timeout",
9224 format!(
9225 "module_id '{module_id}' did not answer health.check within {:?}",
9226 self.health_probe_timeout
9227 ),
9228 )?]),
9229 }
9230 },
9231 )
9232 .await
9233 }
9234
9235 fn supervisor_status(
9236 &self,
9237 module_id: &str,
9238 corr: u64,
9239 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
9240 self.supervisor
9241 .get(module_id)
9242 .map(|module| {
9243 let warming = module.is_warming_for_control("status").map_err(|err| {
9244 RouterError::backend(
9245 0,
9246 corr,
9247 format!(
9248 "failed to read supervisor warming state for module_id '{module_id}': {err}"
9249 ),
9250 )
9251 })?;
9252 module.status_for_control("status").map_err(|err| {
9253 RouterError::backend(
9254 0,
9255 corr,
9256 format!(
9257 "failed to read supervisor status for module_id '{module_id}': {err}"
9258 ),
9259 )
9260 }).map(|status| (status, warming))
9261 })
9262 .transpose()
9263 }
9264
9265 fn guard_module_control_op(
9266 &self,
9267 frame: &Frame,
9268 module_id: &str,
9269 op: &str,
9270 ) -> Result<Option<Frame>, RouterError> {
9271 if self.module_grants_op(module_id, op, frame.header.corr)? {
9272 return Ok(None);
9273 }
9274
9275 Ok(Some(control_error_frame(
9276 frame,
9277 "op_not_allowed",
9278 format!("module_id '{module_id}' did not grant control op '{op}'"),
9279 )?))
9280 }
9281
9282 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
9283 let Some(registration) = self
9284 .registry
9285 .get_module(module_id)
9286 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
9287 else {
9288 return Ok(false);
9289 };
9290 Ok(module_registration_grants_op(®istration.control_ops, op))
9291 }
9292
9293 fn handle_status_update(
9294 &self,
9295 endpoint: ModuleEndpointId,
9296 frame: Frame,
9297 ) -> Result<Vec<Frame>, RouterError> {
9298 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
9299 Ok(update) => update,
9300 Err(err) => {
9301 if is_known_module_push_op(&frame.body) {
9306 return Ok(vec![control_error_frame(
9307 &frame,
9308 "invalid_control_body",
9309 format!("malformed module control push body: {err}"),
9310 )?]);
9311 }
9312 return Ok(Vec::new());
9313 }
9314 };
9315
9316 match update {
9317 ModuleControlPush::RouteStatus {
9318 route_channel,
9319 route_epoch,
9320 status,
9321 } => {
9322 self.forwarding
9323 .cache_status(endpoint, route_channel, route_epoch, status)
9324 .map_err(RouterError::Forwarding)?;
9325 }
9326 }
9327 Ok(Vec::new())
9328 }
9329
9330 fn handle_route_poll(
9331 &self,
9332 ctx: &RouteCtx,
9333 frame: Frame,
9334 route_channel: u16,
9335 route_epoch: u32,
9336 kind: PollKind,
9337 ) -> Result<Vec<Frame>, RouterError> {
9338 let snapshot = self
9339 .forwarding
9340 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
9341 .map_err(RouterError::Forwarding)?;
9342 let response = match (kind, snapshot) {
9343 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
9344 ClientControlResponse::RoutePoll {
9345 route_channel,
9346 route_epoch,
9347 status,
9348 live: None,
9349 }
9350 }
9351 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9352 route_channel,
9353 route_epoch,
9354 status: None,
9355 live: None,
9356 },
9357 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
9358 let live = self
9384 .process_liveness
9385 .as_ref()
9386 .and_then(|source| source.process_live(&module_id))
9387 .unwrap_or(true);
9388 ClientControlResponse::RoutePoll {
9389 route_channel,
9390 route_epoch,
9391 status: None,
9392 live: Some(live),
9393 }
9394 }
9395 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9396 route_channel,
9397 route_epoch,
9398 status: None,
9399 live: Some(false),
9400 },
9401 };
9402
9403 Ok(vec![control_response_body_frame(
9404 &frame,
9405 &response,
9406 "ClientControlResponse::RoutePoll",
9407 )?])
9408 }
9409
9410 pub(crate) fn observe_module_control_completion(
9411 &self,
9412 completion: ModuleControlRpcCompletion,
9413 ) -> bool {
9414 match completion {
9415 ModuleControlRpcCompletion::Unknown => false,
9416 ModuleControlRpcCompletion::Settled => true,
9417 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
9418 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
9419 info!(
9420 module_id = %module_id,
9421 latency_ms,
9422 "late health.check answer proves the module is alive"
9423 );
9424 match self
9425 .supervisor
9426 .record_late_health_answer(&module_id, latency_ms)
9427 {
9428 Ok(true) => {}
9429 Ok(false) => debug!(
9430 module_id = %module_id,
9431 latency_ms,
9432 "late health.check answer has no active supervisor snapshot"
9433 ),
9434 Err(err) => warn!(
9435 module_id = %module_id,
9436 latency_ms,
9437 error = %err,
9438 "failed to record late health.check answer"
9439 ),
9440 }
9441 true
9442 }
9443 }
9444 }
9445
9446 fn refuse_to_end_module_connection_for_a_client(
9467 &self,
9468 module_connection_id: ConnectionId,
9469 corr: u64,
9470 err: ForwardingError,
9471 ) -> Result<(), RouterError> {
9472 if let ForwardingError::ConnectionClosing { connection_id } = err {
9473 if connection_id != module_connection_id {
9474 warn!(
9475 module_connection_id = module_connection_id.get(),
9476 client_connection_id = connection_id.get(),
9477 corr,
9478 "dropping a route.bind response for a closing client; the module connection keeps serving"
9479 );
9480 return Ok(());
9481 }
9482 }
9483 Err(RouterError::Forwarding(err))
9484 }
9485
9486 fn handle_module_relay_response(
9487 &self,
9488 connection_id: ConnectionId,
9489 frame: Frame,
9490 ) -> Result<Vec<Frame>, RouterError> {
9491 let mut secondary_error = None;
9492 let outcome = match frame.header.ty {
9493 FrameType::Response => match module_control_response_probe(&frame.body) {
9494 Ok(probe) if probe.op == "route.bind" => {
9495 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9496 Ok(ModuleControlResponse::RouteBindAck {}) => {
9497 RouteBindRelayOutcome::Accepted
9498 }
9499 Ok(other) => {
9500 let message =
9501 format!("route.bind response carried unexpected body: {other:?}");
9502 secondary_error = Some(control_error_frame(
9503 &frame,
9504 "invalid_control_body",
9505 message.clone(),
9506 )?);
9507 RouteBindRelayOutcome::ModuleGone(message)
9508 }
9509 Err(err) => {
9510 let message = format!("malformed route.bind response body: {err}");
9511 secondary_error = Some(control_error_frame(
9512 &frame,
9513 "invalid_control_body",
9514 message.clone(),
9515 )?);
9516 RouteBindRelayOutcome::ModuleGone(message)
9517 }
9518 }
9519 }
9520 Ok(probe) => {
9521 let outcome = if matches!(probe.op.as_str(), "config.review" | "consent/v1") {
9522 ModuleControlRpcOutcome::RawResponse(frame.body.clone())
9523 } else {
9524 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9525 Ok(response) => ModuleControlRpcOutcome::Response(response),
9526 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9527 "malformed {} response body: {err}",
9528 probe.op
9529 )),
9530 }
9531 };
9532 let completion = self
9533 .forwarding
9534 .complete_module_control_rpc(
9535 connection_id,
9536 frame.header.corr,
9537 Some(&probe.op),
9538 outcome,
9539 )
9540 .map_err(RouterError::Forwarding)?;
9541 if !self.observe_module_control_completion(completion) {
9542 debug!(
9543 connection_id = connection_id.get(),
9544 corr = frame.header.corr,
9545 op = %probe.op,
9546 "dropping late or unknown module-control RPC response"
9547 );
9548 }
9549 return Ok(Vec::new());
9550 }
9551 Err(err) => {
9552 if let Some(expected_op) = self
9553 .forwarding
9554 .pending_module_control_op(connection_id, frame.header.corr)
9555 .map_err(RouterError::Forwarding)?
9556 {
9557 let completion = self
9558 .forwarding
9559 .complete_module_control_rpc(
9560 connection_id,
9561 frame.header.corr,
9562 None,
9563 ModuleControlRpcOutcome::MalformedResponse(format!(
9564 "malformed {expected_op} response body: {err}"
9565 )),
9566 )
9567 .map_err(RouterError::Forwarding)?;
9568 if !self.observe_module_control_completion(completion) {
9569 debug!(
9570 connection_id = connection_id.get(),
9571 corr = frame.header.corr,
9572 "dropping late malformed module-control RPC response"
9573 );
9574 }
9575 return Ok(Vec::new());
9576 }
9577 let message = format!("malformed route.bind response body: {err}");
9578 secondary_error = Some(control_error_frame(
9579 &frame,
9580 "invalid_control_body",
9581 message.clone(),
9582 )?);
9583 RouteBindRelayOutcome::ModuleGone(message)
9584 }
9585 },
9586 FrameType::Error => {
9587 if self
9588 .forwarding
9589 .pending_module_control_op(connection_id, frame.header.corr)
9590 .map_err(RouterError::Forwarding)?
9591 .is_some()
9592 {
9593 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
9594 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
9595 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9596 "malformed module-control ERROR body: {err}"
9597 )),
9598 };
9599 let completion = self
9600 .forwarding
9601 .complete_module_control_rpc(
9602 connection_id,
9603 frame.header.corr,
9604 None,
9605 outcome,
9606 )
9607 .map_err(RouterError::Forwarding)?;
9608 if !self.observe_module_control_completion(completion) {
9609 debug!(
9610 connection_id = connection_id.get(),
9611 corr = frame.header.corr,
9612 "dropping late or unknown module-control RPC error"
9613 );
9614 }
9615 return Ok(Vec::new());
9616 }
9617 match serde_json::from_slice::<ErrorBody>(&frame.body) {
9618 Ok(body) => RouteBindRelayOutcome::Rejected(body),
9619 Err(err) => {
9620 let message = format!("malformed route.bind ERROR body: {err}");
9621 secondary_error = Some(control_error_frame(
9622 &frame,
9623 "invalid_control_body",
9624 message.clone(),
9625 )?);
9626 RouteBindRelayOutcome::ModuleGone(message)
9627 }
9628 }
9629 }
9630 ty => {
9631 return Ok(vec![control_error_frame(
9632 &frame,
9633 "unsupported_control_frame",
9634 format!("unsupported module channel-0 frame {ty:?}"),
9635 )?])
9636 }
9637 };
9638
9639 let settled =
9640 self.forwarding
9641 .complete_pending_relay(connection_id, frame.header.corr, outcome);
9642 let completion = match settled {
9643 Ok(completion) => completion,
9644 Err(err) => {
9645 self.refuse_to_end_module_connection_for_a_client(
9646 connection_id,
9647 frame.header.corr,
9648 err,
9649 )?;
9650 return Ok(secondary_error.into_iter().collect());
9651 }
9652 };
9653 if let Some(target) = completion.abandoned.as_ref() {
9654 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
9655 }
9656 if !completion.settled {
9657 debug!(
9658 connection_id = connection_id.get(),
9659 corr = frame.header.corr,
9660 frame_type = ?frame.header.ty,
9661 "dropping late or unknown route.bind relay response"
9662 );
9663 }
9664 Ok(secondary_error.into_iter().collect())
9665 }
9666
9667 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
9668 debug!(connection_id = connection_id.get(), "handling GOODBYE");
9669 self.cleanup_connection_with_end_reason(
9673 connection_id,
9674 RegistrationEndReason::ExplicitGoodbye,
9675 )
9676 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
9677 Ok(Vec::new())
9678 }
9679}
9680
9681impl Default for ControlHandler {
9682 fn default() -> Self {
9683 Self::new(Arc::new(Registry::default()))
9684 }
9685}
9686
9687impl crate::supervise::SwapPromotionObserver for ControlHandler {
9688 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
9689 self.apply_registration_capabilities(registration);
9690 }
9691}
9692
9693fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
9694 CapabilityRequirementStatus {
9695 consumer: status.consumer,
9696 capability: status.capability,
9697 need: match status.need {
9698 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
9699 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
9700 },
9701 verdict: status.verdict.as_str().to_string(),
9702 episode_seq: status.episode_seq,
9703 config_satisfiable: status.config_satisfiable,
9704 runtime_available: status.runtime_available,
9705 detail: status.detail,
9706 }
9707}
9708
9709fn append_capability_problem_detail(
9710 detail: Option<String>,
9711 capability_detail: Option<String>,
9712) -> Option<String> {
9713 match (detail, capability_detail) {
9714 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
9715 (Some(detail), None) => Some(detail),
9716 (None, Some(capability_detail)) => Some(capability_detail),
9717 (None, None) => None,
9718 }
9719}
9720
9721fn subc_ops() -> Vec<String> {
9722 SUBC_CONTROL_OPS
9723 .iter()
9724 .map(|op| (*op).to_string())
9725 .collect()
9726}
9727
9728fn module_subc_ops() -> Vec<String> {
9729 SUBC_CONTROL_OPS
9730 .iter()
9731 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
9732 .map(|op| (*op).to_string())
9733 .collect()
9734}
9735
9736#[cfg(test)]
9737fn module_baseline_control_ops() -> Vec<String> {
9738 MODULE_BASELINE_CONTROL_OPS
9739 .iter()
9740 .map(|op| (*op).to_string())
9741 .collect()
9742}
9743
9744fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
9745 let mut seen = HashSet::new();
9746 let mut effective = Vec::new();
9747 for op in MODULE_BASELINE_CONTROL_OPS {
9748 if seen.insert((*op).to_string()) {
9749 effective.push((*op).to_string());
9750 }
9751 }
9752 for op in declared.unwrap_or_default() {
9753 if seen.insert(op.clone()) {
9754 effective.push(op);
9755 }
9756 }
9757 effective
9758}
9759
9760fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
9761 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
9762}
9763
9764fn target_module_id(target: &RouteTarget) -> &str {
9765 match target {
9766 RouteTarget::ToolProvider { module_id }
9767 | RouteTarget::ManagementSurface { module_id }
9768 | RouteTarget::InternalService { module_id, .. } => module_id,
9769 }
9770}
9771
9772fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
9773 roles.iter().any(|role| match (target, role) {
9774 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
9775 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
9776 (
9777 RouteTarget::InternalService { service_id, .. },
9778 ProviderRole::InternalService {
9779 service_id: provided,
9780 ..
9781 },
9782 ) => service_id == provided,
9783 _ => false,
9784 })
9785}
9786
9787fn is_routable_role(role: &ProviderRole) -> bool {
9788 matches!(
9789 role,
9790 ProviderRole::ToolProvider { .. }
9791 | ProviderRole::ManagementSurface { .. }
9792 | ProviderRole::InternalService { .. }
9793 )
9794}
9795
9796#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9797enum ControlRequestBodyError {
9798 UnknownOp,
9799 InvalidBody,
9800}
9801
9802#[derive(Debug, Deserialize)]
9803struct ControlOpProbe {
9804 op: String,
9805}
9806
9807fn module_control_response_probe(body: &[u8]) -> Result<ControlOpProbe, serde_json::Error> {
9808 serde_json::from_slice(body).or_else(|original| {
9809 #[derive(Deserialize)]
9810 struct ConsentProbe {
9811 protocol: String,
9812 }
9813 match serde_json::from_slice::<ConsentProbe>(body) {
9814 Ok(probe) if probe.protocol == subc_protocol::consent::CONSENT_V1 => {
9815 Ok(ControlOpProbe { op: probe.protocol })
9816 }
9817 _ => Err(original),
9818 }
9819 })
9820}
9821
9822const MODULE_PUSH_OPS: &[&str] = &["route.status"];
9825
9826fn is_known_module_push_op(body: &[u8]) -> bool {
9827 serde_json::from_slice::<ControlOpProbe>(body)
9828 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
9829 .unwrap_or(false)
9830}
9831
9832fn is_known_module_request_op(body: &[u8]) -> bool {
9833 serde_json::from_slice::<ControlOpProbe>(body)
9834 .map(|probe| is_module_to_subc_op(&probe.op))
9835 .unwrap_or(false)
9836}
9837
9838fn is_module_to_subc_op(op: &str) -> bool {
9839 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
9840}
9841
9842fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
9843 debug!(
9844 op = %op,
9845 connection_id = connection_id.get(),
9846 corr,
9847 "control dispatch"
9848 );
9849}
9850
9851fn log_slow_control_dispatch(
9852 dispatch_started_at: Option<StdInstant>,
9853 op: &'static str,
9854 connection_id: ConnectionId,
9855 corr: u64,
9856 route_open_timings: Option<&RouteOpenTimings>,
9857) {
9858 let Some(dispatch_started_at) = dispatch_started_at else {
9859 return;
9860 };
9861 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
9863 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
9864 if let Some(timings) = route_open_timings {
9865 warn!(
9866 op = %op,
9867 connection_id = connection_id.get(),
9868 corr,
9869 elapsed_ms = elapsed.as_millis() as u64,
9870 module_id = ?timings.module_id,
9871 principal = %timings.principal,
9872 admission_ms = timings.admission.as_millis() as u64,
9873 project_root_ms = timings.project_root.as_millis() as u64,
9874 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
9875 bind_send_ms = timings.bind_send.as_millis() as u64,
9876 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
9877 "slow control dispatch"
9878 );
9879 return;
9880 }
9881 warn!(
9882 op = %op,
9883 connection_id = connection_id.get(),
9884 corr,
9885 elapsed_ms = elapsed.as_millis() as u64,
9886 "slow control dispatch"
9887 );
9888 }
9889}
9890
9891#[cfg(test)]
9893type TestCallerLookup =
9894 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
9895
9896fn privileged_operation(request: &impl PrivilegedRequest) -> Option<PrivilegedOperation<'_>> {
9898 request.privileged_operation()
9899}
9900
9901trait PrivilegedRequest {
9902 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>>;
9903}
9904
9905impl PrivilegedRequest for ClientControlRequest {
9906 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>> {
9907 let request = self;
9908 match request {
9909 ClientControlRequest::SupervisorRestart { module_id, .. } => {
9910 Some(PrivilegedOperation::Restart { module_id })
9911 }
9912 ClientControlRequest::SupervisorSwap { module_id, .. } => {
9913 Some(PrivilegedOperation::Swap { module_id })
9914 }
9915 ClientControlRequest::SupervisorReload { module_id } => {
9916 Some(PrivilegedOperation::Reload { module_id })
9917 }
9918 ClientControlRequest::SupervisorRescan { preview } => {
9919 if *preview {
9920 None
9921 } else {
9922 Some(PrivilegedOperation::Rescan)
9923 }
9924 }
9925 ClientControlRequest::SupervisorRescanChecked {
9930 preview,
9931 candidate,
9932 expect,
9933 } => (!*preview && candidate.is_none() && expect.is_some())
9934 .then_some(PrivilegedOperation::Rescan),
9935 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
9936 Some(PrivilegedOperation::SetEnabled {
9937 module_id,
9938 enabled: *enabled,
9939 })
9940 }
9941 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
9942 Some(PrivilegedOperation::ReleaseReserved { module_id })
9943 }
9944 ClientControlRequest::ServerDescribe {}
9945 | ClientControlRequest::DaemonShutdownIntent { .. }
9948 | ClientControlRequest::CatalogList { .. }
9949 | ClientControlRequest::RouteOpen { .. }
9950 | ClientControlRequest::RoutePoll { .. }
9951 | ClientControlRequest::SupervisorList {}
9952 | ClientControlRequest::SupervisorHealth {}
9953 | ClientControlRequest::SupervisorHealthProbe { .. }
9954 | ClientControlRequest::SupervisorRoutes { .. }
9955 | ClientControlRequest::SupervisorProvenance { .. }
9956 | ClientControlRequest::SupervisorStderrTail { .. }
9957 | ClientControlRequest::SupervisorTerminals { .. }
9958 | ClientControlRequest::SupervisorSpawnSnapshot {}
9959 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
9960 }
9961 }
9962}
9963
9964fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
9965 match request {
9966 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
9967 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
9968 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
9969 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
9970 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
9971 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
9972 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
9973 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
9974 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
9975 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
9976 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
9977 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
9978 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
9979 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
9980 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
9981 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
9982 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
9983 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
9984 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
9985 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
9986 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
9987 }
9988}
9989
9990struct RouteOpenCaller {
9998 principal: String,
10002 scope: String,
10005 harness: String,
10008}
10009
10010impl RouteOpenCaller {
10011 fn new(
10012 principal: Option<&Principal>,
10013 scope: Option<&ScopeSelector>,
10014 identity: &BindIdentity,
10015 ) -> Self {
10016 Self {
10017 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
10018 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
10019 harness: bounded_log_text(&identity.harness),
10020 }
10021 }
10022
10023 fn unknown() -> Self {
10026 Self {
10027 principal: "unknown".to_string(),
10028 scope: "unknown".to_string(),
10029 harness: "unknown".to_string(),
10030 }
10031 }
10032}
10033
10034const MAX_LOGGED_TEXT_CHARS: usize = 64;
10037
10038fn bounded_log_text(value: &str) -> String {
10042 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
10043 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
10044 bounded.push('…');
10045 }
10046 format!("{bounded:?}")
10047}
10048
10049fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
10055 use subc_protocol::scope::ScopeKind;
10056 let kind = match stamp.kind {
10057 ScopeKind::Head => "head",
10058 ScopeKind::Worker => "worker",
10059 ScopeKind::Ephemeral => "ephemeral",
10060 ScopeKind::Federated => "federated",
10061 _ => "unknown",
10062 };
10063 format!(
10064 "{kind}/{}/{}@{}",
10065 principal_label(&stamp.owner),
10066 bounded_log_text(&stamp.scope_ref),
10067 stamp.scope_epoch
10068 )
10069}
10070
10071fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
10078 let epoch = selector
10079 .scope_epoch
10080 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
10081 format!(
10082 "requested/{}/{}@{epoch}",
10083 bounded_log_text(&principal_label(&selector.owner)),
10084 bounded_log_text(&selector.scope_ref),
10085 )
10086}
10087
10088fn principal_label(principal: &Principal) -> String {
10089 match principal {
10090 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
10091 Principal::Direct => "direct".to_string(),
10092 other => format!("{other:?}"),
10093 }
10094}
10095
10096fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
10097 match request {
10098 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
10099 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
10100 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
10101 }
10102 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
10103 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
10104 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
10105 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
10106 ModuleControlRequestFromModule::RouteAuthorize { .. } => {
10107 subc_protocol::scope::ROUTE_AUTHORIZE_OP
10108 }
10109 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
10110 _ => "unsupported",
10111 }
10112}
10113
10114fn parse_client_control_request(
10115 body: &[u8],
10116) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
10117 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
10118 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10119 Ok(probe)
10120 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
10121 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
10122 {
10123 ControlRequestBodyError::InvalidBody
10124 }
10125 Ok(_) => ControlRequestBodyError::UnknownOp,
10126 Err(_) => ControlRequestBodyError::InvalidBody,
10127 };
10128 (err, classification)
10129 })
10130}
10131
10132fn parse_module_control_request_from_module(
10133 body: &[u8],
10134) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
10135 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
10136 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10137 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
10138 Ok(_) => ControlRequestBodyError::UnknownOp,
10139 Err(_) => ControlRequestBodyError::InvalidBody,
10140 };
10141 (err, classification)
10142 })
10143}
10144
10145#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
10146enum ProviderRoleKind {
10147 ToolProvider,
10148 PipelineStage,
10149 ManagementSurface,
10150 InternalService,
10151}
10152
10153fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
10154 match role {
10155 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
10156 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
10157 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
10158 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
10159 }
10160}
10161
10162fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
10163 roles.iter().map(provider_role_kind).collect()
10164}
10165
10166const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
10169
10170#[derive(Debug, Default, PartialEq, Eq)]
10172struct ScopeOutcomeCounts {
10173 created: usize,
10174 replaced: usize,
10175 updated: usize,
10176 unchanged: usize,
10177 refused: usize,
10178}
10179
10180impl ScopeOutcomeCounts {
10181 fn of(results: &[ScopeRecordResult]) -> Self {
10182 let mut counts = Self::default();
10183 for result in results {
10184 let slot = match result.outcome {
10185 ScopeRecordOutcome::Created => &mut counts.created,
10186 ScopeRecordOutcome::Replaced => &mut counts.replaced,
10187 ScopeRecordOutcome::Updated => &mut counts.updated,
10188 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
10189 ScopeRecordOutcome::Refused => &mut counts.refused,
10190 };
10191 *slot += 1;
10192 }
10193 counts
10194 }
10195}
10196
10197#[cfg(test)]
10198mod scope_outcome_count_tests {
10199 use super::*;
10200
10201 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
10202 ScopeRecordResult {
10203 scope_ref: "r".to_string(),
10204 scope_epoch: 1,
10205 outcome,
10206 code: None,
10207 message: None,
10208 version: None,
10209 parent_state: None,
10210 }
10211 }
10212
10213 #[test]
10216 fn every_outcome_is_counted_in_its_own_field() {
10217 let results = [
10218 result(ScopeRecordOutcome::Created),
10219 result(ScopeRecordOutcome::Created),
10220 result(ScopeRecordOutcome::Replaced),
10221 result(ScopeRecordOutcome::Updated),
10222 result(ScopeRecordOutcome::Unchanged),
10223 result(ScopeRecordOutcome::Refused),
10224 result(ScopeRecordOutcome::Refused),
10225 result(ScopeRecordOutcome::Refused),
10226 ];
10227 assert_eq!(
10228 ScopeOutcomeCounts::of(&results),
10229 ScopeOutcomeCounts {
10230 created: 2,
10231 replaced: 1,
10232 updated: 1,
10233 unchanged: 1,
10234 refused: 3,
10235 }
10236 );
10237 }
10238}
10239
10240fn capability_census_trigger(
10244 old: Option<&CapabilityDeclarations>,
10245 new: Option<&CapabilityDeclarations>,
10246) -> bool {
10247 let old_provides = old
10248 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
10249 .unwrap_or_default();
10250 let old_denies = old
10251 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
10252 .unwrap_or_default();
10253 let new = new.cloned().unwrap_or(CapabilityDeclarations {
10254 provides: Vec::new(),
10255 requires: Vec::new(),
10256 must_never_reach: Vec::new(),
10257 });
10258
10259 new.provides
10260 .iter()
10261 .any(|capability| !old_provides.contains(capability))
10262 || new
10263 .must_never_reach
10264 .iter()
10265 .any(|capability| !old_denies.contains(capability))
10266}
10267
10268fn catalog_update_frozen_field_message(
10269 registered: &ModuleManifest,
10270 provides: &[ProviderRole],
10271) -> Option<String> {
10272 let old_has_provides = !registered.provides.is_empty();
10273 let new_has_provides = !provides.is_empty();
10274 if old_has_provides != new_has_provides {
10275 return Some(format!(
10276 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
10277 registered.module_id
10278 ));
10279 }
10280
10281 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
10282 return Some(format!(
10283 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
10284 registered.module_id
10285 ));
10286 }
10287
10288 let registered_concurrency = manifest_concurrency(registered);
10289 let mut candidate = registered.clone();
10290 candidate.provides = provides.to_vec();
10291 let candidate_concurrency = manifest_concurrency(&candidate);
10292 if candidate_concurrency != registered_concurrency {
10293 return Some(format!(
10294 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
10295 registered.module_id, registered_concurrency, candidate_concurrency
10296 ));
10297 }
10298
10299 None
10302}
10303
10304fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
10305 manifest.provides.iter().any(is_routable_role)
10306}
10307
10308fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
10314 manifest
10315 .provides
10316 .iter()
10317 .find_map(|provider| match provider {
10318 ProviderRole::ToolProvider { concurrency, .. }
10319 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
10320 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
10321 })
10322 .unwrap_or(Concurrency::ModuleManaged)
10323}
10324
10325fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
10332 let has_management_surface = manifest
10333 .provides
10334 .iter()
10335 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
10336 if !has_management_surface {
10337 return false;
10338 }
10339 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
10340 return false;
10341 };
10342 let Some(provides) = raw
10343 .get("manifest")
10344 .and_then(|manifest| manifest.get("provides"))
10345 .and_then(serde_json::Value::as_array)
10346 else {
10347 return false;
10348 };
10349 provides.iter().any(|role| {
10354 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
10355 && role.get("concurrency").is_none()
10356 })
10357}
10358
10359fn negotiate_version(peer_version: u8) -> Result<u8, String> {
10360 if peer_version != PROTOCOL_VERSION {
10361 return Err(format!(
10362 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
10363 ));
10364 }
10365 Ok(PROTOCOL_VERSION)
10366}
10367
10368fn pong(frame: &Frame) -> Result<Frame, RouterError> {
10369 Frame::build_with_version(
10370 response_version(frame),
10371 FrameType::Pong,
10372 frame.header.flags,
10373 0,
10374 0,
10375 frame.header.corr,
10376 Vec::new(),
10377 )
10378 .map_err(RouterError::FrameBuild)
10379}
10380
10381pub(crate) fn control_error_frame(
10382 frame: &Frame,
10383 code: &'static str,
10384 message: impl Into<String>,
10385) -> Result<Frame, RouterError> {
10386 control_error_body_frame(
10387 frame,
10388 ErrorBody {
10389 code: code.to_string(),
10390 message: message.into(),
10391 detail: None,
10392 },
10393 )
10394}
10395
10396pub(crate) fn control_error_body_frame(
10404 frame: &Frame,
10405 error: ErrorBody,
10406) -> Result<Frame, RouterError> {
10407 control_error_frame_with_flags(frame, error, daemon_error_flags())
10408}
10409
10410fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
10416 control_error_frame_with_flags(frame, error, control_flags())
10417}
10418
10419fn control_error_frame_with_flags(
10420 frame: &Frame,
10421 error: ErrorBody,
10422 flags: Flags,
10423) -> Result<Frame, RouterError> {
10424 let body = serde_json::to_vec(&error).map_err(|err| {
10425 RouterError::backend(
10426 0,
10427 frame.header.corr,
10428 format!("failed to encode control ERROR: {err}"),
10429 )
10430 })?;
10431
10432 Frame::build_with_version(
10433 response_version(frame),
10434 FrameType::Error,
10435 flags,
10436 0,
10437 0,
10438 frame.header.corr,
10439 body,
10440 )
10441 .map_err(RouterError::FrameBuild)
10442}
10443
10444pub(crate) fn control_response_body_frame<T: Serialize>(
10445 frame: &Frame,
10446 reply: &T,
10447 label: &'static str,
10448) -> Result<Frame, RouterError> {
10449 let body = serde_json::to_vec(reply).map_err(|err| {
10450 RouterError::backend(
10451 0,
10452 frame.header.corr,
10453 format!("failed to encode {label}: {err}"),
10454 )
10455 })?;
10456
10457 Frame::build_with_version(
10458 response_version(frame),
10459 FrameType::Response,
10460 control_flags(),
10461 0,
10462 0,
10463 frame.header.corr,
10464 body,
10465 )
10466 .map_err(RouterError::FrameBuild)
10467}
10468
10469struct HelloAuthority {
10502 record: Arc<crate::spawn_authority::SpawnAuthorityRecord>,
10503 requested: ProviderDeclarations,
10504 froze: std::sync::atomic::AtomicBool,
10507}
10508
10509fn forwarding_error_code(err: &ForwardingError) -> &'static str {
10510 match err {
10511 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
10512 ForwardingError::NoModuleConnection => "target_unavailable",
10513 ForwardingError::ModuleReloading { .. } => "module_reloading",
10514 ForwardingError::ClientRouteChannelExhausted { .. }
10515 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
10516 ForwardingError::StaleModuleEndpoint
10517 | ForwardingError::UnknownReservation { .. }
10518 | ForwardingError::ConnectionClosing { .. }
10519 | ForwardingError::ClientEgressClosed { .. }
10520 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
10521 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
10524 ForwardingError::RelayCorrelationExhausted
10525 | ForwardingError::RouteOpenBuild(_)
10526 | ForwardingError::Poisoned => "forwarding_error",
10527 }
10528}
10529
10530fn response_version(frame: &Frame) -> u8 {
10531 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
10532 frame.header.ver
10533 } else {
10534 PROTOCOL_VERSION
10535 }
10536}
10537
10538fn control_flags() -> Flags {
10539 Flags::new(false, Priority::Passive, false)
10540}
10541
10542fn send_goodbye_target_best_effort(
10547 counters: &DaemonCounters,
10548 target: &GoodbyeTarget,
10549 context: &'static str,
10550) {
10551 let Ok(frame) = Frame::build_with_version(
10552 target.negotiated_ver,
10553 FrameType::Goodbye,
10554 control_flags(),
10555 target.channel,
10556 target.epoch,
10557 0,
10558 Vec::new(),
10559 ) else {
10560 return;
10561 };
10562 crate::forwarding::send_module_route_goodbye(
10563 counters,
10564 &target.sink,
10565 frame,
10566 target.module_id.as_deref(),
10567 context,
10568 );
10569}
10570
10571fn route_close_reason_for_opener(
10572 reason: RouteCloseReason,
10573 accepts_close_reasons: Option<&[String]>,
10574) -> RouteCloseReason {
10575 if reason == RouteCloseReason::AuthorityChanged
10576 && !accepts_close_reasons
10577 .is_some_and(|reasons| reasons.iter().any(|reason| reason == "authority_changed"))
10578 {
10579 RouteCloseReason::Reload
10582 } else {
10583 reason
10584 }
10585}
10586
10587pub(crate) fn send_route_control_pushes(
10588 forwarding: &ForwardingTable,
10589 routes: Vec<EndpointRoute>,
10590 push: ClientControlPush,
10591) {
10592 let reason = match &push {
10593 ClientControlPush::RouteClosing { reason, .. }
10594 | ClientControlPush::RouteClosed { reason, .. } => *reason,
10595 };
10596 let mut targets: Vec<(GoodbyeTarget, RouteCloseReason, Vec<u16>)> = Vec::new();
10597 for route in routes {
10598 let recipient_reason =
10599 route_close_reason_for_opener(reason, route.accepts_close_reasons.as_deref());
10600 let target = route.goodbye_target;
10601 if let Some((existing, _, channels)) = targets.iter_mut().find(|(existing, reason, _)| {
10602 existing.connection_id == target.connection_id && *reason == recipient_reason
10603 }) {
10604 debug_assert_eq!(
10605 existing.negotiated_ver, target.negotiated_ver,
10606 "one connection cannot negotiate multiple frame versions"
10607 );
10608 if !channels.contains(&target.channel) {
10609 channels.push(target.channel);
10610 }
10611 continue;
10612 }
10613 let channel = target.channel;
10614 targets.push((target, recipient_reason, vec![channel]));
10615 }
10616 for (target, recipient_reason, mut channels) in targets {
10617 channels.sort_unstable();
10618 let mut push = push.clone();
10619 match &mut push {
10620 ClientControlPush::RouteClosing {
10621 channels: covered,
10622 reason,
10623 ..
10624 }
10625 | ClientControlPush::RouteClosed {
10626 channels: covered,
10627 reason,
10628 ..
10629 } => {
10630 *covered = channels;
10631 *reason = recipient_reason;
10632 }
10633 }
10634 let body = match serde_json::to_vec(&push) {
10635 Ok(body) => body,
10636 Err(err) => {
10637 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
10638 continue;
10639 }
10640 };
10641 let frame = match Frame::build_with_version(
10642 target.negotiated_ver,
10643 FrameType::Push,
10644 control_flags(),
10645 0,
10646 0,
10647 0,
10648 body.clone(),
10649 ) {
10650 Ok(frame) => frame,
10651 Err(err) => {
10652 warn!(
10653 route_channel = target.channel,
10654 error = %err,
10655 "failed to build route lifecycle control PUSH frame"
10656 );
10657 continue;
10658 }
10659 };
10660 if let Err(err) = target.sink.try_send(frame) {
10661 if target.close_on_delivery_failure() {
10662 warn!(
10663 target_connection_id = target.connection_id.get(),
10664 route_channel = target.channel,
10665 error = %err,
10666 "route lifecycle control PUSH was not delivered to client; closing target connection"
10667 );
10668 let _ = forwarding.escalate_client_delivery_failure(
10669 target.connection_id,
10670 target.channel,
10671 target.epoch,
10672 CloseReason::new(
10673 "route_lifecycle_push_delivery_failed",
10674 format!(
10675 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
10676 target.channel
10677 ),
10678 ),
10679 crate::forwarding::UndeliveredFrame {
10680 module_id: target.module_id.as_deref(),
10681 sink: &target.sink,
10682 },
10683 );
10684 }
10685 }
10686 }
10687}
10688
10689#[cfg(test)]
10690mod tests {
10691 include!("control/agent_commands_tests.rs");
10692 include!("control/resource_revision_tests.rs");
10693 include!("control/route_open_recovery_tests.rs");
10694 #[tokio::test]
10695 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
10696 let registry = Arc::new(Registry::default());
10697 let handler = ControlHandler::new(registry);
10698 assert_eq!(
10699 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10700 None
10701 );
10702 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
10703 hello_via_sink(
10704 &handler,
10705 &ctx,
10706 &mut rx,
10707 hello_frame("daemon", PROTOCOL_VERSION, 1),
10708 )
10709 .await;
10710 assert_eq!(
10711 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10712 Some("daemon".into())
10713 );
10714 assert_eq!(
10715 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
10716 None
10717 );
10718 assert_eq!(
10719 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
10720 None,
10721 "a registered module is not necessarily supervised"
10722 );
10723 assert_eq!(
10724 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
10725 None,
10726 "aggregate provenance has no single target"
10727 );
10728 assert_eq!(
10729 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
10730 None
10731 );
10732 }
10733
10734 mod peer_manifest {
10735 include!("peer_manifest_tests.rs");
10736 }
10737 mod catalog_declarations {
10738 include!("control/catalog_tests.rs");
10739 }
10740 mod activity_read {
10741 include!("activity_read_tests.rs");
10742 }
10743
10744 mod lifecycle_activity {
10745 include!("control_lifecycle_tests.rs");
10746 }
10747 #[cfg(unix)]
10748 #[tokio::test]
10749 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
10750 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
10751 let path = dir.join("subc.jsonc");
10752 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
10753 "program":"/bin/sleep","args":["60"],"protocol":"none",
10754 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
10755 }}}).to_string()).unwrap();
10756 let mut configured = crate::daemon_config::load(&path)
10757 .unwrap()
10758 .unwrap()
10759 .modules
10760 .pop()
10761 .unwrap();
10762 let registry = std::sync::Arc::new(crate::Registry::default());
10763 let handle = crate::SupervisorHandle::new();
10764 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
10765 .with_handle(handle.clone());
10766 let module = supervisor
10767 .supervise_configured_with_health(
10768 configured.module_spec(),
10769 true,
10770 configured.health.clone(),
10771 None,
10772 configured.restart,
10773 )
10774 .unwrap();
10775 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
10776 let before = module.status().unwrap().pid;
10777 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
10778 configured.health.cadence = std::time::Duration::from_secs(3600);
10779 let health_only = handler
10780 .reconcile_supervised_modules(
10781 &supervisor,
10782 vec![configured.clone()],
10783 false,
10784 Principal::Direct,
10785 )
10786 .await
10787 .unwrap();
10788 assert!(
10789 health_only.0.changed_pending_reload.is_empty(),
10790 "health policy is already applied live"
10791 );
10792 assert_eq!(module.status().unwrap().pid, before);
10793 assert_eq!(
10794 module.configuration().unwrap().1.http,
10795 configured.health.http
10796 );
10797 configured.args = vec!["61".into()];
10798 let launch = handler
10799 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
10800 .await
10801 .unwrap();
10802 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
10803 assert_eq!(
10804 module.status().unwrap().pid,
10805 before,
10806 "a launch edit is stored until reload"
10807 );
10808 module.drain().await.unwrap();
10809 }
10810 use cortexkit_test_support::ScratchDir;
10811 use std::{
10812 collections::BTreeMap,
10813 fmt,
10814 path::PathBuf,
10815 sync::{Arc, Mutex},
10816 time::Duration,
10817 };
10818
10819 use serde_json::{json, Value};
10820 use subc_protocol::{
10821 manifest::{
10822 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
10823 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
10824 },
10825 session::HealthStatus,
10826 FrameType,
10827 };
10828
10829 use super::*;
10830 use crate::{
10831 forwarding::{DataRoute, DataRouteState},
10832 registry::ChannelState,
10833 router::FrameSink,
10834 stderr_tail::DEFAULT_MAX_LINE_BYTES,
10835 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
10836 RouteCtx, Router,
10837 };
10838 use tokio::{
10839 sync::mpsc,
10840 time::{sleep, Instant},
10841 };
10842 use tracing::{
10843 field::{Field, Visit},
10844 Event, Subscriber,
10845 };
10846 use tracing_subscriber::{layer::Context, prelude::*, Layer};
10847
10848 fn fake_aft_stub_path() -> PathBuf {
10867 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
10868 assert!(
10869 path.exists(),
10870 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
10871 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
10872 path.display()
10873 );
10874 path
10875 }
10876
10877 fn client_retries(code: &str) -> bool {
10882 subc_protocol::error_codes::is_retryable_route_open(code)
10883 }
10884
10885 #[test]
10896 fn retryability_of_forwarding_codes_matches_the_failure() {
10897 let transient = [
10900 ForwardingError::NoModuleConnection,
10901 ForwardingError::ModuleReloading {
10902 module_id: "m".into(),
10903 },
10904 ForwardingError::StaleModuleEndpoint,
10905 ForwardingError::UnknownReservation {
10906 client_channel: 1,
10907 module_channel: 1,
10908 },
10909 ForwardingError::ConnectionClosing {
10910 connection_id: ConnectionId::new(1),
10911 },
10912 ForwardingError::ClientEgressClosed {
10913 connection_id: ConnectionId::new(1),
10914 },
10915 ForwardingError::ModuleEgressUnavailable {
10916 connection_id: ConnectionId::new(1),
10917 },
10918 ];
10919 for err in transient {
10920 let code = forwarding_error_code(&err);
10921 assert!(
10922 client_retries(code),
10923 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
10924 );
10925 }
10926
10927 let permanent = [
10932 ForwardingError::ConnectionRoleConflict {
10933 connection_id: ConnectionId::new(1),
10934 },
10935 ForwardingError::ClientRouteChannelExhausted {
10936 connection_id: ConnectionId::new(1),
10937 },
10938 ForwardingError::ModuleRouteChannelExhausted {
10939 endpoint: ModuleEndpointId {
10940 connection_id: ConnectionId::new(1),
10941 generation: 1,
10942 },
10943 },
10944 ForwardingError::RelayCorrelationExhausted,
10945 ForwardingError::RouteOpenBuild("x".into()),
10946 ForwardingError::Poisoned,
10947 ];
10948 for err in permanent {
10949 let code = forwarding_error_code(&err);
10950 assert!(
10951 !client_retries(code),
10952 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
10953 );
10954 }
10955 }
10956
10957 #[tokio::test]
10985 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
10986 let handler = ControlHandler::default();
10987 let frame =
10988 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
10989
10990 let stamped = handler
10994 .route_open_principal(ConnectionId::new(1), &frame, None)
10995 .unwrap()
10996 .unwrap();
10997 assert_eq!(
10998 stamped,
10999 Principal::Direct,
11000 "a caller that proved nothing must not be stamped as a supervised module"
11001 );
11002
11003 let forged = handler
11008 .route_open_principal(
11009 ConnectionId::new(1),
11010 &frame,
11011 Some(ConsumerIdentity {
11012 module_id: "aft".to_string(),
11013 launch_nonce: "not-a-real-nonce".to_string(),
11014 }),
11015 )
11016 .unwrap();
11017 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
11018 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
11019 }
11020
11021 #[test]
11038 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
11039 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
11040 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
11041 let ClientControlRequest::RouteOpen {
11042 consumer_identity, ..
11043 } = parsed
11044 else {
11045 panic!("route.open body must parse as RouteOpen");
11046 };
11047 assert_eq!(
11048 consumer_identity,
11049 Some(ConsumerIdentity {
11050 module_id: "aft".to_string(),
11051 launch_nonce: "n".to_string(),
11052 }),
11053 "the wire field name must reach the value route_open_principal reads"
11054 );
11055 }
11056
11057 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
11058 ModuleManifest::builder(module_id, "0.1.0")
11059 .protocol_ver(protocol_ver)
11060 .provides(vec![ProviderRole::ToolProvider {
11061 tools: vec![Tool {
11062 name: "read".to_string(),
11063 description: None,
11064 execution_mode: ExecutionMode::Pure,
11065 schema: json!({"type": "object"}),
11066 }],
11067 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
11068 concurrency: Concurrency::ModuleManaged,
11069 emits_push: true,
11070 sub_supervises: true,
11071 }])
11072 .build()
11073 }
11074
11075 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
11076 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
11077 }
11078
11079 fn hello_frame_with_control_ops(
11080 module_id: &str,
11081 protocol_ver: u8,
11082 corr: u64,
11083 control_ops: Option<Vec<String>>,
11084 ) -> Frame {
11085 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
11086 }
11087
11088 fn hello_frame_with_nonce(
11089 module_id: &str,
11090 protocol_ver: u8,
11091 corr: u64,
11092 launch_nonce: Option<&str>,
11093 ) -> Frame {
11094 hello_frame_full(
11095 module_id,
11096 protocol_ver,
11097 corr,
11098 None,
11099 launch_nonce.map(ToOwned::to_owned),
11100 )
11101 }
11102
11103 fn hello_frame_full(
11104 module_id: &str,
11105 protocol_ver: u8,
11106 corr: u64,
11107 control_ops: Option<Vec<String>>,
11108 launch_nonce: Option<String>,
11109 ) -> Frame {
11110 let body = serde_json::to_vec(&ModuleHelloBody {
11111 manifest: manifest(module_id, protocol_ver),
11112 protocol_ver,
11113 control_ops,
11114 launch_nonce,
11115 })
11116 .unwrap();
11117 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11118 }
11119
11120 fn hello_frame_with_rich_ui(
11121 module_id: &str,
11122 protocol_ver: u8,
11123 corr: u64,
11124 launch_nonce: Option<&str>,
11125 rich_ui: Value,
11126 ) -> Frame {
11127 let mut body = serde_json::to_value(ModuleHelloBody {
11128 manifest: manifest(module_id, protocol_ver),
11129 protocol_ver,
11130 control_ops: None,
11131 launch_nonce: launch_nonce.map(str::to_string),
11132 })
11133 .unwrap();
11134 body["manifest"]["rich_ui"] = rich_ui;
11135 Frame::build(
11136 FrameType::Hello,
11137 control_flags(),
11138 0,
11139 0,
11140 corr,
11141 serde_json::to_vec(&body).unwrap(),
11142 )
11143 .unwrap()
11144 }
11145
11146 fn non_routable_hello_frame_with_control_ops(
11147 module_id: &str,
11148 corr: u64,
11149 control_ops: Option<Vec<String>>,
11150 ) -> Frame {
11151 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11152 manifest.provides.clear();
11153 let body = serde_json::to_vec(&ModuleHelloBody {
11154 manifest,
11155 protocol_ver: PROTOCOL_VERSION,
11156 control_ops,
11157 launch_nonce: None,
11158 })
11159 .unwrap();
11160 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11161 }
11162
11163 fn capability_grammar_hello_frame(
11164 capabilities: Value,
11165 runtime_computed: Option<Value>,
11166 corr: u64,
11167 ) -> Frame {
11168 let mut body = serde_json::to_value(ModuleHelloBody {
11169 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
11170 protocol_ver: PROTOCOL_VERSION,
11171 control_ops: None,
11172 launch_nonce: None,
11173 })
11174 .expect("HELLO body serializes");
11175 body["manifest"]["capabilities"] = capabilities;
11176 if let Some(runtime_computed) = runtime_computed {
11177 body["runtime_computed"] = runtime_computed;
11178 }
11179 Frame::build(
11180 FrameType::Hello,
11181 control_flags(),
11182 0,
11183 0,
11184 corr,
11185 serde_json::to_vec(&body).expect("HELLO body reserializes"),
11186 )
11187 .expect("HELLO frame builds")
11188 }
11189
11190 fn channel_request(channel: u16, corr: u64) -> Frame {
11191 Frame::build(
11192 FrameType::Request,
11193 Flags::new(true, Priority::Interactive, false),
11194 channel,
11195 0,
11196 corr,
11197 b"opaque".to_vec(),
11198 )
11199 .unwrap()
11200 }
11201
11202 fn route_ctx(
11203 connection_id: ConnectionId,
11204 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11205 let (tx, rx) = mpsc::channel(8);
11206 (
11207 RouteCtx {
11208 connection_id,
11209 egress: FrameSink::new(tx),
11210 },
11211 rx,
11212 )
11213 }
11214
11215 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
11216 serde_json::from_slice(&frame.body).unwrap()
11217 }
11218
11219 async fn hello_via_sink(
11224 handler: &ControlHandler,
11225 ctx: &RouteCtx,
11226 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11227 hello: Frame,
11228 ) -> Frame {
11229 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
11230 assert!(
11231 replies.is_empty(),
11232 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
11233 );
11234 let ack = rx
11235 .try_recv()
11236 .expect("HELLO_ACK is queued on the module sink")
11237 .frame;
11238 assert_eq!(ack.header.ty, FrameType::HelloAck);
11239 ack
11240 }
11241
11242 fn parse_error(frame: &Frame) -> Value {
11243 serde_json::from_slice(&frame.body).unwrap()
11244 }
11245
11246 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
11247 serde_json::from_slice(&frame.body).unwrap()
11248 }
11249
11250 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
11251 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
11252 route_channel,
11253 route_epoch: 0,
11254 kind,
11255 })
11256 .unwrap();
11257 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11258 }
11259
11260 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
11261 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
11262 module_id: module_id.to_string(),
11263 })
11264 .unwrap();
11265 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11266 }
11267
11268 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
11269 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
11270 }
11271
11272 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
11274 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
11275 let mut frame = route_open_frame(corr, module_id, root);
11276 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
11277 body["identity"]["project_root"] = serde_json::json!(path.as_path());
11278 frame.body = serde_json::to_vec(&body).unwrap();
11279 frame
11280 }
11281
11282 fn route_open_frame_with_consumer_capabilities(
11283 corr: u64,
11284 module_id: &str,
11285 _project_root: ScratchDir,
11286 consumer_capabilities: Option<Vec<String>>,
11287 ) -> Frame {
11288 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11289 busy_refusals: None,
11290 target: RouteTarget::ToolProvider {
11291 module_id: module_id.to_string(),
11292 },
11293 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
11294 consumer_identity: None,
11295 consumer_capabilities,
11296 role_versions: None,
11297 admission_facts: None,
11298 scope: None,
11299 })
11300 .unwrap();
11301 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11302 }
11303
11304 fn route_open_frame_with_role_versions(
11305 corr: u64,
11306 module_id: &str,
11307 _project_root: ScratchDir,
11308 role_versions: Option<BTreeMap<String, String>>,
11309 ) -> Frame {
11310 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11311 busy_refusals: None,
11312 target: RouteTarget::ToolProvider {
11313 module_id: module_id.to_string(),
11314 },
11315 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11316 consumer_identity: None,
11317 consumer_capabilities: None,
11318 role_versions,
11319 admission_facts: None,
11320 scope: None,
11321 })
11322 .unwrap();
11323 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11324 }
11325
11326 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
11327 entries
11328 .iter()
11329 .map(|(role, version)| (role.to_string(), version.to_string()))
11330 .collect()
11331 }
11332
11333 fn route_open_frame_with_admission_facts(
11334 corr: u64,
11335 module_id: &str,
11336 _project_root: ScratchDir,
11337 consumer_identity: Option<subc_control::ConsumerIdentity>,
11338 facts: Option<Value>,
11339 ) -> Frame {
11340 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11341 busy_refusals: None,
11342 target: RouteTarget::ToolProvider {
11343 module_id: module_id.to_string(),
11344 },
11345 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11346 consumer_identity,
11347 consumer_capabilities: None,
11348 role_versions: None,
11349 admission_facts: facts,
11350 scope: None,
11351 })
11352 .unwrap();
11353 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11354 }
11355
11356 #[derive(Clone, Default)]
11357 struct EventCapture {
11358 events: Arc<Mutex<Vec<CapturedEvent>>>,
11359 }
11360
11361 #[derive(Clone, Debug)]
11362 struct CapturedEvent {
11363 target: String,
11364 level: tracing::Level,
11365 fields: BTreeMap<String, String>,
11366 }
11367
11368 impl EventCapture {
11369 fn events(&self) -> Vec<CapturedEvent> {
11370 self.events.lock().unwrap().clone()
11371 }
11372 }
11373
11374 impl<S> Layer<S> for EventCapture
11375 where
11376 S: Subscriber,
11377 {
11378 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
11379 let mut visitor = EventFieldVisitor::default();
11380 event.record(&mut visitor);
11381 self.events.lock().unwrap().push(CapturedEvent {
11382 target: event.metadata().target().to_string(),
11383 level: *event.metadata().level(),
11384 fields: visitor.fields,
11385 });
11386 }
11387 }
11388
11389 #[derive(Default)]
11390 struct EventFieldVisitor {
11391 fields: BTreeMap<String, String>,
11392 }
11393
11394 impl Visit for EventFieldVisitor {
11395 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
11396 self.fields
11397 .insert(field.name().to_string(), format!("{value:?}"));
11398 }
11399 }
11400
11401 fn health_response(corr: u64, status: HealthStatus) -> Frame {
11402 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
11403 status,
11404 detail: Some("warming".to_string()),
11405 metrics: Some(json!({"queue_depth": 3})),
11406 })
11407 .unwrap();
11408 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11409 }
11410
11411 fn route_bind_ack(corr: u64) -> Frame {
11412 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
11413 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11414 }
11415
11416 fn unique_project_root(label: &str) -> ScratchDir {
11417 ScratchDir::new(label)
11418 }
11419
11420 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
11421 match parse_route_poll(frame) {
11422 ClientControlResponse::RoutePoll {
11423 status: None,
11424 live: Some(live),
11425 ..
11426 } => assert_eq!(live, expected_live),
11427 other => panic!("unexpected route.poll response: {other:?}"),
11428 }
11429 }
11430
11431 fn bind_liveness_route(
11432 registry: &Registry,
11433 forwarding: &ForwardingTable,
11434 module_id: &str,
11435 ) -> (RouteCtx, u16, u32) {
11436 let module_connection = ConnectionId::new(101);
11437 let client_connection = ConnectionId::new(202);
11438 let registration = registry
11439 .register_with_control_ops(
11440 manifest(module_id, PROTOCOL_VERSION),
11441 PROTOCOL_VERSION,
11442 module_connection,
11443 module_baseline_control_ops(),
11444 )
11445 .unwrap();
11446 let (module_tx, _module_rx) = mpsc::channel(8);
11447 let endpoint = forwarding
11448 .register_module_connection(
11449 module_connection,
11450 module_id.to_string(),
11451 PROTOCOL_VERSION,
11452 manifest_concurrency(®istration.manifest),
11453 FrameSink::new(module_tx),
11454 )
11455 .unwrap();
11456 let (client_ctx, _client_rx) = route_ctx(client_connection);
11457 let pending = forwarding
11458 .begin_route_bind_relay_for_test(
11459 client_connection,
11460 client_ctx.egress.clone(),
11461 1,
11462 module_id,
11463 )
11464 .unwrap();
11465 assert_eq!(pending.endpoint, endpoint);
11466 let route_channel = pending.client_channel;
11467 let route_epoch = pending.client_epoch;
11468 forwarding
11469 .complete_pending_relay(
11470 module_connection,
11471 pending.corr,
11472 RouteBindRelayOutcome::Accepted,
11473 )
11474 .unwrap();
11475 (client_ctx, route_channel, route_epoch)
11476 }
11477
11478 struct FakeProcessLiveness {
11479 live: Option<bool>,
11480 }
11481
11482 impl ModuleProcessLiveness for FakeProcessLiveness {
11483 fn process_live(&self, _module_id: &str) -> Option<bool> {
11484 self.live
11485 }
11486 }
11487
11488 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11489 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
11490 {
11491 let registry = Arc::new(Registry::default());
11492 let supervisor_handle = SupervisorHandle::new();
11493 let supervisor = Supervisor::new_for_test(
11494 Arc::clone(®istry),
11495 RestartPolicy::new(1, Duration::from_millis(10)),
11496 )
11497 .with_handle(supervisor_handle.clone());
11498 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
11499 let module = supervisor
11500 .spawn(ModuleSpec {
11501 carry_containment: false,
11502 module_id: "stderr-tail-wire".to_string(),
11503 program: fake_aft_stub_path(),
11504 args: Vec::new(),
11505 env: vec![
11506 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
11507 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
11508 ],
11509 reserved: false,
11510 reserved_prefixes: Vec::new(),
11511 protocol: ModuleProtocol::Subc,
11512 overlap: Default::default(),
11513 })
11514 .unwrap();
11515
11516 let deadline = Instant::now() + Duration::from_secs(5);
11517 loop {
11518 let tail = module.stderr_tail(None, None);
11519 if tail
11520 .entries
11521 .iter()
11522 .any(|entry| matches!(entry, TailEntry::ProcessStart))
11523 && tail.entries.iter().any(|entry| {
11524 matches!(
11525 entry,
11526 TailEntry::Line {
11527 truncated: true,
11528 ..
11529 }
11530 )
11531 })
11532 {
11533 break;
11534 }
11535 assert!(
11536 Instant::now() < deadline,
11537 "module did not produce a truncated line and restart boundary: {tail:?}"
11538 );
11539 sleep(Duration::from_millis(10)).await;
11540 }
11541
11542 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11543 let request = ClientControlRequest::SupervisorStderrTail {
11544 module_id: "stderr-tail-wire".to_string(),
11545 max_lines: None,
11546 max_bytes: None,
11547 };
11548 let frame = Frame::build(
11549 FrameType::Request,
11550 control_flags(),
11551 0,
11552 0,
11553 1,
11554 serde_json::to_vec(&request).unwrap(),
11555 )
11556 .unwrap();
11557 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11558 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11559 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
11560 serde_json::from_slice(&responses[0].body).unwrap()
11561 else {
11562 panic!("expected supervisor.stderr_tail response");
11563 };
11564
11565 assert!(
11566 tail.entries
11567 .iter()
11568 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
11569 "the control response lost the restart boundary"
11570 );
11571 let Some(StderrTailEntry::Line {
11572 text,
11573 truncated,
11574 at_ms,
11575 }) = tail.entries.iter().find(|entry| {
11576 matches!(
11577 entry,
11578 StderrTailEntry::Line {
11579 truncated: true,
11580 ..
11581 }
11582 )
11583 })
11584 else {
11585 panic!("the control response lost the truncated line");
11586 };
11587 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
11588 assert!(*truncated);
11589 assert!(
11590 at_ms.is_some(),
11591 "the control response lost the line's capture time"
11592 );
11593 }
11594
11595 #[tokio::test(flavor = "current_thread")]
11600 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
11601 let dir = ScratchDir::new("terminals-off-worker");
11602 let journal_path = dir.join("terminals.jsonl");
11603 let registry = Arc::new(Registry::default());
11604 let supervisor_handle = SupervisorHandle::new();
11605 let supervisor =
11606 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11607 .with_handle(supervisor_handle.clone())
11608 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
11609 let module = supervisor
11610 .spawn(ModuleSpec {
11611 carry_containment: false,
11612 module_id: "terminal-off-worker".to_string(),
11613 program: fake_aft_stub_path(),
11614 args: Vec::new(),
11615 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11616 reserved: false,
11617 reserved_prefixes: Vec::new(),
11618 protocol: ModuleProtocol::Subc,
11619 overlap: Default::default(),
11620 })
11621 .unwrap();
11622 let deadline = Instant::now() + Duration::from_secs(5);
11623 while module.terminal_history().entries.len() != 2 {
11624 assert!(Instant::now() < deadline, "module did not record two exits");
11625 sleep(Duration::from_millis(10)).await;
11626 }
11627
11628 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
11629 let handler =
11630 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
11631 let frame = Frame::build(
11632 FrameType::Request,
11633 control_flags(),
11634 0,
11635 0,
11636 1,
11637 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
11638 module_id: "terminal-off-worker".to_string(),
11639 })
11640 .unwrap(),
11641 )
11642 .unwrap();
11643 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11644 let spawned_at = std::time::Instant::now();
11645 let read = tokio::spawn({
11646 let handler = Arc::clone(&handler);
11647 async move { handler.handle_control_frame(&ctx, frame).await }
11648 });
11649 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
11652 .await
11653 .unwrap()
11654 .expect("the history read reached its pause");
11655 let elapsed = spawned_at.elapsed();
11656 assert!(
11657 elapsed < Duration::from_secs(2) && !read.is_finished(),
11658 "this task could not run while the history read was paused \
11659 (resumed after {elapsed:?}, read finished: {})",
11660 read.is_finished()
11661 );
11662
11663 drop(release);
11664 let responses = read.await.unwrap().unwrap();
11665 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11666 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
11667 panic!("expected supervisor.terminals response");
11668 };
11669 assert_eq!(terminals.entries.len(), 2);
11670 assert_eq!(terminals.journal_skipped_lines, 0);
11671 assert_eq!(terminals.journal_read_errors, 0);
11672 }
11673
11674 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11675 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
11676 let registry = Arc::new(Registry::default());
11677 let supervisor_handle = SupervisorHandle::new();
11678 let supervisor =
11679 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11680 .with_handle(supervisor_handle.clone());
11681 let module = supervisor
11682 .spawn(ModuleSpec {
11683 carry_containment: false,
11684 module_id: "terminal-golden".to_string(),
11685 program: fake_aft_stub_path(),
11686 args: Vec::new(),
11687 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11688 reserved: false,
11689 reserved_prefixes: Vec::new(),
11690 protocol: ModuleProtocol::Subc,
11691 overlap: Default::default(),
11692 })
11693 .unwrap();
11694
11695 let deadline = Instant::now() + Duration::from_secs(5);
11696 while module.terminal_history().entries.len() != 2 {
11697 assert!(
11698 Instant::now() < deadline,
11699 "module did not retain two terminal exits: {:?}",
11700 module.terminal_history()
11701 );
11702 sleep(Duration::from_millis(10)).await;
11703 }
11704
11705 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11706 let request = ClientControlRequest::SupervisorTerminals {
11707 module_id: "terminal-golden".to_string(),
11708 };
11709 let frame = Frame::build(
11710 FrameType::Request,
11711 control_flags(),
11712 0,
11713 0,
11714 1,
11715 serde_json::to_vec(&request).unwrap(),
11716 )
11717 .unwrap();
11718 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11719 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11720 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11721 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
11722 panic!("expected supervisor.terminals response");
11723 };
11724 assert_eq!(terminals.entries.len(), 2);
11725 assert_eq!(terminals.dropped, 0);
11726
11727 let mut rendered = serde_json::to_value(response).unwrap();
11728 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
11731 for (index, entry) in rendered["entries"]
11732 .as_array_mut()
11733 .expect("terminal response entries array")
11734 .iter_mut()
11735 .enumerate()
11736 {
11737 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
11738 }
11739
11740 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11741 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
11742 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
11743 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11744 std::fs::write(&golden_path, &serialized).unwrap();
11745 }
11746 let expected: Value =
11747 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
11748 assert_eq!(rendered, expected);
11749 }
11750
11751 #[test]
11752 fn hello_registers_manifest_and_returns_ack() {
11753 let registry = Arc::new(Registry::default());
11754 let handler = ControlHandler::new(Arc::clone(®istry));
11755 let conn = ConnectionId::new(1);
11756
11757 let responses = handler
11758 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
11759 .unwrap();
11760
11761 assert_eq!(responses.len(), 1);
11762 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11763 assert_eq!(responses[0].header.channel, 0);
11764 assert_eq!(responses[0].header.corr, 7);
11765 let ack = parse_ack(&responses[0]);
11766 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11767 assert!(ack
11768 .subc_capabilities
11769 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
11770 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
11771 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
11772 assert!(ack
11773 .subc_ops
11774 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
11775 assert!(ack
11776 .subc_ops
11777 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
11778 assert!(ack
11779 .subc_ops
11780 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
11781
11782 let registration = registry.get_module("aft").unwrap().unwrap();
11783 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
11784 assert_eq!(registration.state, ChannelState::Active);
11785 assert_eq!(registration.connection_id, conn);
11786 assert_eq!(registration.control_ops, module_baseline_control_ops());
11787 assert!(registration.manifest.display_name.is_none());
11788 }
11789
11790 #[test]
11791 fn hello_display_name_refuses_invalid_manifest_by_field() {
11792 let registry = Arc::new(Registry::default());
11793 let handler = ControlHandler::new(Arc::clone(®istry));
11794 for (index, invalid) in [
11795 json!(""),
11796 json!("a".repeat(41)),
11797 json!(" Routing"),
11798 json!("a\nb"),
11799 json!("a\u{202e}b"),
11800 json!(7),
11801 ]
11802 .into_iter()
11803 .enumerate()
11804 {
11805 let module_id = format!("bad-display-name-{index}");
11806 let mut body = serde_json::to_value(ModuleHelloBody {
11807 manifest: manifest(&module_id, PROTOCOL_VERSION),
11808 protocol_ver: PROTOCOL_VERSION,
11809 control_ops: None,
11810 launch_nonce: None,
11811 })
11812 .unwrap();
11813 body["manifest"]["display_name"] = invalid;
11814 let hello = Frame::build(
11815 FrameType::Hello,
11816 control_flags(),
11817 0,
11818 0,
11819 7,
11820 serde_json::to_vec(&body).unwrap(),
11821 )
11822 .unwrap();
11823 let response = handler
11824 .handle_control(ConnectionId::new(100 + index as u64), hello)
11825 .unwrap();
11826 assert_eq!(response[0].header.ty, FrameType::Error);
11827 let error = parse_error(&response[0]);
11828 assert_eq!(error["code"], "invalid_manifest");
11829 assert!(error["message"].as_str().unwrap().contains("display_name"));
11830 assert!(registry.get_module(&module_id).unwrap().is_none());
11831 }
11832 }
11833
11834 #[test]
11835 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
11836 let invalid_identifiers = [
11837 ("case_change", "credentials-Provider/v1"),
11838 ("leading_zero", "credentials-provider/v01"),
11839 ("trailing_hyphen", "credentials-provider-/v1"),
11840 ("consecutive_hyphens", "credentials--provider/v1"),
11841 ("uppercase", "Credentials-provider/v1"),
11842 ("missing_v", "credentials-provider/1"),
11843 ("whitespace", "credentials provider/v1"),
11844 ("zero_version", "credentials-provider/v0"),
11845 ("out_of_range_version", "credentials-provider/v4294967296"),
11846 (
11847 "overlength_name",
11848 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
11849 ),
11850 ];
11851 let mut cases = invalid_identifiers
11852 .into_iter()
11853 .map(|(name, identifier)| {
11854 (
11855 format!("identifier_{name}"),
11856 "capabilities.provides[0]".to_string(),
11857 identifier.to_string(),
11858 json!({ "provides": [identifier] }),
11859 None,
11860 )
11861 })
11862 .collect::<Vec<_>>();
11863 cases.extend([
11864 (
11865 "unknown_need".to_string(),
11866 "capabilities.requires[0].need".to_string(),
11867 "deferred".to_string(),
11868 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
11869 None,
11870 ),
11871 (
11872 "duplicate_provides".to_string(),
11873 "capabilities.provides[1]".to_string(),
11874 "credentials-provider/v1".to_string(),
11875 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
11876 None,
11877 ),
11878 (
11879 "duplicate_must_never_reach".to_string(),
11880 "capabilities.must_never_reach[1]".to_string(),
11881 "credentials-provider/v1".to_string(),
11882 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
11883 None,
11884 ),
11885 (
11886 "duplicate_requires_same_need".to_string(),
11887 "capabilities.requires[1]".to_string(),
11888 "credentials-provider/v1".to_string(),
11889 json!({ "requires": [
11890 { "capability": "credentials-provider/v1", "need": "required" },
11891 { "capability": "credentials-provider/v1", "need": "required" }
11892 ] }),
11893 None,
11894 ),
11895 (
11896 "duplicate_requires_conflicting_need".to_string(),
11897 "capabilities.requires[1]".to_string(),
11898 "credentials-provider/v1".to_string(),
11899 json!({ "requires": [
11900 { "capability": "credentials-provider/v1", "need": "required" },
11901 { "capability": "credentials-provider/v1", "need": "optional" }
11902 ] }),
11903 None,
11904 ),
11905 (
11906 "capabilities_root_pointer".to_string(),
11907 "runtime_computed[0]".to_string(),
11908 "/capabilities".to_string(),
11909 json!({}),
11910 Some(json!(["/capabilities"])),
11911 ),
11912 (
11913 "capabilities_descendant_pointer".to_string(),
11914 "runtime_computed[0]".to_string(),
11915 "/capabilities/provides".to_string(),
11916 json!({}),
11917 Some(json!(["/capabilities/provides"])),
11918 ),
11919 (
11920 "malformed_pointer_without_leading_slash".to_string(),
11921 "runtime_computed[0]".to_string(),
11922 "capabilities".to_string(),
11923 json!({}),
11924 Some(json!(["capabilities"])),
11925 ),
11926 (
11927 "malformed_pointer_escape".to_string(),
11928 "runtime_computed[0]".to_string(),
11929 "/roles/~2/tools".to_string(),
11930 json!({}),
11931 Some(json!(["/roles/~2/tools"])),
11932 ),
11933 (
11934 "unknown_capabilities_field".to_string(),
11935 "capabilities.future".to_string(),
11936 "<array>".to_string(),
11937 json!({ "future": [] }),
11938 None,
11939 ),
11940 ]);
11941
11942 for (index, (name, field, value, capabilities, runtime_computed)) in
11943 cases.into_iter().enumerate()
11944 {
11945 let registry = Arc::new(Registry::default());
11946 let handler = ControlHandler::new(Arc::clone(®istry));
11947 let response = handler
11948 .handle_control(
11949 ConnectionId::new((index + 1) as u64),
11950 capability_grammar_hello_frame(
11951 capabilities,
11952 runtime_computed,
11953 index as u64 + 1,
11954 ),
11955 )
11956 .expect("invalid HELLO returns a refusal");
11957
11958 assert_eq!(response.len(), 1, "{name} must emit one refusal");
11959 let error = parse_error(&response[0]);
11960 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
11961 let message = error["message"]
11962 .as_str()
11963 .expect("error message is a string");
11964 assert!(
11965 message.contains(&field),
11966 "{name}: field missing from {message}"
11967 );
11968 assert!(
11969 message.contains(&value),
11970 "{name}: value missing from {message}"
11971 );
11972 assert_eq!(
11973 registry
11974 .active_registration_count()
11975 .expect("registry reads"),
11976 0,
11977 "{name}: refused HELLO must not create a catalog entry"
11978 );
11979 }
11980 }
11981
11982 #[test]
11983 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
11984 let registry = Arc::new(Registry::default());
11985 let handler = ControlHandler::new(Arc::clone(®istry));
11986 let capabilities = json!({
11987 "provides": ["credentials-provider/v1"],
11988 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
11989 "must_never_reach": ["federation-transport/v1"]
11990 });
11991 let response = handler
11992 .handle_control(
11993 ConnectionId::new(99),
11994 capability_grammar_hello_frame(
11995 capabilities.clone(),
11996 Some(json!(["/roles/0/tools"])),
11997 99,
11998 ),
11999 )
12000 .expect("valid HELLO registers");
12001 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12002
12003 let request = Frame::build(
12004 FrameType::Request,
12005 control_flags(),
12006 0,
12007 0,
12008 100,
12009 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12010 .expect("catalog request serializes"),
12011 )
12012 .expect("catalog request frame builds");
12013 let response = handler
12014 .handle_catalog_list(request, None)
12015 .expect("catalog list succeeds");
12016 let ClientControlResponse::CatalogList { modules, .. } =
12017 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
12018 else {
12019 panic!("catalog request must return catalog.list");
12020 };
12021 assert_eq!(modules.len(), 1);
12022 assert_eq!(
12023 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
12024 capabilities
12025 );
12026 }
12027
12028 #[test]
12029 fn catalog_list_mirrors_management_operation_description() {
12030 let registry = Arc::new(Registry::default());
12031 let handler = ControlHandler::new(Arc::clone(®istry));
12032 let description = "List managed records and return their identifiers and metadata.";
12033 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
12034 manifest.provides = vec![ProviderRole::ManagementSurface {
12035 operations: vec![ManagementOperation {
12036 name: "records.list".to_string(),
12037 kind: ManagementOperationKind::Query,
12038 description: Some(description.to_string()),
12039 }],
12040 config_schema: json!({"type": "object"}),
12041 observability: vec![ObservabilitySurface {
12042 name: "records.stats".to_string(),
12043 kind: ObservabilityKind::Snapshot,
12044 }],
12045 identity_scope: vec![IdentityScope::Project],
12046 concurrency: Concurrency::ModuleManaged,
12047 }];
12048 registry
12049 .register_with_control_ops(
12050 manifest,
12051 PROTOCOL_VERSION,
12052 ConnectionId::new(99),
12053 Vec::new(),
12054 )
12055 .expect("described management manifest registers");
12056
12057 let request = Frame::build(
12058 FrameType::Request,
12059 control_flags(),
12060 0,
12061 0,
12062 100,
12063 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12064 .expect("catalog request serializes"),
12065 )
12066 .expect("catalog request frame builds");
12067 let response = handler
12068 .handle_catalog_list(request, None)
12069 .expect("catalog list succeeds");
12070 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
12071 assert_eq!(
12072 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
12073 "catalog.list must preserve the declared operation description verbatim"
12074 );
12075 }
12076
12077 #[test]
12078 fn catalog_list_mirrors_registered_operation_delivery() {
12079 let registry = Arc::new(Registry::default());
12080 let handler = ControlHandler::new(Arc::clone(®istry));
12081 let delivery =
12082 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
12083 let mut body = serde_json::to_value(ModuleHelloBody {
12084 manifest: manifest("watch-management", PROTOCOL_VERSION),
12085 protocol_ver: PROTOCOL_VERSION,
12086 control_ops: None,
12087 launch_nonce: None,
12088 })
12089 .unwrap();
12090 body["manifest"]["provides"] = json!([{
12091 "role": "management_surface",
12092 "operations": [
12093 {"name": "status", "kind": "query"},
12094 {"name": "future", "kind": "query"},
12095 {"name": "reset", "kind": "mutate"}
12096 ],
12097 "config_schema": {}, "observability": [], "identity_scope": []
12098 }]);
12099 body["manifest"]["operation_delivery"] = delivery.clone();
12100 let hello = Frame::build(
12101 FrameType::Hello,
12102 control_flags(),
12103 0,
12104 0,
12105 99,
12106 serde_json::to_vec(&body).unwrap(),
12107 )
12108 .unwrap();
12109 let response = handler
12110 .handle_control(ConnectionId::new(99), hello)
12111 .unwrap();
12112 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12113
12114 let (_, registrations) = registry.list_modules().unwrap();
12115 let registered = ®istrations[0].manifest;
12116 assert!(registered.is_watch_operation("status"));
12117 assert!(!registered.is_watch_operation("future"));
12118 assert!(!registered.is_watch_operation("reset"));
12119 assert!(!registered.is_watch_operation("missing"));
12120
12121 let request = Frame::build(
12122 FrameType::Request,
12123 control_flags(),
12124 0,
12125 0,
12126 100,
12127 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12128 )
12129 .unwrap();
12130 let response = handler.handle_catalog_list(request, None).unwrap();
12131 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
12132 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
12133 let ClientControlResponse::CatalogList { modules, .. } =
12134 serde_json::from_slice(&response[0].body).unwrap()
12135 else {
12136 panic!("expected catalog.list");
12137 };
12138 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
12139 }
12140
12141 #[test]
12142 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
12143 let registry = Arc::new(Registry::default());
12144 let supervisor = SupervisorHandle::new();
12145 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12146 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12147 let descriptor = json!({
12148 "schema": "ckios.richui.manifest/1",
12149 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12150 "bytes": 4096,
12151 "chunks": 4
12152 });
12153 let connection = ConnectionId::new(201);
12154 let hello = handler
12155 .handle_control(
12156 connection,
12157 hello_frame_with_rich_ui(
12158 "vault",
12159 PROTOCOL_VERSION,
12160 201,
12161 Some("the-real-nonce"),
12162 descriptor.clone(),
12163 ),
12164 )
12165 .unwrap();
12166 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12167
12168 let request =
12169 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
12170 let provides = registry
12171 .get_module_by_connection(connection)
12172 .unwrap()
12173 .unwrap()
12174 .manifest
12175 .provides;
12176 handler
12177 .handle_catalog_update(connection, request, provides, None, Some(false))
12178 .unwrap();
12179
12180 let request = Frame::build(
12181 FrameType::Request,
12182 control_flags(),
12183 0,
12184 0,
12185 203,
12186 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12187 )
12188 .unwrap();
12189 let response = handler.handle_catalog_list(request, None).unwrap();
12190 let ClientControlResponse::CatalogList { modules, .. } =
12191 serde_json::from_slice(&response[0].body).unwrap()
12192 else {
12193 panic!("expected catalog.list");
12194 };
12195 assert_eq!(
12196 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
12197 descriptor
12198 );
12199 assert!(modules[0].nonce_verified);
12200 }
12201
12202 fn hello_with_role_declarations(declarations: Value) -> Frame {
12203 let mut body = serde_json::to_value(ModuleHelloBody {
12204 manifest: manifest("role-test", PROTOCOL_VERSION),
12205 protocol_ver: PROTOCOL_VERSION,
12206 control_ops: None,
12207 launch_nonce: None,
12208 })
12209 .unwrap();
12210 body["manifest"]["role_declarations"] = declarations;
12211 Frame::build(
12212 FrameType::Hello,
12213 control_flags(),
12214 0,
12215 0,
12216 301,
12217 serde_json::to_vec(&body).unwrap(),
12218 )
12219 .unwrap()
12220 }
12221
12222 fn update_role_declarations_frame(declarations: Value) -> Frame {
12223 Frame::build(
12224 FrameType::Request,
12225 control_flags(),
12226 0,
12227 0,
12228 303,
12229 serde_json::to_vec(&json!({
12230 "op": "catalog.update_role_declarations",
12231 "role_declarations": declarations,
12232 }))
12233 .unwrap(),
12234 )
12235 .unwrap()
12236 }
12237
12238 #[tokio::test]
12239 async fn role_declarations_update_replaces_then_clears_catalog() {
12240 let registry = Arc::new(Registry::default());
12241 let handler = ControlHandler::new(Arc::clone(®istry));
12242 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12243 let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
12244 hello_via_sink(
12245 &handler,
12246 &ctx,
12247 &mut rx,
12248 hello_with_role_declarations(initial.clone()),
12249 )
12250 .await;
12251 assert_eq!(
12252 role_declarations_catalog(&handler).0["role_declarations"],
12253 initial
12254 );
12255 let before = registry.get_module("role-test").unwrap().unwrap();
12256 for declarations in [
12257 json!({"media-provider/v1": {"engines": ["new"]}}),
12258 json!({}),
12259 ] {
12260 let response = handler
12261 .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
12262 .await
12263 .unwrap();
12264 assert_eq!(response[0].header.ty, FrameType::Response);
12265 assert_eq!(
12266 serde_json::from_slice::<Value>(&response[0].body).unwrap(),
12267 json!({"op": "catalog.update_role_declarations"})
12268 );
12269 let (raw, entry) = role_declarations_catalog(&handler);
12270 assert_eq!(
12271 serde_json::to_value(entry.role_declarations).unwrap(),
12272 declarations
12273 );
12274 if declarations.as_object().unwrap().is_empty() {
12275 assert!(raw.get("role_declarations").is_none());
12276 } else {
12277 assert_eq!(raw["role_declarations"], declarations);
12278 }
12279 let after = registry.get_module("role-test").unwrap().unwrap();
12280 let mut expected = before.manifest.clone();
12281 expected.role_declarations = serde_json::from_value(declarations).unwrap();
12282 assert_eq!(after.manifest, expected);
12283 assert_eq!(after.ready, before.ready);
12284 }
12285 }
12286
12287 #[tokio::test]
12288 async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
12289 let registry = Arc::new(Registry::default());
12290 let handler = ControlHandler::new(registry);
12291 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12292 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12293 hello_via_sink(
12294 &handler,
12295 &ctx,
12296 &mut rx,
12297 hello_with_role_declarations(initial.clone()),
12298 )
12299 .await;
12300 for (invalid, reason) in [
12301 (json!(null), "role_declarations must be an object"),
12302 (json!([]), "role_declarations must be an object"),
12303 (
12304 json!({"bad key": {}}),
12305 "role_declarations key must use capability-name grammar",
12306 ),
12307 (
12308 json!({"media-provider/v1": []}),
12309 "role_declarations values must be objects",
12310 ),
12311 (
12312 json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
12313 "role_declarations exceeds 16384 compact JSON bytes",
12314 ),
12315 ] {
12316 let response = handler
12317 .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
12318 .await
12319 .unwrap();
12320 let error = parse_error(&response[0]);
12321 assert_eq!(error["code"], "invalid_role_declarations");
12322 assert_eq!(error["message"], reason);
12323 assert_eq!(
12324 role_declarations_catalog(&handler).0["role_declarations"],
12325 initial
12326 );
12327 }
12328 }
12329
12330 #[tokio::test]
12331 async fn role_declarations_update_is_owned_by_the_registered_connection() {
12332 let registry = Arc::new(Registry::default());
12333 let handler = ControlHandler::new(Arc::clone(®istry));
12334 let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
12335 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12336 hello_via_sink(
12337 &handler,
12338 &owner,
12339 &mut owner_rx,
12340 hello_with_role_declarations(initial.clone()),
12341 )
12342 .await;
12343 let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
12344 let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
12345 let response = handler
12346 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12347 .await
12348 .unwrap();
12349 assert_eq!(parse_error(&response[0])["code"], "not_registered");
12350 hello_via_sink(
12351 &handler,
12352 &other,
12353 &mut other_rx,
12354 hello_frame("other", PROTOCOL_VERSION, 304),
12355 )
12356 .await;
12357 let response = handler
12358 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12359 .await
12360 .unwrap();
12361 assert_eq!(response[0].header.ty, FrameType::Response);
12362 assert_eq!(
12363 serde_json::to_value(
12364 registry
12365 .get_module("role-test")
12366 .unwrap()
12367 .unwrap()
12368 .manifest
12369 .role_declarations
12370 )
12371 .unwrap(),
12372 initial
12373 );
12374 assert_eq!(
12375 serde_json::to_value(
12376 registry
12377 .get_module("other")
12378 .unwrap()
12379 .unwrap()
12380 .manifest
12381 .role_declarations
12382 )
12383 .unwrap(),
12384 replacement
12385 );
12386 }
12387
12388 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
12389 let request = Frame::build(
12390 FrameType::Request,
12391 control_flags(),
12392 0,
12393 0,
12394 302,
12395 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12396 )
12397 .unwrap();
12398 let response = handler.handle_catalog_list(request, None).unwrap();
12399 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
12400 let ClientControlResponse::CatalogList { mut modules, .. } =
12401 serde_json::from_slice(&response[0].body).unwrap()
12402 else {
12403 panic!("expected catalog.list");
12404 };
12405 (raw["modules"][0].clone(), modules.remove(0))
12406 }
12407
12408 #[test]
12409 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
12410 let registry = Arc::new(Registry::default());
12411 let handler = ControlHandler::new(Arc::clone(®istry));
12412 let declarations = json!({
12413 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
12414 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
12415 });
12416 let connection = ConnectionId::new(301);
12417 let response = handler
12418 .handle_control(
12419 connection,
12420 hello_with_role_declarations(declarations.clone()),
12421 )
12422 .unwrap();
12423 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12424 let registered = registry
12425 .get_module_by_connection(connection)
12426 .unwrap()
12427 .unwrap()
12428 .manifest;
12429 assert_eq!(
12433 serde_json::to_value(®istered.role_declarations).unwrap(),
12434 declarations
12435 );
12436 let (raw, entry) = role_declarations_catalog(&handler);
12437 assert_eq!(raw["role_declarations"], declarations);
12438 assert_eq!(entry.role_declarations, registered.role_declarations);
12439
12440 let request =
12441 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
12442 handler
12443 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
12444 .unwrap();
12445 let (raw, entry) = role_declarations_catalog(&handler);
12446 assert_eq!(raw["role_declarations"], declarations);
12447 assert_eq!(entry.role_declarations, registered.role_declarations);
12448 }
12449
12450 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
12451 let registry = Arc::new(Registry::default());
12452 let handler = ControlHandler::new(Arc::clone(®istry));
12453 let capture = EventCapture::default();
12454 let _subscriber =
12455 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12456 let response = handler
12457 .handle_control(
12458 ConnectionId::new(301),
12459 hello_with_role_declarations(declarations),
12460 )
12461 .unwrap();
12462 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12463 assert!(registry
12464 .get_module_by_connection(ConnectionId::new(301))
12465 .unwrap()
12466 .unwrap()
12467 .manifest
12468 .role_declarations
12469 .is_empty());
12470 let (raw, entry) = role_declarations_catalog(&handler);
12471 assert!(raw.get("role_declarations").is_none());
12472 assert!(entry.role_declarations.is_empty());
12473 let warnings: Vec<_> = capture
12474 .events()
12475 .into_iter()
12476 .filter(|event| {
12477 event.fields.get("message").map(String::as_str)
12478 == Some("dropping whole role_declarations map from HELLO")
12479 })
12480 .collect();
12481 assert_eq!(warnings.len(), 1);
12482 assert_eq!(warnings[0].level, tracing::Level::WARN);
12483 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
12484 assert_eq!(warnings[0].fields["reason"], reason);
12485 }
12486
12487 #[test]
12488 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
12489 for value in [
12490 json!(null),
12491 json!(false),
12492 json!(0),
12493 json!("opaque"),
12494 json!([]),
12495 ] {
12496 assert_hello_drops_role_declarations_with_warning(
12497 json!({"routing/v1": {}, "broken/v1": value}),
12498 "role_declarations values must be objects",
12499 );
12500 }
12501 assert_hello_drops_role_declarations_with_warning(
12502 json!([]),
12503 "role_declarations must be an object",
12504 );
12505 }
12506
12507 #[test]
12508 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
12509 assert_hello_drops_role_declarations_with_warning(
12510 json!({"routing/v1": {}, "bad key": {}}),
12511 "role_declarations key must use capability-name grammar",
12512 );
12513 }
12514
12515 #[test]
12516 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
12517 assert_hello_drops_role_declarations_with_warning(
12518 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
12519 "role_declarations exceeds 16384 compact JSON bytes",
12520 );
12521 }
12522
12523 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
12524 let mut body = serde_json::to_value(ModuleHelloBody {
12525 manifest: manifest("flow-ops", PROTOCOL_VERSION),
12526 protocol_ver: PROTOCOL_VERSION,
12527 control_ops: None,
12528 launch_nonce: None,
12529 })
12530 .unwrap();
12531 body["manifest"]["provides"] = json!([{
12532 "role": "management_surface",
12533 "operations": [
12534 {"name": "diagnostics.findings", "kind": "query"},
12535 {"name": "reset", "kind": "mutate"}
12536 ],
12537 "config_schema": {}, "observability": [], "identity_scope": []
12538 }]);
12539 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
12540 if let Some(flow_operations) = flow_operations {
12541 body["manifest"]["flow_operations"] = flow_operations;
12542 }
12543 Frame::build(
12544 FrameType::Hello,
12545 control_flags(),
12546 0,
12547 0,
12548 401,
12549 serde_json::to_vec(&body).unwrap(),
12550 )
12551 .unwrap()
12552 }
12553
12554 #[test]
12555 fn catalog_list_mirrors_flow_operations_unchanged() {
12556 for (capability, flow_operations) in [
12559 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
12560 ("call-modes/v1", json!(["diagnostics.findings"])),
12561 ("flow-scopes/v1", json!([])),
12562 ] {
12563 let registry = Arc::new(Registry::default());
12564 let handler = ControlHandler::new(Arc::clone(®istry));
12565 let connection = ConnectionId::new(401);
12566 let response = handler
12567 .handle_control(
12568 connection,
12569 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
12570 )
12571 .unwrap();
12572 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12573 let registered = registry
12574 .get_module_by_connection(connection)
12575 .unwrap()
12576 .unwrap()
12577 .manifest;
12578 assert_eq!(
12579 serde_json::to_value(®istered.flow_operations).unwrap(),
12580 flow_operations
12581 );
12582 let (raw, entry) = role_declarations_catalog(&handler);
12583 assert_eq!(raw["flow_operations"], flow_operations);
12584 assert_eq!(entry.flow_operations, registered.flow_operations);
12585 }
12586
12587 let registry = Arc::new(Registry::default());
12588 let handler = ControlHandler::new(Arc::clone(®istry));
12589 let response = handler
12590 .handle_control(
12591 ConnectionId::new(401),
12592 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
12593 )
12594 .unwrap();
12595 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12596 let (raw, entry) = role_declarations_catalog(&handler);
12597 assert!(raw.get("flow_operations").is_none());
12598 assert_eq!(entry.flow_operations, None);
12599 }
12600
12601 #[test]
12602 fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
12603 let oversized: serde_json::Map<String, Value> = (0..257)
12604 .map(|n| (format!("tool:t{n}"), json!(true)))
12605 .collect();
12606 for (capabilities, map, valid) in [
12607 (
12608 json!(["scope-federated/v1"]),
12609 json!({"tool:read":true,"management:reset":true}),
12610 true,
12611 ),
12612 (json!(["scope-federated/v1"]), json!({}), true),
12613 (json!([]), json!({"tool:read":true}), false),
12614 (
12615 json!(["scope-federated/v1"]),
12616 json!({"tool:valid":true,"tool:bad":false}),
12617 false,
12618 ),
12619 (
12620 json!(["scope-federated/v1"]),
12621 json!({"tool:valid":true,"management:missing":true}),
12622 false,
12623 ),
12624 (
12625 json!(["scope-federated/v1"]),
12626 Value::Object(oversized),
12627 false,
12628 ),
12629 ] {
12630 let capture = EventCapture::default();
12631 let _subscriber = tracing::subscriber::set_default(
12632 tracing_subscriber::registry().with(capture.clone()),
12633 );
12634 let registry = Arc::new(Registry::default());
12635 let handler = ControlHandler::new(Arc::clone(®istry));
12636 let hello = hello_with_flow_operations(capabilities, None);
12637 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12638 body["manifest"]["operation_deferral"] = map.clone();
12639 let hello = Frame::build(
12640 FrameType::Hello,
12641 control_flags(),
12642 0,
12643 0,
12644 401,
12645 serde_json::to_vec(&body).unwrap(),
12646 )
12647 .unwrap();
12648 let response = handler
12649 .handle_control(ConnectionId::new(401), hello)
12650 .unwrap();
12651 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12652 let (raw, entry) = role_declarations_catalog(&handler);
12653 if valid {
12654 assert_eq!(raw["operation_deferral"], map);
12655 assert_eq!(
12656 serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
12657 map
12658 );
12659 } else {
12660 assert!(entry.operation_deferral.is_none());
12661 assert!(raw.get("operation_deferral").is_none());
12662 assert_eq!(
12663 capture
12664 .events()
12665 .iter()
12666 .filter(|event| event.fields.get("message").map(String::as_str)
12667 == Some("dropping operation_deferral from HELLO"))
12668 .count(),
12669 1
12670 );
12671 }
12672 }
12673 }
12674
12675 fn assert_hello_drops_flow_operations_with_warning(
12676 capabilities: Value,
12677 flow_operations: Value,
12678 reason: &str,
12679 ) {
12680 let registry = Arc::new(Registry::default());
12681 let handler = ControlHandler::new(Arc::clone(®istry));
12682 let capture = EventCapture::default();
12683 let _subscriber =
12684 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12685 let response = handler
12686 .handle_control(
12687 ConnectionId::new(401),
12688 hello_with_flow_operations(capabilities, Some(flow_operations)),
12689 )
12690 .unwrap();
12691 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12692 let registered = registry
12693 .get_module_by_connection(ConnectionId::new(401))
12694 .unwrap()
12695 .expect("the module still registers")
12696 .manifest;
12697 assert_eq!(registered.module_id, "flow-ops");
12698 assert_eq!(registered.flow_operations, None);
12699 let (raw, entry) = role_declarations_catalog(&handler);
12700 assert_eq!(raw["module_id"], "flow-ops");
12701 assert!(raw.get("flow_operations").is_none());
12702 assert_eq!(entry.flow_operations, None);
12703 let warnings: Vec<_> = capture
12704 .events()
12705 .into_iter()
12706 .filter(|event| {
12707 event.fields.get("message").map(String::as_str)
12708 == Some("dropping flow_operations from HELLO")
12709 })
12710 .collect();
12711 assert_eq!(warnings.len(), 1);
12712 assert_eq!(warnings[0].level, tracing::Level::WARN);
12713 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
12714 assert_eq!(warnings[0].fields["reason"], reason);
12715 }
12716
12717 #[test]
12718 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
12719 assert_hello_drops_flow_operations_with_warning(
12720 json!(["flow-scopes/v1"]),
12721 json!(["diagnostics.findings", "missing"]),
12722 "flow_operations names 'missing', which is not a declared management operation",
12723 );
12724 }
12725
12726 #[test]
12727 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
12728 assert_hello_drops_flow_operations_with_warning(
12729 json!([]),
12730 json!(["diagnostics.findings"]),
12731 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12732 );
12733 assert_hello_drops_flow_operations_with_warning(
12735 json!(["agent-run-scopes/v1"]),
12736 json!([]),
12737 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12738 );
12739 }
12740
12741 #[test]
12742 fn hello_malformed_flow_operations_is_dropped_with_warning() {
12743 assert_hello_drops_flow_operations_with_warning(
12744 json!(["flow-scopes/v1"]),
12745 json!(["reset", "reset"]),
12746 "flow_operations names 'reset' more than once",
12747 );
12748 assert_hello_drops_flow_operations_with_warning(
12749 json!(["flow-scopes/v1"]),
12750 json!(vec!["reset"; 257]),
12751 "flow_operations lists 257 names, more than 256",
12752 );
12753 assert_hello_drops_flow_operations_with_warning(
12754 json!(["flow-scopes/v1"]),
12755 json!("reset"),
12756 "flow_operations must be an array of operation names",
12757 );
12758 }
12759
12760 #[test]
12761 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
12762 let registry = Arc::new(Registry::default());
12763 let handler = ControlHandler::new(registry.clone());
12764 let descriptor = json!({
12765 "schema": "ckios.richui.manifest/1",
12766 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12767 "bytes": 128,
12768 "chunks": 2
12769 });
12770 let malformed = [
12771 ("schema", json!("")),
12772 ("schema", json!("a".repeat(65))),
12773 ("schema", json!("Bad schema")),
12774 (
12775 "manifest_digest",
12776 json!(format!("sha256:{}", "A".repeat(64))),
12777 ),
12778 ("manifest_digest", json!("sha256:abc")),
12779 ("bytes", json!(0)),
12780 ("bytes", json!(16_777_217)),
12781 ("bytes", json!("128")),
12782 ("chunks", json!(0)),
12783 ("chunks", json!(1025)),
12784 ("chunks", json!("2")),
12785 ];
12786 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
12787 let mut invalid = descriptor.clone();
12788 invalid[field] = invalid_value;
12789 let response = handler
12790 .handle_control(
12791 ConnectionId::new(210 + index as u64),
12792 hello_frame_with_rich_ui(
12793 &format!("bad-rich-ui-{index}"),
12794 PROTOCOL_VERSION,
12795 210 + index as u64,
12796 None,
12797 invalid,
12798 ),
12799 )
12800 .unwrap();
12801 assert_eq!(response[0].header.ty, FrameType::Error);
12802 let error = parse_error(&response[0]);
12803 assert_eq!(error["code"], "invalid_manifest");
12804 assert!(error["message"]
12805 .as_str()
12806 .unwrap()
12807 .contains(&format!("rich_ui.{field}")));
12808 assert!(registry
12809 .get_module(&format!("bad-rich-ui-{index}"))
12810 .unwrap()
12811 .is_none());
12812 }
12813 }
12814
12815 #[test]
12816 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
12817 let registry = Arc::new(Registry::default());
12818 let handler = ControlHandler::new(Arc::clone(®istry));
12819 let connection = ConnectionId::new(219);
12820 registry
12821 .register_with_control_ops(
12822 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
12823 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
12824 "bad schema",
12825 format!("sha256:{}", "a".repeat(64)),
12826 128,
12827 2,
12828 )))
12829 .build(),
12830 PROTOCOL_VERSION,
12831 connection,
12832 Vec::new(),
12833 )
12834 .unwrap();
12835 let request =
12836 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
12837 let response = handler
12838 .handle_catalog_update(connection, request, Vec::new(), None, None)
12839 .unwrap();
12840 assert_eq!(response[0].header.ty, FrameType::Error);
12841 let error = parse_error(&response[0]);
12842 assert_eq!(error["code"], "invalid_manifest");
12843 assert!(error["message"]
12844 .as_str()
12845 .unwrap()
12846 .contains("rich_ui.schema"));
12847 }
12848
12849 #[test]
12850 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
12851 let registry = Arc::new(Registry::default());
12852 let supervisor = SupervisorHandle::new();
12853 supervisor.open_swap("provider", "candidate-nonce".to_string());
12854 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12855 let connection = ConnectionId::new(218);
12856 let hello = handler
12857 .handle_control(
12858 connection,
12859 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
12860 )
12861 .unwrap();
12862 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12863 assert!(registry.list_modules().unwrap().1.is_empty());
12864
12865 registry.promote_candidate("provider").unwrap().unwrap();
12866 let request = Frame::build(
12867 FrameType::Request,
12868 control_flags(),
12869 0,
12870 0,
12871 219,
12872 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12873 )
12874 .unwrap();
12875 let response = handler.handle_catalog_list(request, None).unwrap();
12876 let ClientControlResponse::CatalogList { modules, .. } =
12877 serde_json::from_slice(&response[0].body).unwrap()
12878 else {
12879 panic!("expected catalog.list");
12880 };
12881 assert_eq!(modules.len(), 1);
12882 assert!(modules[0].nonce_verified);
12883 }
12884
12885 #[test]
12886 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
12887 for provenance_json in [
12888 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
12889 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
12890 ] {
12891 let registry = Arc::new(Registry::default());
12892 let handler = ControlHandler::new(registry);
12893 let hello = Frame::build(
12894 FrameType::Hello,
12895 control_flags(),
12896 0,
12897 0,
12898 220,
12899 serde_json::to_vec(&ModuleHelloBody {
12900 manifest: ModuleManifest::builder("built", "0.1.0")
12901 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
12902 .build(),
12903 protocol_ver: PROTOCOL_VERSION,
12904 control_ops: None,
12905 launch_nonce: None,
12906 })
12907 .unwrap(),
12908 )
12909 .unwrap();
12910 assert_eq!(
12911 handler
12912 .handle_control(ConnectionId::new(220), hello)
12913 .unwrap()[0]
12914 .header
12915 .ty,
12916 FrameType::HelloAck
12917 );
12918 let request = Frame::build(
12919 FrameType::Request,
12920 control_flags(),
12921 0,
12922 0,
12923 221,
12924 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12925 )
12926 .unwrap();
12927 let response = handler.handle_catalog_list(request, None).unwrap();
12928 let ClientControlResponse::CatalogList { modules, .. } =
12929 serde_json::from_slice(&response[0].body).unwrap()
12930 else {
12931 panic!("expected catalog.list");
12932 };
12933 assert_eq!(modules.len(), 1);
12934 assert_eq!(
12935 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
12936 format!(
12937 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
12938 )
12939 .as_bytes()
12940 );
12941 }
12942 }
12943
12944 #[test]
12945 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
12946 let registry = Arc::new(Registry::default());
12947 let handler = ControlHandler::new(Arc::clone(®istry));
12948 let hello = Frame::build(
12949 FrameType::Hello,
12950 control_flags(),
12951 0,
12952 0,
12953 220,
12954 serde_json::to_vec(&ModuleHelloBody {
12955 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
12956 protocol_ver: PROTOCOL_VERSION,
12957 control_ops: None,
12958 launch_nonce: Some("unverified".to_string()),
12959 })
12960 .unwrap(),
12961 )
12962 .unwrap();
12963 assert_eq!(
12964 handler
12965 .handle_control(ConnectionId::new(220), hello)
12966 .unwrap()[0]
12967 .header
12968 .ty,
12969 FrameType::HelloAck
12970 );
12971 let request = Frame::build(
12972 FrameType::Request,
12973 control_flags(),
12974 0,
12975 0,
12976 221,
12977 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12978 )
12979 .unwrap();
12980 let response = handler.handle_catalog_list(request, None).unwrap();
12981 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
12982 assert!(wire["modules"][0].get("provenance").is_none());
12983 let ClientControlResponse::CatalogList { modules, .. } =
12984 serde_json::from_slice(&response[0].body).unwrap()
12985 else {
12986 panic!("expected catalog.list");
12987 };
12988 assert_eq!(
12989 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
12990 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
12991 );
12992 assert!(!modules[0].nonce_verified);
12993 assert!(modules[0].provenance.is_none());
12994 }
12995
12996 fn catalog_legacy_projection(mut entry: CatalogEntry) -> CatalogEntry {
12998 assert_eq!(entry.first_party, Some(false));
12999 entry.incarnation = None;
13000 entry.spawn = None;
13001 entry.first_party = None;
13002 entry.origin = None;
13003 entry.requested_declarations_digest = None;
13004 entry.approved_declarations_digest = None;
13005 entry
13006 }
13007
13008 #[test]
13009 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
13010 let registry = Arc::new(Registry::default());
13011 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13012 [("vault".to_string(), true), ("squatter".to_string(), true)],
13013 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13014 );
13015 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
13016 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13017 provides: vec!["credentials-provider/v1".to_string()],
13018 requires: Vec::new(),
13019 must_never_reach: Vec::new(),
13020 });
13021 let frame = Frame::build(
13022 FrameType::Hello,
13023 control_flags(),
13024 0,
13025 0,
13026 77,
13027 serde_json::to_vec(&ModuleHelloBody {
13028 manifest: squatter,
13029 protocol_ver: PROTOCOL_VERSION,
13030 control_ops: None,
13031 launch_nonce: None,
13032 })
13033 .expect("HELLO serializes"),
13034 )
13035 .expect("HELLO frame builds");
13036 let response = handler
13037 .handle_control(ConnectionId::new(77), frame)
13038 .expect("reserved claim receives a typed refusal");
13039 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13040 assert_eq!(
13041 registry
13042 .active_registration_count()
13043 .expect("registry reads"),
13044 0,
13045 "a reserved capability refusal must not leave a catalog entry"
13046 );
13047 }
13048
13049 #[test]
13050 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
13051 for settlement in ["timeout", "inconclusive", "drop"] {
13052 let breakers = RouteBindBreakers::default();
13053 let RouteBindAdmission::Admitted {
13054 guard: mut old,
13055 probe: false,
13056 } = breakers.admit("prov")
13057 else {
13058 panic!("ordinary relay admitted")
13059 };
13060 let RouteBindAdmission::Admitted {
13061 guard: mut opener, ..
13062 } = breakers.admit("prov")
13063 else {
13064 panic!("second relay admitted")
13065 };
13066 assert!(
13067 !opener
13068 .record_timeout(1, Duration::ZERO)
13069 .unwrap()
13070 .reopened_after_probe
13071 );
13072 let RouteBindAdmission::Admitted {
13073 guard: mut probe,
13074 probe: true,
13075 } = breakers.admit("prov")
13076 else {
13077 panic!("one cooldown probe admitted")
13078 };
13079 match settlement {
13080 "timeout" => assert!(
13081 !old.record_timeout(1, Duration::ZERO)
13082 .unwrap()
13083 .reopened_after_probe
13084 ),
13085 "inconclusive" => old.record_inconclusive(),
13086 "drop" => drop(old),
13087 _ => unreachable!(),
13088 }
13089 assert!(
13090 matches!(
13091 breakers.admit("prov"),
13092 RouteBindAdmission::Refused {
13093 probe_in_flight: true,
13094 ..
13095 }
13096 ),
13097 "{settlement} of a pre-open relay cannot release the real probe"
13098 );
13099 assert!(
13100 probe
13101 .record_timeout(1, Duration::ZERO)
13102 .unwrap()
13103 .reopened_after_probe
13104 );
13105 assert!(matches!(
13106 breakers.admit("prov"),
13107 RouteBindAdmission::Admitted { probe: true, .. }
13108 ));
13109 }
13110 let breakers = RouteBindBreakers::default();
13111 let admit = || match breakers.admit("prov") {
13112 RouteBindAdmission::Admitted { guard, .. } => guard,
13113 _ => panic!("relay admitted"),
13114 };
13115 admit().record_timeout(1, Duration::ZERO);
13116 let mut old_probe = admit();
13117 breakers.reset_for_new_module_connection("prov");
13118 admit().record_timeout(1, Duration::ZERO);
13119 let _new_probe = admit();
13120 old_probe.record_inconclusive();
13121 assert!(matches!(
13122 breakers.admit("prov"),
13123 RouteBindAdmission::Refused {
13124 probe_in_flight: true,
13125 ..
13126 }
13127 ));
13128 }
13129
13130 #[tokio::test]
13131 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
13132 for candidate in [false, true] {
13133 let registry = Arc::new(Registry::default());
13134 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13135 [("vault".to_string(), true), ("squatter".to_string(), true)],
13136 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13137 );
13138 let conn = ConnectionId::new(77);
13139 let (ctx, mut rx) = route_ctx(conn);
13140 let initial = capability_manifest("squatter", &[], &[]);
13141 if candidate {
13142 registry
13143 .register_candidate_with_control_ops(
13144 initial.clone(),
13145 PROTOCOL_VERSION,
13146 conn,
13147 module_baseline_control_ops(),
13148 )
13149 .unwrap();
13150 handler
13151 .forwarding
13152 .register_candidate_module_connection(
13153 conn,
13154 "squatter".to_string(),
13155 PROTOCOL_VERSION,
13156 manifest_concurrency(&initial),
13157 ctx.egress.clone(),
13158 )
13159 .unwrap();
13160 } else {
13161 hello_via_sink(
13162 &handler,
13163 &ctx,
13164 &mut rx,
13165 hello_frame_with_manifest(initial.clone(), 1),
13166 )
13167 .await;
13168 }
13169 let response = handler
13170 .handle_control_frame(
13171 &ctx,
13172 catalog_update_with_capabilities_frame(
13173 2,
13174 capability_manifest("squatter", &["credentials-provider/v1"], &[])
13175 .capabilities
13176 .unwrap(),
13177 ),
13178 )
13179 .await
13180 .unwrap();
13181 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13182 assert_eq!(
13183 registry
13184 .get_module_by_connection(conn)
13185 .unwrap()
13186 .unwrap()
13187 .manifest,
13188 initial
13189 );
13190 }
13191 }
13192
13193 #[test]
13194 #[cfg(not(feature = "dev-desktop-approval"))]
13195 fn server_describe_default_build_advertises_no_config_capability() {
13196 let assert_default = |handler: &ControlHandler| {
13197 let request = Frame::build(
13198 FrameType::Request,
13199 control_flags(),
13200 0,
13201 0,
13202 79,
13203 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13204 )
13205 .unwrap();
13206 let responses = handler.handle_server_describe(request).unwrap();
13207 assert_eq!(responses.len(), 1);
13208 let response: Value = serde_json::from_slice(&responses[0].body).unwrap();
13209 assert_eq!(response["op"], "server.describe");
13210 let capabilities = response["capabilities"]
13211 .as_array()
13212 .expect("capability array");
13213 assert!(
13214 !capabilities.is_empty(),
13215 "exercise an actual describe response, not an empty fixture"
13216 );
13217 assert!(
13218 capabilities
13219 .iter()
13220 .all(|capability| !capability.as_str().unwrap().starts_with("config.")),
13221 "a default build must not announce configuration authority: {capabilities:?}"
13222 );
13223 };
13224 assert_default(&ControlHandler::new(Arc::new(Registry::default())));
13225 #[cfg(unix)]
13226 {
13227 let (_rig, _owner, _supervisor, handler, _spec) =
13231 crate::config_gate::runtime::RuntimeOwner::unactivated_supervisor_fixture_for_test(
13232 );
13233 assert_default(&handler);
13234 }
13235 }
13236
13237 #[test]
13238 fn server_describe_surfaces_required_capability_verdict_fields() {
13239 let registry = Arc::new(Registry::default());
13240 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13241 [
13242 ("consumer".to_string(), true),
13243 ("provider".to_string(), false),
13244 ],
13245 BTreeMap::new(),
13246 );
13247 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13248 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13249 provides: Vec::new(),
13250 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13251 capability: "credentials-provider/v1".to_string(),
13252 need: subc_protocol::manifest::CapabilityNeed::Required,
13253 }],
13254 must_never_reach: Vec::new(),
13255 });
13256 let hello = Frame::build(
13257 FrameType::Hello,
13258 control_flags(),
13259 0,
13260 0,
13261 78,
13262 serde_json::to_vec(&ModuleHelloBody {
13263 manifest: consumer,
13264 protocol_ver: PROTOCOL_VERSION,
13265 control_ops: None,
13266 launch_nonce: None,
13267 })
13268 .expect("HELLO serializes"),
13269 )
13270 .expect("HELLO frame builds");
13271 handler
13272 .handle_control(ConnectionId::new(78), hello)
13273 .expect("consumer registers");
13274 let describe = Frame::build(
13275 FrameType::Request,
13276 control_flags(),
13277 0,
13278 0,
13279 79,
13280 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
13281 .expect("request serializes"),
13282 )
13283 .expect("describe frame builds");
13284 let response = handler
13285 .handle_server_describe(describe)
13286 .expect("server.describe succeeds");
13287 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
13288 let requirement = &rendered["capability_requirements"][0];
13289 assert_eq!(requirement["consumer"], "consumer");
13290 assert_eq!(requirement["verdict"], "never_provided");
13291 assert_eq!(requirement["episode_seq"], 1);
13292 assert_eq!(requirement["config_satisfiable"], false);
13293 assert_eq!(requirement["runtime_available"], false);
13294 assert!(requirement["detail"]
13295 .as_str()
13296 .expect("detail string")
13297 .contains("credentials-provider/v1"));
13298 }
13299
13300 fn deregister_required_provider(
13306 daemon_shutdown_begun: bool,
13307 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
13308 const CAPABILITY: &str = "project-identity/v1";
13309 let registry = Arc::new(Registry::default());
13310 let daemon_shutdown = DaemonShutdownFlag::default();
13311 let handler = ControlHandler::new(Arc::clone(®istry))
13312 .with_capability_config(
13313 [
13314 ("consumer".to_string(), true),
13315 ("provider".to_string(), true),
13316 ],
13317 BTreeMap::new(),
13318 )
13319 .with_daemon_shutdown_flag(daemon_shutdown.clone());
13320 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13321 consumer.capabilities = Some(CapabilityDeclarations {
13322 provides: Vec::new(),
13323 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13324 capability: CAPABILITY.to_string(),
13325 need: subc_protocol::manifest::CapabilityNeed::Required,
13326 }],
13327 must_never_reach: Vec::new(),
13328 });
13329 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
13330 let provider_conn = ConnectionId::new(182);
13331 for (conn, manifest, corr) in [
13332 (ConnectionId::new(181), consumer, 181),
13333 (provider_conn, provider, 182),
13334 ] {
13335 let hello = handler
13336 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
13337 .expect("module registers");
13338 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13339 }
13340 assert_eq!(
13341 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13342 Some(CapabilityVerdict::Provided),
13343 "the registered provider must satisfy the requirement first"
13344 );
13345 if daemon_shutdown_begun {
13346 daemon_shutdown.set_for_test();
13347 }
13348
13349 let capture = EventCapture::default();
13350 let _subscriber =
13351 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13352 handler
13353 .cleanup_connection(provider_conn)
13354 .expect("provider deregisters");
13355 let events = capture
13356 .events()
13357 .into_iter()
13358 .filter(|event| {
13359 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
13360 })
13361 .collect();
13362 (
13363 events,
13364 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13365 )
13366 }
13367
13368 #[test]
13369 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
13370 let (events, verdict) = deregister_required_provider(true);
13371 assert_eq!(
13372 verdict,
13373 Some(CapabilityVerdict::NeverProvided),
13374 "shutdown must not change the verdict route.open refusals read"
13375 );
13376 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13377 let event = &events[0];
13378 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
13379 assert_eq!(
13380 event.fields.get("cause").map(String::as_str),
13381 Some("\"daemon_shutdown\""),
13382 "{event:?}"
13383 );
13384 assert_eq!(
13385 event.fields.get("verdict").map(String::as_str),
13386 Some("\"never_provided\""),
13387 "{event:?}"
13388 );
13389 }
13390
13391 #[test]
13392 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
13393 let (events, verdict) = deregister_required_provider(false);
13394 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
13395 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13396 let event = &events[0];
13397 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
13398 assert_eq!(event.fields.get("cause"), None, "{event:?}");
13399 assert_eq!(
13400 event.fields.get("verdict").map(String::as_str),
13401 Some("\"never_provided\""),
13402 "{event:?}"
13403 );
13404 }
13405
13406 #[test]
13407 fn catalog_list_omits_capabilities_for_legacy_manifest() {
13408 let registry = Arc::new(Registry::default());
13409 let handler = ControlHandler::new(Arc::clone(®istry));
13410 let hello = handler
13411 .handle_control(
13412 ConnectionId::new(101),
13413 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
13414 )
13415 .expect("legacy HELLO registers");
13416 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13417
13418 let request = Frame::build(
13419 FrameType::Request,
13420 control_flags(),
13421 0,
13422 0,
13423 102,
13424 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
13425 .expect("catalog request serializes"),
13426 )
13427 .expect("catalog request frame builds");
13428 let response = handler
13429 .handle_catalog_list(request, None)
13430 .expect("catalog list succeeds");
13431 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
13432 assert!(
13433 body["modules"][0].get("capabilities").is_none(),
13434 "legacy manifest must retain an absent capabilities field on catalog.list"
13435 );
13436 }
13437
13438 #[test]
13439 fn hello_ack_omits_storage_when_no_storage_config() {
13440 let registry = Arc::new(Registry::default());
13441 let handler = ControlHandler::new(Arc::clone(®istry));
13442 let responses = handler
13443 .handle_control(
13444 ConnectionId::new(1),
13445 hello_frame("aft", PROTOCOL_VERSION, 7),
13446 )
13447 .unwrap();
13448 let ack = parse_ack(&responses[0]);
13449 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
13450 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
13451 }
13452
13453 #[tokio::test]
13454 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
13455 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
13456 let registry = Arc::new(Registry::default());
13457 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
13458 let responses = handler
13459 .handle_control(
13460 ConnectionId::new(1),
13461 hello_frame("aft", PROTOCOL_VERSION, 7),
13462 )
13463 .unwrap();
13464 let ack = parse_ack(&responses[0]);
13465 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
13466
13467 let described = handler
13468 .handle_control_frame(
13469 &route_ctx(ConnectionId::new(2)).0,
13470 Frame::build(
13471 FrameType::Request,
13472 control_flags(),
13473 0,
13474 0,
13475 9,
13476 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13477 )
13478 .unwrap(),
13479 )
13480 .await
13481 .unwrap();
13482 let ClientControlResponse::ServerDescribe { machine_id, .. } =
13483 serde_json::from_slice(&described[0].body).unwrap()
13484 else {
13485 panic!("server.describe answered with another shape");
13486 };
13487 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
13488 }
13489
13490 #[test]
13491 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
13492 let registry = Arc::new(Registry::default());
13495 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
13496 crate::daemon_config::StorageConfig::Sqlite {
13497 data_home: std::path::PathBuf::from("/data"),
13498 },
13499 ));
13500
13501 let responses = handler
13502 .handle_control(
13503 ConnectionId::new(1),
13504 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
13505 )
13506 .unwrap();
13507 let ack = parse_ack(&responses[0]);
13508 assert_eq!(
13509 ack.storage,
13510 Some(serde_json::json!({
13511 "module_id": "alfonso-routing",
13512 "storage_namespace": "default",
13513 "isolation": { "kind": "module" },
13514 "backend": {
13515 "backend": "sqlite",
13516 "path": "/data/cortexkit/alfonso-routing/store.db"
13517 }
13518 })),
13519 "the delivered descriptor is the module's own sqlite store path"
13520 );
13521 }
13522
13523 #[test]
13524 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
13525 let registry = Arc::new(Registry::default());
13526 let handler = ControlHandler::new(Arc::clone(®istry));
13527 let conn = ConnectionId::new(1);
13528 let responses = handler
13529 .handle_control(
13530 conn,
13531 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13532 )
13533 .unwrap();
13534 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
13535 let registration = registry.get_module("aft").unwrap().unwrap();
13536 assert_eq!(registration.control_ops, module_baseline_control_ops());
13537
13538 let frame =
13539 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
13540 assert!(handler
13541 .guard_module_control_op(&frame, "aft", "route.bind")
13542 .unwrap()
13543 .is_none());
13544 let error = handler
13545 .guard_module_control_op(&frame, "aft", "test.synthetic")
13546 .unwrap()
13547 .expect("synthetic ungranted op should be rejected");
13548 assert_eq!(error.header.ty, FrameType::Error);
13549 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
13550 }
13551
13552 #[test]
13553 fn hello_control_ops_some_adds_optional_grants() {
13554 let registry = Arc::new(Registry::default());
13555 let handler = ControlHandler::new(Arc::clone(®istry));
13556 handler
13557 .handle_control(
13558 ConnectionId::new(1),
13559 hello_frame_with_control_ops(
13560 "aft",
13561 PROTOCOL_VERSION,
13562 7,
13563 Some(vec![
13564 "future.synthetic".to_string(),
13565 "route.bind".to_string(),
13566 ]),
13567 ),
13568 )
13569 .unwrap();
13570 let registration = registry.get_module("aft").unwrap().unwrap();
13571 assert_eq!(
13572 registration.control_ops,
13573 vec![
13574 "route.bind".to_string(),
13575 "route.status".to_string(),
13576 "future.synthetic".to_string(),
13577 ]
13578 );
13579 let frame =
13580 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
13581 assert!(handler
13582 .guard_module_control_op(&frame, "aft", "future.synthetic")
13583 .unwrap()
13584 .is_none());
13585 }
13586
13587 #[tokio::test]
13588 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
13589 let registry = Arc::new(Registry::default());
13590 let forwarding = Arc::new(ForwardingTable::default());
13591 let handler =
13592 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13593 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
13594 hello_via_sink(
13595 &handler,
13596 &module_ctx,
13597 &mut module_rx,
13598 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13599 )
13600 .await;
13601
13602 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
13603 let responses = handler
13604 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
13605 .await
13606 .unwrap();
13607 assert_eq!(responses.len(), 1);
13608 assert_eq!(responses[0].header.ty, FrameType::Error);
13609 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
13610 assert!(module_rx.try_recv().is_err());
13611 }
13612
13613 #[tokio::test]
13614 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
13615 let registry = Arc::new(Registry::default());
13616 let forwarding = Arc::new(ForwardingTable::default());
13617 let handler =
13618 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13619 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
13620 hello_via_sink(
13621 &handler,
13622 &module_ctx,
13623 &mut module_rx,
13624 hello_frame_with_control_ops(
13625 "aft",
13626 PROTOCOL_VERSION,
13627 7,
13628 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13629 ),
13630 )
13631 .await;
13632
13633 let project_root = unique_project_root("demux");
13634 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
13635 let route_handler = handler.clone();
13636 let route_task = tokio::spawn(async move {
13637 route_handler
13638 .handle_control_frame(
13639 &route_client_ctx,
13640 route_open_frame(100, "aft", project_root),
13641 )
13642 .await
13643 .unwrap()
13644 });
13645 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13646 .await
13647 .unwrap()
13648 .unwrap();
13649 assert!(matches!(
13650 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
13651 ModuleControlRequest::RouteBind { .. }
13652 ));
13653
13654 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
13655 let health_handler = handler.clone();
13656 let health_task = tokio::spawn(async move {
13657 health_handler
13658 .handle_control_frame(
13659 &health_client_ctx,
13660 supervisor_health_probe_frame(101, "aft"),
13661 )
13662 .await
13663 .unwrap()
13664 });
13665 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13666 .await
13667 .unwrap()
13668 .unwrap();
13669 assert_eq!(
13670 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13671 ModuleControlRequest::HealthCheck {}
13672 );
13673
13674 handler
13675 .handle_control_frame(
13676 &module_ctx,
13677 health_response(health_frame.header.corr, HealthStatus::Degraded),
13678 )
13679 .await
13680 .unwrap();
13681 let health_response = health_task.await.unwrap();
13682 assert_eq!(health_response.len(), 1);
13683 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13684 ClientControlResponse::SupervisorHealthProbe {
13685 module_id,
13686 status,
13687 detail,
13688 metrics,
13689 } => {
13690 assert_eq!(module_id, "aft");
13691 assert_eq!(status, HealthStatus::Degraded);
13692 assert_eq!(detail.as_deref(), Some("warming"));
13693 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
13694 }
13695 other => panic!("unexpected health response: {other:?}"),
13696 }
13697
13698 handler
13699 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13700 .await
13701 .unwrap();
13702 let route_response = route_task.await.unwrap();
13703 assert!(route_response.is_empty());
13704 let published = route_client_rx.recv().await.unwrap();
13705 assert!(matches!(
13706 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13707 ClientControlResponse::RouteOpen { .. }
13708 ));
13709 }
13710
13711 async fn relay_route_open(
13716 handler: &ControlHandler,
13717 client_connection: ConnectionId,
13718 client_egress: &FrameSink,
13719 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13720 corr: u64,
13721 module_id: &str,
13722 project_root_label: &str,
13723 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
13724 let ctx = RouteCtx {
13725 connection_id: client_connection,
13726 egress: client_egress.clone(),
13727 };
13728 let handler = handler.clone();
13729 let project_root = unique_project_root(project_root_label);
13730 let module_id = module_id.to_string();
13731 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
13732 let task = tokio::spawn(async move {
13733 let _guard = tracing::dispatcher::set_default(&dispatch);
13734 handler
13735 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
13736 .await
13737 .unwrap()
13738 });
13739 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13740 .await
13741 .expect("module receives the relayed route.bind")
13742 .expect("module egress is open");
13743 (task, bind.frame)
13744 }
13745
13746 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
13747 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
13748 ModuleControlRequest::RouteBind {
13749 route_channel,
13750 epoch,
13751 ..
13752 } => (route_channel, epoch),
13753 other => panic!("expected a route.bind request, got {other:?}"),
13754 }
13755 }
13756
13757 fn published_route(frame: &Frame) -> (u16, u32) {
13758 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
13759 ClientControlResponse::RouteOpen {
13760 route_channel,
13761 route_epoch,
13762 ..
13763 } => (route_channel, route_epoch),
13764 other => panic!("expected a route.open response, got {other:?}"),
13765 }
13766 }
13767
13768 #[tokio::test]
13788 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
13789 let registry = Arc::new(Registry::default());
13790 let forwarding = Arc::new(ForwardingTable::default());
13791 let handler =
13792 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13793
13794 let module_connection = ConnectionId::new(30);
13795 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13796 hello_via_sink(
13797 &handler,
13798 &module_ctx,
13799 &mut module_rx,
13800 hello_frame("aft", PROTOCOL_VERSION, 7),
13801 )
13802 .await;
13803
13804 let dying_client = ConnectionId::new(31);
13805 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
13806
13807 let (first_task, first_bind) = relay_route_open(
13810 &handler,
13811 dying_client,
13812 &dying_ctx.egress,
13813 &mut module_rx,
13814 100,
13815 "aft",
13816 "closing-first",
13817 )
13818 .await;
13819 handler
13820 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
13821 .await
13822 .unwrap();
13823 assert!(first_task.await.unwrap().is_empty());
13824 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
13825
13826 let (second_task, second_bind) = relay_route_open(
13828 &handler,
13829 dying_client,
13830 &dying_ctx.egress,
13831 &mut module_rx,
13832 101,
13833 "aft",
13834 "closing-second",
13835 )
13836 .await;
13837 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
13838
13839 assert!(forwarding
13842 .escalate_client_delivery_failure(
13843 dying_client,
13844 first_channel,
13845 first_epoch,
13846 CloseReason::new(
13847 "module_to_client_delivery_failed",
13848 "client egress refused a module frame",
13849 ),
13850 crate::forwarding::UndeliveredFrame {
13851 module_id: None,
13852 sink: &dying_ctx.egress,
13853 },
13854 )
13855 .unwrap());
13856 assert!(!dying_ctx.egress.is_closed());
13857
13858 let ack = handler
13860 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
13861 .await;
13862 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13863 if module_loop_error.is_some() {
13864 handler.cleanup_connection(module_connection).unwrap();
13868 }
13869 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13873 .await
13874 .ok()
13875 .flatten();
13876
13877 assert!(
13879 registry
13880 .get_module_by_connection(module_connection)
13881 .unwrap()
13882 .is_some(),
13883 "one client's closing connection ended the shared module connection: \
13884 {module_loop_error:?}"
13885 );
13886 let cotenant = ConnectionId::new(32);
13888 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13889 let (cotenant_task, cotenant_bind) = relay_route_open(
13890 &handler,
13891 cotenant,
13892 &cotenant_ctx.egress,
13893 &mut module_rx,
13894 102,
13895 "aft",
13896 "closing-cotenant",
13897 )
13898 .await;
13899 handler
13900 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
13901 .await
13902 .unwrap();
13903 assert!(cotenant_task.await.unwrap().is_empty());
13904 let (cotenant_channel, cotenant_epoch) =
13905 published_route(&cotenant_rx.recv().await.unwrap());
13906 assert!(matches!(
13907 forwarding
13908 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
13909 .unwrap(),
13910 DataRoute::Client(DataRouteState::Bound(_))
13911 ));
13912
13913 let goodbye = post_ack_module_frame
13916 .expect("module receives a GOODBYE for the abandoned route channel");
13917 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13918 assert_eq!(goodbye.header.channel, abandoned_channel);
13919 assert_eq!(goodbye.header.epoch, abandoned_epoch);
13920
13921 assert!(dying_rx.try_recv().is_err());
13925 let second_response = second_task.await.unwrap();
13926 assert_eq!(second_response.len(), 1);
13927 assert_eq!(
13928 parse_error(&second_response[0])["code"],
13929 "target_unavailable"
13930 );
13931 }
13932
13933 async fn catalog_snapshot(
13936 handler: &ControlHandler,
13937 client_ctx: &RouteCtx,
13938 corr: u64,
13939 ) -> (u64, Vec<(String, bool)>) {
13940 let body =
13941 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
13942 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
13943 let reply = handler
13944 .handle_control_frame(client_ctx, request)
13945 .await
13946 .unwrap()
13947 .pop()
13948 .unwrap();
13949 let ClientControlResponse::CatalogList {
13950 generation,
13951 modules,
13952 ..
13953 } = serde_json::from_slice(&reply.body).unwrap()
13954 else {
13955 panic!("not a catalog.list reply");
13956 };
13957 (
13958 generation,
13959 modules
13960 .into_iter()
13961 .map(|entry| (entry.module_id, entry.ready))
13962 .collect(),
13963 )
13964 }
13965
13966 fn hello_held_before_forwarding() -> (
13972 ControlHandler,
13973 Arc<std::sync::Barrier>,
13974 std::thread::JoinHandle<Vec<Frame>>,
13975 RouteCtx,
13976 mpsc::Receiver<crate::router::OutboundFrame>,
13977 ) {
13978 let hold = Arc::new(std::sync::Barrier::new(2));
13979 let handler = ControlHandler::with_forwarding(
13980 Arc::new(Registry::default()),
13981 Arc::new(ForwardingTable::default()),
13982 )
13983 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
13984 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
13985 let hello = {
13986 let handler = handler.clone();
13987 let module_ctx = module_ctx.clone();
13988 std::thread::spawn(move || {
13989 handler
13990 .handle_hello(
13991 module_ctx.connection_id,
13992 Some(module_ctx.egress.clone()),
13993 hello_frame("aft", PROTOCOL_VERSION, 7),
13994 )
13995 .unwrap()
13996 })
13997 };
13998 (handler, hold, hello, module_ctx, module_rx)
13999 }
14000
14001 fn lookup_during_hello_publication<T: Send + 'static>(
14007 registry: &Registry,
14008 hold: &std::sync::Barrier,
14009 lookup: impl FnOnce() -> T + Send + 'static,
14010 ) -> T {
14011 let attempted = registry.observe_next_lock_attempt();
14012 let reader = std::thread::spawn(lookup);
14013 if attempted.recv().unwrap() {
14014 let result = reader.join().unwrap();
14015 hold.wait();
14016 result
14017 } else {
14018 hold.wait();
14019 reader.join().unwrap()
14020 }
14021 }
14022
14023 #[test]
14024 fn catalog_read_after_hello_ack_waits_for_publication() {
14025 let registry = Arc::new(Registry::default());
14026 let hold = Arc::new(std::sync::Barrier::new(2));
14027 let mut handler = ControlHandler::with_forwarding(
14028 Arc::clone(®istry),
14029 Arc::new(ForwardingTable::default()),
14030 );
14031 handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
14032 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14033 let hello = {
14034 let handler = handler.clone();
14035 std::thread::spawn(move || {
14036 handler
14037 .handle_hello(
14038 module_ctx.connection_id,
14039 Some(module_ctx.egress),
14040 hello_frame("aft", PROTOCOL_VERSION, 7),
14041 )
14042 .unwrap()
14043 })
14044 };
14045 hold.wait();
14046 assert_eq!(
14047 module_rx.try_recv().unwrap().frame.header.ty,
14048 FrameType::HelloAck
14049 );
14050 let replies = lookup_during_hello_publication(®istry, &hold, move || {
14051 let request = Frame::build(
14054 FrameType::Request,
14055 control_flags(),
14056 0,
14057 0,
14058 300,
14059 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
14060 )
14061 .unwrap();
14062 handler.handle_catalog_list(request, None).unwrap()
14063 });
14064 assert!(hello.join().unwrap().is_empty());
14065 let ClientControlResponse::CatalogList {
14066 generation,
14067 modules,
14068 ..
14069 } = serde_json::from_slice(&replies[0].body).unwrap()
14070 else {
14071 panic!("not a catalog.list reply");
14072 };
14073 assert_eq!(
14074 modules
14075 .iter()
14076 .map(|entry| entry.module_id.as_str())
14077 .collect::<Vec<_>>(),
14078 vec!["aft"]
14079 );
14080 assert_eq!(generation, 1);
14081 }
14082
14083 #[tokio::test]
14092 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
14093 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
14094 let client = ConnectionId::new(41);
14095 let (client_ctx, mut client_rx) = route_ctx(client);
14096
14097 hold.wait();
14098 assert!(handler.registry.get_module("aft").unwrap().is_some());
14100 assert!(!handler
14101 .forwarding
14102 .has_live_module_connection("aft")
14103 .unwrap());
14104 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
14105 assert!(
14106 listed.is_empty(),
14107 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
14108 );
14109 let refused = handler
14110 .handle_control_frame(
14111 &client_ctx,
14112 route_open_frame(301, "aft", unique_project_root("listed-window")),
14113 )
14114 .await
14115 .unwrap();
14116 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
14117 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
14118 let duplicate = handler
14119 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
14120 .await
14121 .unwrap();
14122 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14123
14124 hold.wait();
14125 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14126 let ack = module_rx.try_recv().unwrap().frame;
14127 assert_eq!(ack.header.ty, FrameType::HelloAck);
14128 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14129 assert_eq!(listed, vec![("aft".to_string(), true)]);
14130
14131 let (open_task, bind) = relay_route_open(
14132 &handler,
14133 client,
14134 &client_ctx.egress,
14135 &mut module_rx,
14136 303,
14137 "aft",
14138 "listed-after",
14139 )
14140 .await;
14141 assert!(handler
14142 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14143 .await
14144 .unwrap()
14145 .is_empty());
14146 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
14147 published_route(&client_rx.try_recv().unwrap().frame);
14148 }
14149
14150 #[tokio::test]
14157 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
14158 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
14159 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14160 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14161
14162 hold.wait();
14163 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
14164 assert!(listed.is_empty());
14165 assert_eq!(
14166 during, before,
14167 "the generation moved while the module was not listed"
14168 );
14169
14170 hold.wait();
14171 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14172 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14173 assert_eq!(listed, vec![("aft".to_string(), true)]);
14174 assert_eq!(
14175 after,
14176 before + 1,
14177 "the module appeared without exactly one generation bump"
14178 );
14179 }
14180
14181 #[tokio::test]
14186 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
14187 let registry = Arc::new(Registry::default());
14188 let handler = ControlHandler::with_forwarding(
14189 Arc::clone(®istry),
14190 Arc::new(ForwardingTable::default()),
14191 );
14192 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14193 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14194 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14195 drop(module_rx);
14196
14197 let replies = handler
14198 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
14199 .await
14200 .unwrap();
14201 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
14202 assert!(registry.get_module("aft").unwrap().is_none());
14203 assert!(registry
14204 .get_module_by_connection(module_ctx.connection_id)
14205 .unwrap()
14206 .is_none());
14207 assert_eq!(
14208 catalog_snapshot(&handler, &client_ctx, 301).await,
14209 (before, Vec::new())
14210 );
14211 }
14212
14213 async fn route_open_held_after_bind_send() -> (
14217 ControlHandler,
14218 Arc<tokio::sync::Barrier>,
14219 RouteCtx,
14220 mpsc::Receiver<crate::router::OutboundFrame>,
14221 ) {
14222 let hold = Arc::new(tokio::sync::Barrier::new(2));
14223 let handler = ControlHandler::with_forwarding(
14224 Arc::new(Registry::default()),
14225 Arc::new(ForwardingTable::default()),
14226 )
14227 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
14228 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14229 hello_via_sink(
14230 &handler,
14231 &module_ctx,
14232 &mut module_rx,
14233 hello_frame("aft", PROTOCOL_VERSION, 7),
14234 )
14235 .await;
14236 (handler, hold, module_ctx, module_rx)
14237 }
14238
14239 #[tokio::test]
14246 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
14247 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14248 let client = ConnectionId::new(41);
14249 let (client_ctx, mut client_rx) = route_ctx(client);
14250
14251 let (open_task, bind) = relay_route_open(
14252 &handler,
14253 client,
14254 &client_ctx.egress,
14255 &mut module_rx,
14256 200,
14257 "aft",
14258 "acked-before-mark",
14259 )
14260 .await;
14261 let (module_channel, module_epoch) = route_bind_channel(&bind);
14262 hold.wait().await;
14264
14265 assert!(handler
14267 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14268 .await
14269 .unwrap()
14270 .is_empty());
14271 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
14272
14273 hold.wait().await;
14274 assert!(
14275 open_task.await.unwrap().is_empty(),
14276 "route.open answers accepted"
14277 );
14278
14279 match module_rx.try_recv() {
14282 Err(mpsc::error::TryRecvError::Empty) => {}
14283 Ok(outbound) => panic!(
14284 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
14285 outbound.frame.header.ty,
14286 outbound.frame.header.channel,
14287 outbound.frame.header.epoch
14288 ),
14289 Err(err) => panic!("module egress closed: {err}"),
14290 }
14291
14292 let router = Router::with_control_handler(Arc::new(handler.clone()));
14295 let request = Frame::build(
14296 FrameType::Request,
14297 Flags::new(true, Priority::Interactive, false),
14298 client_channel,
14299 client_epoch,
14300 201,
14301 b"first request".to_vec(),
14302 )
14303 .unwrap();
14304 router
14305 .route_for_connection(&client_ctx, request)
14306 .await
14307 .unwrap();
14308 let delivered = module_rx.try_recv().unwrap().frame;
14309 assert_eq!(delivered.header.ty, FrameType::Request);
14310 assert_eq!(
14311 (delivered.header.channel, delivered.header.epoch),
14312 (module_channel, module_epoch)
14313 );
14314 assert_eq!(delivered.body, b"first request");
14315 }
14316
14317 #[tokio::test]
14323 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
14324 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14325 let client = ConnectionId::new(42);
14326 let (client_ctx, _client_rx) = route_ctx(client);
14327
14328 let (open_task, bind) = relay_route_open(
14329 &handler,
14330 client,
14331 &client_ctx.egress,
14332 &mut module_rx,
14333 300,
14334 "aft",
14335 "released-before-mark",
14336 )
14337 .await;
14338 let (module_channel, module_epoch) = route_bind_channel(&bind);
14339 hold.wait().await;
14340
14341 handler.cleanup_connection(client).unwrap();
14342 assert!(
14343 module_rx.try_recv().is_err(),
14344 "client teardown must not GOODBYE a bind route.open is still queueing"
14345 );
14346
14347 hold.wait().await;
14348 let response = open_task.await.unwrap();
14349 assert_eq!(response.len(), 1);
14350 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14351
14352 let goodbye = module_rx
14353 .try_recv()
14354 .expect("module is told to drop the binding the daemon abandoned")
14355 .frame;
14356 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14357 assert_eq!(
14358 (goodbye.header.channel, goodbye.header.epoch),
14359 (module_channel, module_epoch)
14360 );
14361 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14362 }
14363
14364 #[tokio::test]
14369 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
14370 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14371 let client = ConnectionId::new(43);
14372 let (client_ctx, client_rx) = route_ctx(client);
14373
14374 let (open_task, bind) = relay_route_open(
14375 &handler,
14376 client,
14377 &client_ctx.egress,
14378 &mut module_rx,
14379 400,
14380 "aft",
14381 "gone-client-before-mark",
14382 )
14383 .await;
14384 let (module_channel, module_epoch) = route_bind_channel(&bind);
14385 hold.wait().await;
14386
14387 drop(client_rx);
14388 assert!(client_ctx.egress.is_closed());
14389 handler
14390 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14391 .await
14392 .unwrap();
14393
14394 hold.wait().await;
14395 let response = open_task.await.unwrap();
14396 assert_eq!(response.len(), 1);
14397 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14398
14399 let goodbye = module_rx
14400 .try_recv()
14401 .expect("module is told to drop the binding it acked")
14402 .frame;
14403 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14404 assert_eq!(
14405 (goodbye.header.channel, goodbye.header.epoch),
14406 (module_channel, module_epoch)
14407 );
14408 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14409 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14410 }
14411
14412 #[test]
14419 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
14420 let handler = ControlHandler::default();
14421 let module_connection = ConnectionId::new(30);
14422 let client_connection = ConnectionId::new(31);
14423
14424 handler
14425 .refuse_to_end_module_connection_for_a_client(
14426 module_connection,
14427 77,
14428 ForwardingError::ConnectionClosing {
14429 connection_id: client_connection,
14430 },
14431 )
14432 .expect("a closing client must never end the module connection");
14433
14434 assert!(matches!(
14435 handler.refuse_to_end_module_connection_for_a_client(
14436 module_connection,
14437 78,
14438 ForwardingError::ConnectionClosing {
14439 connection_id: module_connection,
14440 },
14441 ),
14442 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
14443 connection_id
14444 })) if connection_id == module_connection
14445 ));
14446 assert!(matches!(
14447 handler.refuse_to_end_module_connection_for_a_client(
14448 module_connection,
14449 79,
14450 ForwardingError::Poisoned,
14451 ),
14452 Err(RouterError::Forwarding(ForwardingError::Poisoned))
14453 ));
14454 assert!(matches!(
14455 handler.refuse_to_end_module_connection_for_a_client(
14456 module_connection,
14457 80,
14458 ForwardingError::StaleModuleEndpoint,
14459 ),
14460 Err(RouterError::Forwarding(
14461 ForwardingError::StaleModuleEndpoint
14462 ))
14463 ));
14464 }
14465
14466 #[tokio::test]
14479 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
14480 let registry = Arc::new(Registry::default());
14481 let forwarding = Arc::new(ForwardingTable::default());
14482 let supervisor = SupervisorHandle::new();
14483 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14484 let handler =
14485 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14486 .with_supervisor(supervisor);
14487
14488 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14489 hello_via_sink(
14490 &handler,
14491 &target_ctx,
14492 &mut target_rx,
14493 hello_frame("target", PROTOCOL_VERSION, 1),
14494 )
14495 .await;
14496
14497 let wrong_nonce = handler
14501 .handle_control_frame(
14502 &route_ctx(ConnectionId::new(91)).0,
14503 route_open_frame_with_admission_facts(
14504 20,
14505 "target",
14506 unique_project_root("admission-facts"),
14507 Some(subc_control::ConsumerIdentity {
14508 module_id: "fed".to_string(),
14509 launch_nonce: "not-the-real-nonce".to_string(),
14510 }),
14511 None,
14512 ),
14513 )
14514 .await
14515 .unwrap();
14516 assert_eq!(
14517 parse_error(&wrong_nonce[0])["code"],
14518 "bad_consumer_identity",
14519 "a mismatched launch nonce must be refused, not stamped Reserved"
14520 );
14521
14522 let never_spawned = handler
14526 .handle_control_frame(
14527 &route_ctx(ConnectionId::new(92)).0,
14528 route_open_frame_with_admission_facts(
14529 21,
14530 "target",
14531 unique_project_root("admission-facts"),
14532 Some(subc_control::ConsumerIdentity {
14533 module_id: "never-spawned".to_string(),
14534 launch_nonce: "any-nonce".to_string(),
14535 }),
14536 None,
14537 ),
14538 )
14539 .await
14540 .unwrap();
14541 assert_eq!(
14542 parse_error(&never_spawned[0])["code"],
14543 "bad_consumer_identity",
14544 "an unspawned module_id must be refused rather than accepted for lack of a record"
14545 );
14546 }
14547
14548 #[tokio::test]
14571 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
14572 let registry = Arc::new(Registry::default());
14573 let forwarding = Arc::new(ForwardingTable::default());
14574 let supervisor = SupervisorHandle::new();
14575 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14576 let handler =
14577 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14578 .with_supervisor(supervisor);
14579
14580 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
14581 hello_via_sink(
14582 &handler,
14583 &target_ctx,
14584 &mut target_rx,
14585 hello_frame("target", PROTOCOL_VERSION, 1),
14586 )
14587 .await;
14588
14589 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
14590 let route_handler = handler.clone();
14591 let route_task = tokio::spawn(async move {
14592 route_handler
14593 .handle_control_frame(
14594 &client_ctx,
14595 route_open_frame_with_admission_facts(
14596 30,
14597 "target",
14598 unique_project_root("admission-facts"),
14599 Some(subc_control::ConsumerIdentity {
14600 module_id: "fed".to_string(),
14601 launch_nonce: "fed-nonce".to_string(),
14602 }),
14603 None,
14604 ),
14605 )
14606 .await
14607 .unwrap()
14608 });
14609
14610 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14617 .await
14618 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
14619 .expect("module control channel closed before route.bind");
14620 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14621 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14622 panic!("expected route.bind")
14623 };
14624 assert_eq!(
14625 principal,
14626 Some(Principal::Reserved {
14627 module_id: "fed".to_string()
14628 }),
14629 "a correctly attested consumer must be stamped Reserved for its own id"
14630 );
14631
14632 handler
14633 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14634 .await
14635 .unwrap();
14636 assert!(route_task.await.unwrap().is_empty());
14637 assert!(
14638 matches!(
14639 serde_json::from_slice::<ClientControlResponse>(
14640 &client_rx.recv().await.unwrap().body
14641 )
14642 .unwrap(),
14643 ClientControlResponse::RouteOpen { .. }
14644 ),
14645 "the route must actually open, not merely avoid an error"
14646 );
14647 }
14648
14649 #[tokio::test(start_paused = true)]
14650 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
14651 let registry = Arc::new(Registry::default());
14652 let forwarding = Arc::new(ForwardingTable::default());
14653 let supervisor = SupervisorHandle::new();
14654 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14655 let handler =
14656 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14657 .with_supervisor(supervisor);
14658
14659 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
14660 hello_via_sink(
14661 &handler,
14662 &target_ctx,
14663 &mut target_rx,
14664 hello_frame("target", PROTOCOL_VERSION, 1),
14665 )
14666 .await;
14667
14668 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
14669 let direct_handler = handler.clone();
14670 let direct_open = tokio::spawn(async move {
14671 direct_handler
14672 .handle_control_frame(
14673 &direct_ctx,
14674 route_open_frame(2, "target", unique_project_root("route-census-direct")),
14675 )
14676 .await
14677 .unwrap()
14678 });
14679 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14680 .await
14681 .expect("no direct route.bind within 5s")
14682 .expect("target control channel closed before direct route.bind");
14683 handler
14684 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
14685 .await
14686 .unwrap();
14687 assert!(direct_open.await.unwrap().is_empty());
14688 let _ = direct_rx.recv().await.unwrap();
14689
14690 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
14691 let reserved_handler = handler.clone();
14692 let reserved_open = tokio::spawn(async move {
14693 reserved_handler
14694 .handle_control_frame(
14695 &reserved_ctx,
14696 route_open_frame_with_admission_facts(
14697 3,
14698 "target",
14699 unique_project_root("admission-facts"),
14700 Some(ConsumerIdentity {
14701 module_id: "fed".to_string(),
14702 launch_nonce: "fed-nonce".to_string(),
14703 }),
14704 None,
14705 ),
14706 )
14707 .await
14708 .unwrap()
14709 });
14710 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14711 .await
14712 .expect("no reserved route.bind within 5s")
14713 .expect("target control channel closed before reserved route.bind");
14714 handler
14715 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
14716 .await
14717 .unwrap();
14718 assert!(reserved_open.await.unwrap().is_empty());
14719 let _ = reserved_rx.recv().await.unwrap();
14720
14721 forwarding
14722 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
14723 .unwrap();
14724 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
14725 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
14726 module_id: Some("target".to_string()),
14727 })
14728 .unwrap();
14729 let census_frame =
14730 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
14731 let response = handler
14732 .handle_control_frame(&census_ctx, census_frame)
14733 .await
14734 .unwrap()
14735 .pop()
14736 .unwrap();
14737 let actual: Value = serde_json::from_slice(&response.body).unwrap();
14738 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
14739 assert!(matches!(
14740 decoded,
14741 ClientControlResponse::SupervisorRoutes { .. }
14742 ));
14743 let routes = actual["modules"][0]["routes"].as_array().unwrap();
14744 assert_eq!(routes.len(), 2);
14745 assert!(routes.iter().all(|route| route["draining"] == true));
14746 assert!(
14749 routes.iter().all(|route| route["drain_reason"] == "reload"),
14750 "draining routes must name the drain's reason: {routes:?}"
14751 );
14752 assert!(routes.iter().any(|route| {
14753 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
14754 }));
14755 assert!(routes.iter().any(|route| {
14756 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
14757 }));
14758
14759 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14760 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
14761 if std::env::var_os("UPDATE_GOLDEN").is_some() {
14762 std::fs::write(
14763 &golden_path,
14764 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
14765 )
14766 .unwrap();
14767 }
14768 let expected: Value =
14769 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
14770 assert_eq!(actual, expected);
14771 }
14772
14773 async fn query_live_roots(
14774 handler: &ControlHandler,
14775 module_ctx: &RouteCtx,
14776 ) -> ModuleControlResponseToModule {
14777 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
14778 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
14779 let response = handler
14780 .handle_control_frame(module_ctx, frame)
14781 .await
14782 .unwrap()
14783 .pop()
14784 .unwrap();
14785 serde_json::from_slice(&response.body).unwrap()
14786 }
14787
14788 #[tokio::test(start_paused = true)]
14789 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
14790 let registry = Arc::new(Registry::default());
14791 let forwarding = Arc::new(ForwardingTable::default());
14792 let handler = ControlHandler::with_forwarding(registry, forwarding);
14793 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
14794 hello_via_sink(
14795 &handler,
14796 &target_ctx,
14797 &mut target_rx,
14798 hello_frame("target", PROTOCOL_VERSION, 1),
14799 )
14800 .await;
14801 let root = unique_project_root("live-roots-known");
14802 let path = ProjectRootId::from_path_allowing_missing(root.path())
14803 .unwrap()
14804 .as_path()
14805 .to_path_buf();
14806 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
14807 let open_handler = handler.clone();
14808 let opened = tokio::spawn(async move {
14809 open_handler
14810 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
14811 .await
14812 .unwrap()
14813 });
14814 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14815 .await
14816 .unwrap()
14817 .unwrap();
14818 handler
14819 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
14820 .await
14821 .unwrap();
14822 assert!(opened.await.unwrap().is_empty());
14823 let _ = client_rx.recv().await.unwrap();
14824
14825 let root = unique_project_root("live-roots-pending");
14826 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
14827 .unwrap()
14828 .as_path()
14829 .to_path_buf();
14830 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
14831 let open_handler = handler.clone();
14832 let pending = tokio::spawn(async move {
14833 open_handler
14834 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
14835 .await
14836 .unwrap()
14837 });
14838 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14839 .await
14840 .unwrap()
14841 .unwrap();
14842 let actual = query_live_roots(&handler, &target_ctx).await;
14843 let ModuleControlResponseToModule::LiveRoots {
14844 roots,
14845 unknown_root_bindings,
14846 total_bindings,
14847 } = actual
14848 else {
14849 panic!("expected live roots")
14850 };
14851 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
14852 assert_eq!(unknown_root_bindings, 0);
14853 assert_eq!(
14854 roots.len(),
14855 2,
14856 "root-known arm must retain each canonical root"
14857 );
14858 assert_eq!(
14859 total_bindings,
14860 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14861 );
14862 let counts = roots
14863 .iter()
14864 .map(|root| (root.project_root.clone(), root.bound, root.pending))
14865 .collect::<Vec<_>>();
14866 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
14867 expected.sort_by(|a, b| a.0.cmp(&b.0));
14868 assert_eq!(
14869 counts, expected,
14870 "roots must sort by path and count pending separately"
14871 );
14872 handler
14873 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
14874 .await
14875 .unwrap();
14876 assert!(pending.await.unwrap().is_empty());
14877 }
14878
14879 #[tokio::test(start_paused = true)]
14880 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
14881 let forwarding = Arc::new(ForwardingTable::default());
14882 let handler =
14883 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14884 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
14885 hello_via_sink(
14886 &handler,
14887 &target_ctx,
14888 &mut target_rx,
14889 hello_frame("target", PROTOCOL_VERSION, 1),
14890 )
14891 .await;
14892 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
14893 let pending = forwarding
14894 .begin_route_bind_relay_for_test(
14895 client_ctx.connection_id,
14896 client_ctx.egress.clone(),
14897 2,
14898 "target",
14899 )
14900 .unwrap();
14901 forwarding
14902 .complete_pending_relay(
14903 target_ctx.connection_id,
14904 pending.corr,
14905 RouteBindRelayOutcome::Accepted,
14906 )
14907 .unwrap();
14908 let actual = query_live_roots(&handler, &target_ctx).await;
14909 let ModuleControlResponseToModule::LiveRoots {
14910 roots,
14911 unknown_root_bindings,
14912 total_bindings,
14913 } = actual
14914 else {
14915 panic!("expected live roots")
14916 };
14917 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
14918 assert_eq!(
14919 unknown_root_bindings, 1,
14920 "unknown-root arm must not read as no bindings"
14921 );
14922 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
14923 assert_eq!(
14924 total_bindings,
14925 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14926 );
14927 }
14928
14929 #[tokio::test(start_paused = true)]
14937 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
14938 let forwarding = Arc::new(ForwardingTable::default());
14939 let handler =
14940 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14941 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
14942 let replies = handler
14943 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
14944 .await
14945 .unwrap();
14946 let queued_by_hello = module_rx.len();
14947
14948 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
14949 let open_handler = handler.clone();
14950 let open = tokio::spawn(async move {
14951 open_handler
14952 .handle_control_frame(
14953 &client_ctx,
14954 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
14955 )
14956 .await
14957 .unwrap()
14958 });
14959 let mut spins = 0;
14961 while module_rx.len() == queued_by_hello {
14962 spins += 1;
14963 assert!(spins < 10_000, "route.open never queued a route.bind");
14964 tokio::task::yield_now().await;
14965 }
14966
14967 for reply in replies {
14969 module_ctx.egress.send(reply).await.unwrap();
14970 }
14971
14972 let first = module_rx.recv().await.unwrap().frame;
14973 assert_eq!(
14974 first.header.ty,
14975 FrameType::HelloAck,
14976 "the first frame a registering module reads must be its HELLO_ACK"
14977 );
14978 assert_eq!(first.header.corr, 7);
14979 let second = module_rx.recv().await.unwrap().frame;
14980 assert_eq!(second.header.ty, FrameType::Request);
14981 assert!(
14982 matches!(
14983 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
14984 ModuleControlRequest::RouteBind { .. }
14985 ),
14986 "the route.bind follows the ack"
14987 );
14988 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
14989
14990 handler
14991 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
14992 .await
14993 .unwrap();
14994 assert!(open.await.unwrap().is_empty());
14995 let _ = client_rx.recv().await.unwrap();
14996 }
14997
14998 #[tokio::test(start_paused = true)]
14999 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
15000 let handler = ControlHandler::with_forwarding(
15001 Arc::new(Registry::default()),
15002 Arc::new(ForwardingTable::default()),
15003 );
15004 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
15005 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
15006 hello_via_sink(
15007 &handler,
15008 &first_ctx,
15009 &mut first_rx,
15010 hello_frame("first", PROTOCOL_VERSION, 1),
15011 )
15012 .await;
15013 hello_via_sink(
15014 &handler,
15015 &second_ctx,
15016 &mut second_rx,
15017 hello_frame("second", PROTOCOL_VERSION, 2),
15018 )
15019 .await;
15020 let root = unique_project_root("second-only");
15021 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
15022 let cloned = handler.clone();
15023 let open = tokio::spawn(async move {
15024 cloned
15025 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
15026 .await
15027 .unwrap()
15028 });
15029 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
15030 .await
15031 .unwrap()
15032 .unwrap();
15033 let first = query_live_roots(&handler, &first_ctx).await;
15034 let second = query_live_roots(&handler, &second_ctx).await;
15035 assert!(
15036 matches!(
15037 first,
15038 ModuleControlResponseToModule::LiveRoots {
15039 total_bindings: 0,
15040 ..
15041 }
15042 ),
15043 "cross-module scope must not expose another module's roots"
15044 );
15045 assert!(
15046 matches!(
15047 second,
15048 ModuleControlResponseToModule::LiveRoots {
15049 total_bindings: 1,
15050 ..
15051 }
15052 ),
15053 "second module must see its pending route"
15054 );
15055 handler
15056 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
15057 .await
15058 .unwrap();
15059 assert!(open.await.unwrap().is_empty());
15060 }
15061
15062 #[tokio::test(start_paused = true)]
15063 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
15064 let handler = ControlHandler::with_forwarding(
15065 Arc::new(Registry::default()),
15066 Arc::new(ForwardingTable::default()),
15067 );
15068 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
15069 hello_via_sink(
15070 &handler,
15071 &target_ctx,
15072 &mut target_rx,
15073 hello_frame("target", PROTOCOL_VERSION, 1),
15074 )
15075 .await;
15076 let actual = query_live_roots(&handler, &target_ctx).await;
15077 let ModuleControlResponseToModule::LiveRoots {
15078 roots,
15079 unknown_root_bindings,
15080 total_bindings,
15081 } = actual
15082 else {
15083 panic!("expected live roots")
15084 };
15085 assert!(roots.is_empty());
15086 assert_eq!(unknown_root_bindings, 0);
15087 assert_eq!(total_bindings, 0);
15088 assert_eq!(
15089 total_bindings,
15090 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15091 );
15092 }
15093
15094 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
15101 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
15102 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
15103 let text = std::fs::read_to_string(&path)
15104 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
15105 let vectors: Vec<(String, Value)> = text
15106 .lines()
15107 .filter(|line| !line.trim().is_empty())
15108 .map(|line| {
15109 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
15110 let id = entry["corpus_id"]
15111 .as_str()
15112 .expect("every vector carries a corpus_id")
15113 .to_string();
15114 (id, entry["package"].clone())
15115 })
15116 .collect();
15117 assert_eq!(
15121 vectors.len(),
15122 3,
15123 "vendored fed corpus changed size; re-sync from subc-federation"
15124 );
15125
15126 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
15141 let richest = vectors
15142 .iter()
15143 .filter_map(|(_, package)| package.as_object())
15144 .map(|object| {
15145 object
15146 .keys()
15147 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
15148 .count()
15149 })
15150 .max()
15151 .unwrap_or(0);
15152 assert!(
15153 richest >= 2,
15154 "vendored corpus no longer carries a package with unmodelled fields, \
15155 so the relay test can no longer distinguish a verbatim relay from a lossy one"
15156 );
15157
15158 vectors
15159 }
15160
15161 #[tokio::test]
15169 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
15170 for (corpus_id, package) in fed_admission_facts_vectors() {
15171 let registry = Arc::new(Registry::default());
15172 let forwarding = Arc::new(ForwardingTable::default());
15173 let supervisor = SupervisorHandle::new();
15174 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15175 let handler =
15176 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15177 .with_supervisor(supervisor)
15178 .with_admission_facts_config(
15179 Some("fed".to_string()),
15180 Some(vec!["target".to_string()]),
15181 );
15182
15183 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
15184 hello_via_sink(
15185 &handler,
15186 &target_ctx,
15187 &mut target_rx,
15188 hello_frame("target", PROTOCOL_VERSION, 1),
15189 )
15190 .await;
15191
15192 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
15193 let route_handler = handler.clone();
15194 let expected = package.clone();
15195 let route_task = tokio::spawn(async move {
15196 route_handler
15197 .handle_control_frame(
15198 &client_ctx,
15199 route_open_frame_with_admission_facts(
15200 20,
15201 "target",
15202 unique_project_root("admission-facts"),
15203 Some(subc_control::ConsumerIdentity {
15204 module_id: "fed".to_string(),
15205 launch_nonce: "fed-nonce".to_string(),
15206 }),
15207 Some(package),
15208 ),
15209 )
15210 .await
15211 .unwrap()
15212 });
15213
15214 let bind_frame = target_rx.recv().await.unwrap();
15215 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15216 let ModuleControlRequest::RouteBind {
15217 admission_facts, ..
15218 } = bind
15219 else {
15220 panic!("{corpus_id}: expected route.bind")
15221 };
15222 assert_eq!(
15223 admission_facts,
15224 Some(expected),
15225 "{corpus_id}: relay must not add, drop or reshape any field"
15226 );
15227
15228 handler
15229 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15230 .await
15231 .unwrap();
15232 route_task.await.unwrap();
15233 }
15234 }
15235
15236 #[tokio::test]
15237 async fn route_bind_never_relays_the_openers_build_class() {
15238 let registry = Arc::new(Registry::default());
15239 let forwarding = Arc::new(ForwardingTable::default());
15240 let handler =
15241 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15242 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
15243 hello_via_sink(
15244 &handler,
15245 &target_ctx,
15246 &mut target_rx,
15247 hello_frame("target", PROTOCOL_VERSION, 1),
15248 )
15249 .await;
15250
15251 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
15252 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
15253 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15254 busy_refusals: None,
15255 target: RouteTarget::ToolProvider {
15256 module_id: "target".to_string(),
15257 },
15258 identity,
15259 consumer_identity: None,
15260 consumer_capabilities: None,
15261 role_versions: None,
15262 admission_facts: None,
15263 scope: None,
15264 })
15265 .unwrap();
15266 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
15267
15268 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
15269 let route_handler = handler.clone();
15270 let route_task = tokio::spawn(async move {
15271 route_handler
15272 .handle_control_frame(&client_ctx, open)
15273 .await
15274 .unwrap()
15275 });
15276 let bind_frame = target_rx.recv().await.unwrap();
15277 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15278 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
15279 panic!("expected route.bind")
15280 };
15281 assert_eq!(
15282 identity.build_class, None,
15283 "route.bind carried the build class"
15284 );
15285 assert!(
15286 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
15287 "the route.bind bytes still name build_class"
15288 );
15289 handler
15290 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15291 .await
15292 .unwrap();
15293 route_task.await.unwrap();
15294 }
15295
15296 #[tokio::test]
15297 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
15298 let registry = Arc::new(Registry::default());
15299 let forwarding = Arc::new(ForwardingTable::default());
15300 let supervisor = SupervisorHandle::new();
15301 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15302 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
15303 let handler =
15304 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15305 .with_supervisor(supervisor)
15306 .with_admission_facts_config(
15307 Some("fed".to_string()),
15308 Some(vec!["target".to_string()]),
15309 );
15310
15311 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
15312 hello_via_sink(
15313 &handler,
15314 &target_ctx,
15315 &mut target_rx,
15316 hello_frame("target", PROTOCOL_VERSION, 1),
15317 )
15318 .await;
15319 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
15320 hello_via_sink(
15321 &handler,
15322 &other_ctx,
15323 &mut other_rx,
15324 hello_frame("other", PROTOCOL_VERSION, 2),
15325 )
15326 .await;
15327
15328 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
15329 let expected_facts = facts.clone();
15330 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
15331 let route_handler = handler.clone();
15332 let route_task = tokio::spawn(async move {
15333 route_handler
15334 .handle_control_frame(
15335 &client_ctx,
15336 route_open_frame_with_admission_facts(
15337 10,
15338 "target",
15339 unique_project_root("admission-facts"),
15340 Some(subc_control::ConsumerIdentity {
15341 module_id: "fed".to_string(),
15342 launch_nonce: "fed-nonce".to_string(),
15343 }),
15344 Some(facts.clone()),
15345 ),
15346 )
15347 .await
15348 .unwrap()
15349 });
15350 let bind_frame = target_rx.recv().await.unwrap();
15351 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15352 let ModuleControlRequest::RouteBind {
15353 admission_facts, ..
15354 } = bind
15355 else {
15356 panic!("expected route.bind")
15357 };
15358 assert_eq!(admission_facts, Some(expected_facts));
15359 handler
15360 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15361 .await
15362 .unwrap();
15363 assert!(route_task.await.unwrap().is_empty());
15364 assert!(matches!(
15365 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
15366 .unwrap(),
15367 ClientControlResponse::RouteOpen { .. }
15368 ));
15369
15370 let direct = handler
15371 .handle_control_frame(
15372 &route_ctx(ConnectionId::new(73)).0,
15373 route_open_frame_with_admission_facts(
15374 11,
15375 "target",
15376 unique_project_root("admission-facts"),
15377 None,
15378 Some(json!({"x": 1})),
15379 ),
15380 )
15381 .await
15382 .unwrap();
15383 assert_eq!(
15384 parse_error(&direct[0])["code"],
15385 "admission_facts_not_permitted"
15386 );
15387
15388 let different_reserved = handler
15389 .handle_control_frame(
15390 &route_ctx(ConnectionId::new(77)).0,
15391 route_open_frame_with_admission_facts(
15392 15,
15393 "target",
15394 unique_project_root("admission-facts"),
15395 Some(subc_control::ConsumerIdentity {
15396 module_id: "other".to_string(),
15397 launch_nonce: "other-nonce".to_string(),
15398 }),
15399 Some(json!({"x": 1})),
15400 ),
15401 )
15402 .await
15403 .unwrap();
15404 assert_eq!(
15405 parse_error(&different_reserved[0])["code"],
15406 "admission_facts_not_permitted"
15407 );
15408
15409 let other_target = handler
15410 .handle_control_frame(
15411 &route_ctx(ConnectionId::new(74)).0,
15412 route_open_frame_with_admission_facts(
15413 12,
15414 "other",
15415 unique_project_root("admission-facts"),
15416 Some(subc_control::ConsumerIdentity {
15417 module_id: "fed".to_string(),
15418 launch_nonce: "fed-nonce".to_string(),
15419 }),
15420 Some(json!({"x": 1})),
15421 ),
15422 )
15423 .await
15424 .unwrap();
15425 assert_eq!(
15426 parse_error(&other_target[0])["code"],
15427 "admission_facts_target_not_allowed"
15428 );
15429
15430 let nonexistent = handler
15431 .handle_control_frame(
15432 &route_ctx(ConnectionId::new(75)).0,
15433 route_open_frame_with_admission_facts(
15434 13,
15435 "missing",
15436 unique_project_root("admission-facts"),
15437 None,
15438 Some(json!({"x": 1})),
15439 ),
15440 )
15441 .await
15442 .unwrap();
15443 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
15444
15445 let described = handler
15446 .handle_control_frame(
15447 &route_ctx(ConnectionId::new(76)).0,
15448 Frame::build(
15449 FrameType::Request,
15450 control_flags(),
15451 0,
15452 0,
15453 14,
15454 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15455 )
15456 .unwrap(),
15457 )
15458 .await
15459 .unwrap();
15460 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15461 serde_json::from_slice(&described[0].body).unwrap()
15462 else {
15463 panic!("expected server.describe response")
15464 };
15465 assert!(capabilities
15466 .iter()
15467 .any(|cap| cap == "admission_facts_relay_v1"));
15468 }
15469
15470 #[tokio::test]
15471 async fn admission_facts_without_configured_carrier_are_rejected() {
15472 let registry = Arc::new(Registry::default());
15473 let forwarding = Arc::new(ForwardingTable::default());
15474 let handler = ControlHandler::with_forwarding(registry, forwarding);
15475 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
15476 hello_via_sink(
15477 &handler,
15478 &target_ctx,
15479 &mut target_rx,
15480 hello_frame("target", PROTOCOL_VERSION, 1),
15481 )
15482 .await;
15483
15484 let responses = handler
15485 .handle_control_frame(
15486 &route_ctx(ConnectionId::new(79)).0,
15487 route_open_frame_with_admission_facts(
15488 16,
15489 "target",
15490 unique_project_root("admission-facts"),
15491 None,
15492 Some(json!({"x": 1})),
15493 ),
15494 )
15495 .await
15496 .unwrap();
15497 assert_eq!(
15498 parse_error(&responses[0])["code"],
15499 "admission_facts_not_permitted"
15500 );
15501 }
15502
15503 #[tokio::test]
15504 async fn route_open_relays_consumer_capabilities_verbatim() {
15505 let registry = Arc::new(Registry::default());
15506 let forwarding = Arc::new(ForwardingTable::default());
15507 let handler =
15508 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15509 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
15510 hello_via_sink(
15511 &handler,
15512 &module_ctx,
15513 &mut module_rx,
15514 hello_frame("aft", PROTOCOL_VERSION, 7),
15515 )
15516 .await;
15517
15518 let expected = vec!["elicitation".to_string(), "roots".to_string()];
15519 let expected_for_request = expected.clone();
15520 let project_root = unique_project_root("consumer-capabilities-present");
15521 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
15522 let route_handler = handler.clone();
15523 let route_task = tokio::spawn(async move {
15524 route_handler
15525 .handle_control_frame(
15526 &client_ctx,
15527 route_open_frame_with_consumer_capabilities(
15528 401,
15529 "aft",
15530 project_root,
15531 Some(expected_for_request),
15532 ),
15533 )
15534 .await
15535 .unwrap()
15536 });
15537 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15538 .await
15539 .unwrap()
15540 .unwrap();
15541 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15542 let ModuleControlRequest::RouteBind {
15543 consumer_capabilities,
15544 ..
15545 } = bind
15546 else {
15547 panic!("expected route.bind request, got {bind:?}");
15548 };
15549 assert_eq!(consumer_capabilities, Some(expected.clone()));
15550
15551 handler
15552 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15553 .await
15554 .unwrap();
15555 let route_response = route_task.await.unwrap();
15556 assert!(route_response.is_empty());
15557 let published = client_rx.recv().await.unwrap();
15558 assert!(matches!(
15559 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15560 ClientControlResponse::RouteOpen { .. }
15561 ));
15562 }
15563
15564 #[tokio::test]
15565 async fn route_open_without_consumer_capabilities_relays_none() {
15566 let registry = Arc::new(Registry::default());
15567 let forwarding = Arc::new(ForwardingTable::default());
15568 let handler =
15569 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15570 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
15571 hello_via_sink(
15572 &handler,
15573 &module_ctx,
15574 &mut module_rx,
15575 hello_frame("aft", PROTOCOL_VERSION, 7),
15576 )
15577 .await;
15578
15579 let project_root = unique_project_root("consumer-capabilities-absent");
15580 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
15581 let route_handler = handler.clone();
15582 let route_task = tokio::spawn(async move {
15583 route_handler
15584 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
15585 .await
15586 .unwrap()
15587 });
15588 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15589 .await
15590 .unwrap()
15591 .unwrap();
15592 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15593 let ModuleControlRequest::RouteBind {
15594 consumer_capabilities,
15595 ..
15596 } = bind
15597 else {
15598 panic!("expected route.bind request, got {bind:?}");
15599 };
15600 assert_eq!(consumer_capabilities, None);
15601
15602 handler
15603 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15604 .await
15605 .unwrap();
15606 let route_response = route_task.await.unwrap();
15607 assert!(route_response.is_empty());
15608 let published = client_rx.recv().await.unwrap();
15609 assert!(matches!(
15610 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15611 ClientControlResponse::RouteOpen { .. }
15612 ));
15613 }
15614
15615 async fn bind_role_versions_for(
15619 sent: Option<BTreeMap<String, String>>,
15620 connection: u64,
15621 ) -> Option<BTreeMap<String, String>> {
15622 let registry = Arc::new(Registry::default());
15623 let forwarding = Arc::new(ForwardingTable::default());
15624 let handler =
15625 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15626 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
15627 hello_via_sink(
15628 &handler,
15629 &module_ctx,
15630 &mut module_rx,
15631 hello_frame("aft", PROTOCOL_VERSION, 7),
15632 )
15633 .await;
15634 let project_root = unique_project_root("role-versions");
15635 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
15636 let route_handler = handler.clone();
15637 let route_task = tokio::spawn(async move {
15638 route_handler
15639 .handle_control_frame(
15640 &client_ctx,
15641 route_open_frame_with_role_versions(403, "aft", project_root, sent),
15642 )
15643 .await
15644 .unwrap()
15645 });
15646 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15647 .await
15648 .expect("a well-formed route.open reaches the module as a bind")
15649 .unwrap();
15650 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15651 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
15652 panic!("expected route.bind request, got {bind:?}");
15653 };
15654 handler
15655 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15656 .await
15657 .unwrap();
15658 assert!(route_task.await.unwrap().is_empty());
15659 let published = client_rx.recv().await.unwrap();
15660 assert!(matches!(
15661 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15662 ClientControlResponse::RouteOpen { .. }
15663 ));
15664 role_versions
15665 }
15666
15667 #[tokio::test]
15668 async fn route_open_relays_role_versions_verbatim() {
15669 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
15670 assert_eq!(
15671 bind_role_versions_for(Some(sent.clone()), 141).await,
15672 Some(sent)
15673 );
15674 }
15675
15676 #[tokio::test]
15679 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
15680 assert_eq!(bind_role_versions_for(None, 143).await, None);
15681 assert_eq!(
15682 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
15683 None
15684 );
15685 }
15686
15687 #[tokio::test]
15688 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
15689 let registry = Arc::new(Registry::default());
15690 let forwarding = Arc::new(ForwardingTable::default());
15691 let handler =
15692 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15693 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
15694 hello_via_sink(
15695 &handler,
15696 &module_ctx,
15697 &mut module_rx,
15698 hello_frame("aft", PROTOCOL_VERSION, 7),
15699 )
15700 .await;
15701
15702 let nine: BTreeMap<String, String> = (0..9)
15703 .map(|index| (format!("role-{index}"), "v1".to_string()))
15704 .collect();
15705 for (label, malformed) in [
15706 (
15707 "invalid role name",
15708 role_versions(&[("Tool_Provider", "v1")]),
15709 ),
15710 ("invalid version", role_versions(&[("tool-provider", "v0")])),
15711 ("nine entries", nine),
15712 ] {
15713 let responses = tokio::time::timeout(
15716 Duration::from_secs(1),
15717 handler.handle_control_frame(
15718 &route_ctx(ConnectionId::new(148)).0,
15719 route_open_frame_with_role_versions(
15720 404,
15721 "aft",
15722 unique_project_root("role-versions-malformed"),
15723 Some(malformed),
15724 ),
15725 ),
15726 )
15727 .await
15728 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
15729 .unwrap();
15730 assert_eq!(responses.len(), 1, "{label}");
15731 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
15732 let error = parse_error(&responses[0]);
15733 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
15734 assert_eq!(
15735 error["detail"]["field"], "role_versions",
15736 "{label}: {error}"
15737 );
15738 assert!(
15739 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
15740 "{label}: a malformed declaration is terminal"
15741 );
15742 assert!(
15743 module_rx.try_recv().is_err(),
15744 "{label}: the module must never see a bind"
15745 );
15746 }
15747 }
15748
15749 #[tokio::test]
15753 async fn route_role_versions_capability_is_advertised() {
15754 let handler = ControlHandler::new(Arc::new(Registry::default()));
15755 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
15756 let ack = hello_via_sink(
15757 &handler,
15758 &ctx,
15759 &mut rx,
15760 hello_frame("m", PROTOCOL_VERSION, 1),
15761 )
15762 .await;
15763 let ack = parse_ack(&ack);
15764 assert!(
15765 ack.subc_capabilities
15766 .iter()
15767 .any(|c| c == "route-role-versions/v1"),
15768 "{:?}",
15769 ack.subc_capabilities
15770 );
15771
15772 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15773 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15774 let reply = handler
15775 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
15776 .await
15777 .unwrap()
15778 .pop()
15779 .unwrap();
15780 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15781 serde_json::from_slice(&reply.body).unwrap()
15782 else {
15783 panic!("not a server.describe reply");
15784 };
15785 assert!(
15786 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
15787 "{capabilities:?}"
15788 );
15789 }
15790
15791 #[tokio::test]
15795 async fn catalog_events_capability_is_advertised() {
15796 let handler = ControlHandler::new(Arc::new(Registry::default()));
15797 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15798 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15799 let reply = handler
15800 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
15801 .await
15802 .unwrap()
15803 .pop()
15804 .unwrap();
15805 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15806 serde_json::from_slice(&reply.body).unwrap()
15807 else {
15808 panic!("not a server.describe reply");
15809 };
15810 assert!(
15811 capabilities
15812 .iter()
15813 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
15814 "{capabilities:?}"
15815 );
15816 }
15817
15818 #[tokio::test]
15819 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
15820 let handler = ControlHandler::new(Arc::new(Registry::default()));
15821 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
15822 let ack = hello_via_sink(
15823 &handler,
15824 &module,
15825 &mut module_rx,
15826 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
15827 )
15828 .await;
15829 let ack = parse_ack(&ack);
15830 assert!(
15831 ack.subc_capabilities
15832 .iter()
15833 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15834 "{:?}",
15835 ack.subc_capabilities
15836 );
15837 assert!(ack
15838 .subc_capabilities
15839 .iter()
15840 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15841
15842 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
15843 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15844 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15845 let reply = handler
15846 .handle_control_frame(&client, frame)
15847 .await
15848 .unwrap()
15849 .pop()
15850 .unwrap();
15851 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15852 serde_json::from_slice(&reply.body).unwrap()
15853 else {
15854 panic!("not a server.describe reply");
15855 };
15856 assert!(
15857 capabilities
15858 .iter()
15859 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15860 "{capabilities:?}"
15861 );
15862 assert!(capabilities
15863 .iter()
15864 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15865 }
15866
15867 #[tokio::test]
15868 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
15869 let handler = ControlHandler::new(Arc::new(Registry::default()));
15870 let (module, mut rx) = route_ctx(ConnectionId::new(156));
15871 let ack = hello_via_sink(
15872 &handler,
15873 &module,
15874 &mut rx,
15875 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
15876 )
15877 .await;
15878 assert!(parse_ack(&ack)
15879 .subc_capabilities
15880 .iter()
15881 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15882 assert!(parse_ack(&ack)
15883 .subc_capabilities
15884 .iter()
15885 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15886 let (client, _rx) = route_ctx(ConnectionId::new(157));
15887 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15888 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15889 let reply = handler
15890 .handle_control_frame(&client, frame)
15891 .await
15892 .unwrap()
15893 .pop()
15894 .unwrap();
15895 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15896 serde_json::from_slice(&reply.body).unwrap()
15897 else {
15898 panic!("server.describe reply")
15899 };
15900 assert!(capabilities
15901 .iter()
15902 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15903 assert!(capabilities
15904 .iter()
15905 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15906 }
15907
15908 #[tokio::test]
15909 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
15910 let handler = ControlHandler::new(Arc::new(Registry::default()));
15911 let (module, mut rx) = route_ctx(ConnectionId::new(158));
15912 let ack = hello_via_sink(
15913 &handler,
15914 &module,
15915 &mut rx,
15916 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
15917 )
15918 .await;
15919 assert!(parse_ack(&ack)
15920 .subc_capabilities
15921 .iter()
15922 .any(|cap| cap == "scope-workdir/v1"));
15923 let (client, _rx) = route_ctx(ConnectionId::new(159));
15924 let frame = Frame::build(
15925 FrameType::Request,
15926 control_flags(),
15927 0,
15928 0,
15929 5,
15930 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15931 )
15932 .unwrap();
15933 let reply = handler
15934 .handle_control_frame(&client, frame)
15935 .await
15936 .unwrap()
15937 .remove(0);
15938 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15939 serde_json::from_slice(&reply.body).unwrap()
15940 else {
15941 panic!("server.describe reply")
15942 };
15943 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
15944 }
15945
15946 #[tokio::test]
15947 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
15948 let handler = ControlHandler::new(Arc::new(Registry::default()));
15949 let (module, mut rx) = route_ctx(ConnectionId::new(160));
15950 let ack = hello_via_sink(
15951 &handler,
15952 &module,
15953 &mut rx,
15954 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
15955 )
15956 .await;
15957 assert!(parse_ack(&ack)
15958 .subc_capabilities
15959 .iter()
15960 .any(|cap| cap == "worker-runs/v1"));
15961 let (client, _rx) = route_ctx(ConnectionId::new(161));
15962 let frame = Frame::build(
15963 FrameType::Request,
15964 control_flags(),
15965 0,
15966 0,
15967 5,
15968 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15969 )
15970 .unwrap();
15971 let reply = handler
15972 .handle_control_frame(&client, frame)
15973 .await
15974 .unwrap()
15975 .remove(0);
15976 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15977 serde_json::from_slice(&reply.body).unwrap()
15978 else {
15979 panic!("server.describe reply")
15980 };
15981 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
15982 }
15983
15984 #[tokio::test]
15985 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
15986 let handler = ControlHandler::new(Arc::new(Registry::default()));
15987 let (module, mut rx) = route_ctx(ConnectionId::new(154));
15988 let ack = hello_via_sink(
15989 &handler,
15990 &module,
15991 &mut rx,
15992 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
15993 )
15994 .await;
15995 assert!(parse_ack(&ack)
15996 .subc_capabilities
15997 .iter()
15998 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
15999 let (client, _rx) = route_ctx(ConnectionId::new(155));
16000 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16001 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16002 let reply = handler
16003 .handle_control_frame(&client, frame)
16004 .await
16005 .unwrap()
16006 .pop()
16007 .unwrap();
16008 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16009 serde_json::from_slice(&reply.body).unwrap()
16010 else {
16011 panic!("not a server.describe reply")
16012 };
16013 assert!(capabilities
16014 .iter()
16015 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16016 }
16017
16018 #[tokio::test]
16019 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
16020 let registry = Arc::new(Registry::default());
16021 let forwarding = Arc::new(ForwardingTable::default());
16022 let handler =
16023 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16024 .with_health_probe_timeout(Duration::from_secs(5));
16025 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
16026 hello_via_sink(
16027 &handler,
16028 &module_ctx,
16029 &mut module_rx,
16030 non_routable_hello_frame_with_control_ops(
16031 "mcp",
16032 300,
16033 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16034 ),
16035 )
16036 .await;
16037 assert!(registry
16038 .get_module("mcp")
16039 .unwrap()
16040 .unwrap()
16041 .manifest
16042 .provides
16043 .is_empty());
16044
16045 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
16046 let route_response = handler
16047 .handle_control_frame(
16048 &route_client_ctx,
16049 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
16050 )
16051 .await
16052 .unwrap();
16053 assert_eq!(route_response[0].header.ty, FrameType::Error);
16054 assert_eq!(
16055 parse_error(&route_response[0])["code"],
16056 "target_unavailable"
16057 );
16058 assert!(parse_error(&route_response[0])["message"]
16059 .as_str()
16060 .unwrap()
16061 .contains("does not provide the requested target"));
16062 assert!(module_rx.try_recv().is_err());
16063
16064 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
16065 let health_handler = handler.clone();
16066 let health_task = tokio::spawn(async move {
16067 health_handler
16068 .handle_control_frame(
16069 &health_client_ctx,
16070 supervisor_health_probe_frame(302, "mcp"),
16071 )
16072 .await
16073 .unwrap()
16074 });
16075 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16076 .await
16077 .unwrap()
16078 .unwrap();
16079 assert_eq!(
16080 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
16081 ModuleControlRequest::HealthCheck {}
16082 );
16083 handler
16084 .handle_control_frame(
16085 &module_ctx,
16086 health_response(health_frame.header.corr, HealthStatus::Ok),
16087 )
16088 .await
16089 .unwrap();
16090 let health_response = health_task.await.unwrap();
16091 assert_eq!(health_response[0].header.ty, FrameType::Response);
16092 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
16093 ClientControlResponse::SupervisorHealthProbe {
16094 module_id, status, ..
16095 } => {
16096 assert_eq!(module_id, "mcp");
16097 assert_eq!(status, HealthStatus::Ok);
16098 }
16099 other => panic!("unexpected health response: {other:?}"),
16100 }
16101
16102 forwarding
16107 .cleanup_connection(module_ctx.connection_id)
16108 .unwrap();
16109 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
16110 let cleanup_response = tokio::time::timeout(
16111 Duration::from_millis(200),
16112 handler.handle_control_frame(
16113 &cleanup_probe_ctx,
16114 supervisor_health_probe_frame(303, "mcp"),
16115 ),
16116 )
16117 .await
16118 .expect("probe should fail immediately when the control lane is gone")
16119 .unwrap();
16120 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
16121 assert_eq!(
16122 parse_error(&cleanup_response[0])["code"],
16123 "target_unavailable"
16124 );
16125 assert!(parse_error(&cleanup_response[0])["message"]
16126 .as_str()
16127 .unwrap()
16128 .contains("no module connection"));
16129
16130 handler
16131 .cleanup_connection(module_ctx.connection_id)
16132 .unwrap();
16133 }
16134
16135 #[tokio::test]
16136 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
16137 let registry = Arc::new(Registry::default());
16138 let supervisor_handle = SupervisorHandle::new();
16139 let supervisor =
16140 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
16141 .with_handle(supervisor_handle.clone())
16142 .with_connection_file_path(
16143 std::env::temp_dir()
16144 .join(format!("subc-route-open-warming-{}", std::process::id())),
16145 );
16146 let module = supervisor
16147 .supervise_configured(
16148 ModuleSpec {
16149 carry_containment: false,
16150 module_id: "warming".to_string(),
16151 program: fake_aft_stub_path(),
16152 args: Vec::new(),
16153 env: Vec::new(),
16154 reserved: false,
16155 reserved_prefixes: Vec::new(),
16156 protocol: ModuleProtocol::Subc,
16157 overlap: Default::default(),
16158 },
16159 true,
16160 )
16161 .unwrap();
16162 assert_eq!(module.state().unwrap(), ModuleState::Running);
16163
16164 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16165 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
16166 let response = handler
16167 .handle_control_frame(
16168 &ctx,
16169 route_open_frame(304, "warming", unique_project_root("warming")),
16170 )
16171 .await
16172 .unwrap();
16173 module.stop().await.unwrap();
16174
16175 assert_eq!(response[0].header.ty, FrameType::Error);
16176 let error = parse_error(&response[0]);
16177 assert_eq!(error["code"], "module_warming");
16178 assert!(error["message"]
16179 .as_str()
16180 .unwrap()
16181 .contains("state=running, enabled=true, live=false"));
16182 }
16183
16184 #[test]
16185 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
16186 let handler = ControlHandler::new(Arc::new(Registry::default()));
16187 let capture = EventCapture::default();
16188 let _subscriber =
16189 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16190 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
16191 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
16192 let pending = (0..limit).collect::<Vec<_>>();
16193 let response = handler
16194 .route_open_capacity_refusal(
16195 &ctx,
16196 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
16197 "busy",
16198 pending.len(),
16199 limit,
16200 )
16201 .unwrap();
16202 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16203 let event = capture
16204 .events()
16205 .into_iter()
16206 .find(|event| {
16207 event.target == "control"
16208 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
16209 })
16210 .expect("connection admission refusal event");
16211 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16212 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16213 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
16216 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
16217 }
16218
16219 #[test]
16220 fn route_open_target_cap_logs_admission_reason_and_capacity() {
16221 let handler = ControlHandler::new(Arc::new(Registry::default()));
16222 let capture = EventCapture::default();
16223 let _subscriber =
16224 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16225 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
16226 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
16227 let guards = (0..limit)
16228 .map(|_| {
16229 handler
16230 .route_bind_concurrency
16231 .try_admit("busy", limit)
16232 .unwrap()
16233 })
16234 .collect::<Vec<_>>();
16235 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
16236 Err(in_flight) => in_flight,
16237 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
16238 };
16239 let response = handler
16240 .route_open_target_capacity_refusal(
16241 &ctx,
16242 &RouteOpenCaller::unknown(),
16243 &route_open_frame(397, "busy", unique_project_root("target-cap")),
16244 "busy",
16245 in_flight,
16246 )
16247 .unwrap();
16248 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16249 let event = capture
16250 .events()
16251 .into_iter()
16252 .find(|event| {
16253 event.target == "control"
16254 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
16255 })
16256 .expect("target admission refusal event");
16257 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16258 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16259 drop(guards);
16260 }
16261
16262 #[cfg(unix)]
16263 #[tokio::test(start_paused = true)]
16264 async fn supervisor_list_shows_a_failed_modules_next_retry() {
16265 let registry = Arc::new(Registry::default());
16266 let handle = SupervisorHandle::new();
16267 let supervisor =
16268 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
16269 .with_handle(handle.clone());
16270 let module = supervisor
16271 .spawn(crate::ModuleSpec {
16272 carry_containment: false,
16273 module_id: "retry-list".into(),
16274 program: PathBuf::from("/bin/sh"),
16275 args: vec!["-c".into(), "exit 17".into()],
16276 env: vec![],
16277 reserved: false,
16278 reserved_prefixes: vec![],
16279 protocol: ModuleProtocol::None,
16280 overlap: Default::default(),
16281 })
16282 .unwrap();
16283 let deadline = std::time::Instant::now() + Duration::from_secs(10);
16284 while module.state().unwrap() != crate::ModuleState::Failed {
16285 assert!(
16286 std::time::Instant::now() < deadline,
16287 "waiting for budget failure"
16288 );
16289 tokio::task::yield_now().await;
16290 }
16291 let handler = ControlHandler::new(registry).with_supervisor(handle);
16292 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16293 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16294 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16295 assert_eq!(body["modules"][0]["state"], "failed");
16296 let retry = body["modules"][0]["next_retry_at_ms"]
16297 .as_u64()
16298 .expect("scheduled retry on the wire");
16299 let now = std::time::SystemTime::now()
16300 .duration_since(std::time::UNIX_EPOCH)
16301 .unwrap()
16302 .as_millis() as u64;
16303 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
16304 module.set_enabled(false).await.unwrap();
16305 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16306 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16307 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16308 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
16309 }
16310
16311 #[tokio::test]
16312 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
16313 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
16314 let supervisor =
16315 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
16316 let module = supervisor
16317 .spawn(crate::ModuleSpec {
16318 carry_containment: false,
16319 module_id: "pid-read-budget".into(),
16320 program: std::env::current_exe().unwrap(),
16321 args: vec![
16322 "--exact".into(),
16323 "control::tests::pid_resource_read_fixture".into(),
16324 ],
16325 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16326 .into_iter()
16327 .map(|key| (key.into(), dir.display().to_string()))
16328 .chain(std::iter::once((
16329 "SUBC_SUBTREE_PID_FIXTURE".into(),
16330 "1".into(),
16331 )))
16332 .collect(),
16333 reserved: false,
16334 reserved_prefixes: Vec::new(),
16335 protocol: ModuleProtocol::None,
16336 overlap: Default::default(),
16337 })
16338 .unwrap();
16339 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16340 while !matches!(
16341 module.child_resource_usage(),
16342 subc_control::ChildResourceUsage::Measured(_)
16343 ) {
16344 assert!(
16345 tokio::time::Instant::now() < ready_deadline,
16346 "fixture process was not measurable"
16347 );
16348 tokio::time::sleep(Duration::from_millis(10)).await;
16349 }
16350 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
16351 let reading = {
16352 let _guard = crate::subtree_resources::fault_read_guard().await;
16353 crate::subtree_resources::read_process(module.clone()).await
16354 };
16355 stop_resource_fixture(&module, &dir).await;
16356 assert!(
16357 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
16358 "{reading:?}"
16359 );
16360 }
16361
16362 #[test]
16363 fn pid_resource_read_fixture() {
16364 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
16365 let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
16366 let deadline = std::time::Instant::now() + Duration::from_secs(60);
16367 while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
16368 std::thread::sleep(Duration::from_millis(5));
16369 }
16370 std::fs::write(dir.join("module-stopped"), "").unwrap();
16371 }
16372 }
16373
16374 async fn stop_resource_fixture(
16375 module: &crate::supervise::SupervisedModule,
16376 dir: &std::path::Path,
16377 ) {
16378 let stopping = module.stop();
16379 tokio::pin!(stopping);
16380 tokio::select! {
16381 result = &mut stopping => result.unwrap(),
16382 () = async {
16383 while module.state().unwrap() != crate::supervise::ModuleState::Draining {
16384 tokio::time::sleep(Duration::from_millis(5)).await;
16385 }
16386 std::fs::write(dir.join("stop-module"), "").unwrap();
16387 } => stopping.await.unwrap(),
16388 }
16389 #[cfg(windows)]
16390 assert!(
16391 dir.join("module-stopped").exists(),
16392 "the resource fixture must exit cooperatively, not wait for the drain kill"
16393 );
16394 assert_eq!(
16395 module.state().unwrap(),
16396 crate::supervise::ModuleState::Stopped
16397 );
16398 }
16399
16400 #[tokio::test]
16401 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
16402 let registry = Arc::new(Registry::default());
16403 let handle = SupervisorHandle::new();
16404 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16405 .with_handle(handle.clone());
16406 supervisor
16407 .supervise_configured(
16408 crate::ModuleSpec {
16409 module_id: "disabled-resource-reading".into(),
16410 program: std::env::current_exe().unwrap(),
16411 args: Vec::new(),
16412 env: Vec::new(),
16413 reserved: false,
16414 reserved_prefixes: Vec::new(),
16415 protocol: ModuleProtocol::None,
16416 overlap: Default::default(),
16417 carry_containment: false,
16418 },
16419 false,
16420 )
16421 .unwrap();
16422 let handler = ControlHandler::new(registry).with_supervisor(handle);
16423 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16424 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16425 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16426 assert!(body["modules"][0].get("tree").is_none());
16427 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
16428 }
16429
16430 #[tokio::test]
16431 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
16432 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
16433 let registry = Arc::new(Registry::default());
16434 let handle = SupervisorHandle::new();
16435 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16436 .with_handle(handle.clone());
16437 let module = supervisor
16438 .spawn(crate::ModuleSpec {
16439 carry_containment: false,
16440 module_id: "list-process-priority".into(),
16441 program: std::env::current_exe().unwrap(),
16442 args: vec![
16443 "--exact".into(),
16444 "control::tests::pid_resource_read_fixture".into(),
16445 ],
16446 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16447 .into_iter()
16448 .map(|key| (key.into(), dir.display().to_string()))
16449 .chain(std::iter::once((
16450 "SUBC_SUBTREE_PID_FIXTURE".into(),
16451 "1".into(),
16452 )))
16453 .collect(),
16454 reserved: false,
16455 reserved_prefixes: Vec::new(),
16456 protocol: ModuleProtocol::None,
16457 overlap: Default::default(),
16458 })
16459 .unwrap();
16460 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16461 loop {
16462 let image = module.running_image_agreement().await;
16463 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
16464 break;
16465 }
16466 assert!(
16467 tokio::time::Instant::now() < ready_deadline,
16468 "the fixture process never became observable: {image:?}"
16469 );
16470 tokio::time::sleep(Duration::from_millis(10)).await;
16471 }
16472 let guard = crate::subtree_resources::fault_read_guard().await;
16473 let (target, release, _) = crate::subtree_resources::held();
16474 module.set_subtree_target_for_test(target);
16475 let handler = ControlHandler::new(registry).with_supervisor(handle);
16476 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16477 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16478 drop(release);
16479 drop(guard);
16480 stop_resource_fixture(&module, &dir).await;
16481 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16482 assert_eq!(
16483 body["modules"][0]["resources"]["status"], "measured",
16484 "the optional tree timeout must not discard the process reading: {body}"
16485 );
16486 assert!(body["modules"][0].get("tree").is_none());
16487 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
16488 assert_eq!(
16489 body["modules"][0]["pending_reload"]["image"]["status"], "match",
16490 "the tree timeout must not hide a completed image observation"
16491 );
16492 }
16493
16494 #[cfg(any(target_os = "linux", target_os = "macos"))]
16495 #[tokio::test]
16496 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
16497 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
16498 .with_daemon_provenance(
16499 std::process::id(),
16500 0,
16501 Some(std::env::current_exe().unwrap()),
16502 None,
16503 None,
16504 );
16505 let (probe, release, began) = ExecutableIdentityProbe::held();
16506 handler.daemon_provenance.probe = probe;
16507 let observe = |handler: ControlHandler, corr| {
16508 tokio::spawn(async move {
16509 let (ctx, _) = route_ctx(ConnectionId::new(corr));
16510 let frame =
16511 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
16512 .unwrap();
16513 handler
16514 .handle_supervisor_provenance(&ctx, frame, None)
16515 .await
16516 .unwrap()
16517 })
16518 };
16519 let explicit = observe(handler.clone(), 114);
16520 began.await.unwrap();
16521 let concurrent = observe(handler.clone(), 115);
16522 tokio::time::sleep(Duration::from_millis(300)).await;
16525 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16526 let list = tokio::time::timeout(
16527 Duration::from_secs(1),
16528 handler.handle_supervisor_list(frame),
16529 )
16530 .await;
16531 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
16532 drop(release);
16533 let replies = explicit.await.unwrap();
16534 let concurrent_replies = concurrent.await.unwrap();
16535 assert!(
16536 explicit_was_pending,
16537 "explicit provenance must await the full observation"
16538 );
16539 let list = list
16540 .expect("a concurrent explicit probe must not stall list")
16541 .unwrap();
16542 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
16543 assert_eq!(list_body["op"], "supervisor.list");
16544 for replies in [replies, concurrent_replies] {
16545 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16546 assert_eq!(
16547 body["daemon"]["daemon_observed"]["running_image"]["status"],
16548 "match"
16549 );
16550 }
16551 }
16552
16553 #[tokio::test]
16554 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
16555 let registry = Arc::new(Registry::default());
16556 let handle = SupervisorHandle::new();
16557 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16558 .with_handle(handle.clone());
16559 let module = supervisor
16560 .supervise_configured(
16561 crate::ModuleSpec {
16562 carry_containment: false,
16563 module_id: "memory-read".into(),
16564 program: PathBuf::from("disabled-module"),
16565 args: Vec::new(),
16566 env: Vec::new(),
16567 reserved: false,
16568 reserved_prefixes: Vec::new(),
16569 protocol: ModuleProtocol::Subc,
16570 overlap: Default::default(),
16571 },
16572 true,
16573 )
16574 .unwrap();
16575 let _guard = crate::subtree_resources::fault_read_guard().await;
16576 let (target, release, began) = crate::subtree_resources::held();
16577 let stalled_target = target.clone();
16580 let stalled = tokio::task::spawn_blocking(move || {
16581 stalled_target.tree(&subc_os::ProcessTable::default())
16582 });
16583 began.await.unwrap();
16584 module.set_subtree_target_for_test(target);
16585 let handler = ControlHandler::new(registry).with_supervisor(handle);
16586 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
16587 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16588 let replies = handler
16589 .handle_client_control_request(
16590 &ctx,
16591 frame,
16592 ClientControlRequest::SupervisorList {},
16593 &mut None,
16594 )
16595 .await
16596 .unwrap();
16597 assert!(
16598 replies.is_empty(),
16599 "list reply must be detached from the connection reader"
16600 );
16601 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16602 let replies = handler
16603 .handle_client_control_request(
16604 &ctx,
16605 frame,
16606 ClientControlRequest::SupervisorSpawnSnapshot {},
16607 &mut None,
16608 )
16609 .await
16610 .unwrap();
16611 assert_eq!(
16612 replies.len(),
16613 1,
16614 "another request must complete while the subtree read stalls"
16615 );
16616 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16617 .await
16618 .unwrap()
16619 .unwrap();
16620 release.send(()).unwrap();
16621 drop(release);
16622 stalled.await.unwrap();
16623 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
16624 assert!(
16625 body["modules"][0].get("tree").is_none(),
16626 "an unknown tree reading must be omitted after the bounded read expires"
16627 );
16628 handler
16632 .resource_tracker
16633 .read(handler.supervisor.list(), false, "released_tree_fixture")
16634 .await;
16635 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16636 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16637 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16638 assert_eq!(
16639 body["modules"][0]["tree"]["source"], "process_tree",
16640 "the next request must export the completed refresh"
16641 );
16642 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
16643 }
16644
16645 #[tokio::test]
16646 async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
16647 use subc_control::{
16648 AgentCommandResources, ChildMemoryKind, ChildResourceReading,
16649 ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
16650 TreeReading, TreeSource,
16651 };
16652 let registry = Arc::new(Registry::default());
16653 let handle = SupervisorHandle::new();
16654 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16655 .with_handle(handle.clone());
16656 supervisor
16657 .supervise_configured(
16658 crate::ModuleSpec {
16659 carry_containment: false,
16660 module_id: "budget-worker".into(),
16661 program: PathBuf::from("disabled-module"),
16662 args: Vec::new(),
16663 env: Vec::new(),
16664 reserved: false,
16665 reserved_prefixes: Vec::new(),
16666 protocol: ModuleProtocol::Subc,
16667 overlap: Default::default(),
16668 },
16669 true,
16670 )
16671 .unwrap();
16672 let handler = ControlHandler::new(registry).with_supervisor(handle);
16673 let now = u64::try_from(
16674 std::time::SystemTime::now()
16675 .duration_since(std::time::UNIX_EPOCH)
16676 .unwrap()
16677 .as_millis(),
16678 )
16679 .unwrap();
16680 let measured = now - 1_000;
16681 let hour = CpuLastHour::new(100, 60_000);
16682 let daemon = DaemonResources::new(7, 1, "test")
16683 .with_measured_at_ms(Some(measured))
16684 .with_resources(Some(ChildResourceUsage::Measured(
16685 ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
16686 )))
16687 .with_cpu_last_hour(Some(hour.clone()));
16688 let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
16689 .with_measured_at_ms(Some(measured))
16690 .with_cpu_last_hour(Some(hour.clone()));
16691 let tree = TreeReading::new(
16692 150,
16693 50,
16694 ChildMemoryKind::ResidentSet,
16695 1,
16696 TreeSource::ProcessTree,
16697 )
16698 .with_measured_at_ms(Some(measured));
16699 handler.resource_tracker.seed_committed_for_test(
16700 &handler.supervisor.list(),
16701 crate::resource_totals::Readings {
16702 modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
16703 daemon: Some(daemon),
16704 agent_commands: Some(vec![agents]),
16705 ..Default::default()
16706 },
16707 now,
16708 );
16709 list_diagnostics::scope(async {
16710 tokio::time::sleep(Duration::from_millis(110)).await;
16711 assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
16712 let request =
16713 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16714 let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
16715 let ClientControlResponse::SupervisorList {
16716 modules,
16717 daemon,
16718 agent_commands,
16719 ..
16720 } = serde_json::from_slice(&replies[0].body).unwrap()
16721 else {
16722 panic!("expected a list response")
16723 };
16724 assert!(matches!(
16725 modules[0].resources,
16726 Some(ChildResourceUsage::Unavailable {
16727 reason: ChildResourceUnavailableReason::Unreadable
16728 })
16729 ));
16730 assert_eq!(
16731 (
16732 modules[0].tree.as_ref().unwrap().cpu_ms,
16733 modules[0].tree.as_ref().unwrap().measured_at_ms
16734 ),
16735 (150, Some(measured))
16736 );
16737 let daemon = daemon.unwrap();
16738 let agents = agent_commands.unwrap();
16739 assert_eq!(daemon.measured_at_ms, Some(measured));
16740 assert_eq!(
16741 (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
16742 (1, 50, Some(measured))
16743 );
16744 for hour in [
16745 modules[0].cpu_last_hour.as_ref(),
16746 daemon.cpu_last_hour.as_ref(),
16747 agents[0].cpu_last_hour.as_ref(),
16748 ] {
16749 let hour = hour.unwrap();
16750 assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
16751 }
16752 })
16753 .await;
16754 }
16755
16756 #[tokio::test]
16757 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
16758 let registry = Arc::new(Registry::default());
16759 let handle = SupervisorHandle::new();
16760 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16761 .with_handle(handle.clone());
16762 supervisor
16763 .supervise_configured(
16764 crate::ModuleSpec {
16765 carry_containment: false,
16766 module_id: "drift".into(),
16767 program: PathBuf::from("disabled-module"),
16768 args: Vec::new(),
16769 env: Vec::new(),
16770 reserved: false,
16771 reserved_prefixes: Vec::new(),
16772 protocol: ModuleProtocol::Subc,
16773 overlap: Default::default(),
16774 },
16775 false,
16776 )
16777 .unwrap();
16778 let handler = ControlHandler::new(registry).with_supervisor(handle);
16779 for configured in [None, Some(false), Some(true)] {
16780 handler.capability_evaluator.configure(
16781 configured.map(|enabled| ("drift".to_string(), enabled)),
16782 BTreeMap::new(),
16783 );
16784 let frame =
16785 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16786 let replies = handler
16787 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
16788 .await
16789 .unwrap();
16790 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16791 assert!(body.get("configured_enabled").is_none());
16792 assert_eq!(body["op"], "supervisor.ack");
16793 let frame =
16794 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16795 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16796 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16797 assert_eq!(
16798 body["modules"][0].get("configured_enabled"),
16799 configured.map(Value::Bool).as_ref()
16800 );
16801 assert!(body["modules"][0].get("tree").is_none());
16802 assert!(body["modules"][0].get("subtree").is_none());
16803 }
16804 }
16805
16806 #[tokio::test(flavor = "current_thread")]
16807 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
16808 let handler = ControlHandler::new(Arc::new(Registry::default()));
16809 let capture = EventCapture::default();
16810 let _subscriber =
16811 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16812 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
16813 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16814
16815 let _ = handler
16816 .handle_client_control_request(
16817 &ctx,
16818 frame,
16819 ClientControlRequest::SupervisorSetEnabled {
16820 module_id: "broca".to_string(),
16821 enabled: false,
16822 },
16823 &mut None,
16824 )
16825 .await
16826 .unwrap();
16827
16828 let events = capture
16829 .events()
16830 .into_iter()
16831 .filter(|event| {
16832 event.target == "control"
16833 && event.fields.get("message").map(String::as_str)
16834 == Some("supervisor request received")
16835 })
16836 .collect::<Vec<_>>();
16837 assert_eq!(events.len(), 1, "one supervisor request log line");
16838 let event = &events[0];
16839 assert_eq!(event.level, tracing::Level::INFO);
16840 assert_eq!(
16841 event.fields.get("op"),
16842 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
16843 );
16844 assert_eq!(
16845 event.fields.get("module_id"),
16846 Some(&"\"broca\"".to_string())
16847 );
16848 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
16849 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
16850 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
16851 }
16852
16853 #[tokio::test(flavor = "current_thread")]
16854 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
16855 let registry = Arc::new(Registry::default());
16856 let module_connection = ConnectionId::new(303);
16857 registry
16858 .register_with_control_ops(
16859 manifest("aft", PROTOCOL_VERSION),
16860 PROTOCOL_VERSION,
16861 module_connection,
16862 module_baseline_control_ops(),
16863 )
16864 .unwrap();
16865 let handler = ControlHandler::new(registry);
16866 let capture = EventCapture::default();
16867 let _subscriber =
16868 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16869 let (ctx, _rx) = route_ctx(module_connection);
16870 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16871
16872 let _ = handler
16873 .handle_client_control_request(
16874 &ctx,
16875 frame,
16876 ClientControlRequest::SupervisorSetEnabled {
16877 module_id: "broca".to_string(),
16878 enabled: false,
16879 },
16880 &mut None,
16881 )
16882 .await
16883 .unwrap();
16884
16885 let callers = capture
16886 .events()
16887 .into_iter()
16888 .filter(|event| {
16889 event.fields.get("message").map(String::as_str)
16890 == Some("supervisor request received")
16891 })
16892 .map(|event| event.fields.get("caller").cloned())
16893 .collect::<Vec<_>>();
16894 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
16895 }
16896
16897 #[tokio::test(flavor = "current_thread")]
16898 async fn supervisor_list_does_not_log_request_received() {
16899 let handler = ControlHandler::new(Arc::new(Registry::default()));
16900 let capture = EventCapture::default();
16901 let _subscriber =
16902 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16903 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
16904 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16905
16906 handler
16907 .handle_client_control_request(
16908 &ctx,
16909 frame,
16910 ClientControlRequest::SupervisorList {},
16911 &mut None,
16912 )
16913 .await
16914 .unwrap();
16915
16916 assert!(capture.events().into_iter().all(|event| {
16917 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
16918 }));
16919 }
16920
16921 #[tokio::test(flavor = "current_thread")]
16922 async fn supervisor_rescan_preview_does_not_log_request_received() {
16923 let handler = ControlHandler::new(Arc::new(Registry::default()));
16924 let capture = EventCapture::default();
16925 let _subscriber =
16926 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16927 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
16928 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16929
16930 handler
16931 .handle_client_control_request(
16932 &ctx,
16933 frame,
16934 ClientControlRequest::SupervisorRescan { preview: true },
16935 &mut None,
16936 )
16937 .await
16938 .unwrap();
16939
16940 assert!(capture.events().into_iter().all(|event| {
16941 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
16942 }));
16943 }
16944
16945 struct ApproveOperator;
16946 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
16947 fn prompt(
16948 &self,
16949 _: &str,
16950 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
16951 ) -> crate::operator_confirm::ProviderResult {
16952 crate::operator_confirm::ProviderResult::Approved
16953 }
16954 }
16955
16956 #[tokio::test]
16957 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
16958 assert_eq!(
16959 ControlHandler::default().privileged_operator_prompt,
16960 PrivilegedOperatorPrompt::Log
16961 );
16962 for (initial, changed, running) in [
16963 ("log", "enforce", PrivilegedOperatorPrompt::Log),
16964 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
16965 ] {
16966 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
16967 let path = dir.join("subc.jsonc");
16968 std::fs::write(
16969 &path,
16970 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
16971 )
16972 .unwrap();
16973 let config = crate::daemon_config::load(&path).unwrap().unwrap();
16974 let registry = Arc::new(Registry::default());
16975 let handle = SupervisorHandle::new();
16976 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
16977 .with_handle(handle.clone());
16978 let handler = ControlHandler::new(registry)
16979 .with_supervisor(handle)
16980 .with_privileged_operator_prompt(config.privileged_operator_prompt)
16981 .with_supervisor_rescan(supervisor, &path, config.port);
16982 handler.forwarding.operator_confirms().configure(
16983 Arc::new(ApproveOperator),
16984 crate::operator_confirm::OperatorLimits::default(),
16985 );
16986 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
16987
16988 for (disk_mode, needs_restart) in [
16989 (Some(initial), false),
16990 (Some(changed), true),
16991 (Some(changed), true),
16992 (None, running == PrivilegedOperatorPrompt::Enforce),
16993 (Some(initial), false),
16994 ] {
16995 let mut doc = json!({"version":1});
16996 if let Some(mode) = disk_mode {
16997 doc["privileged_operator_prompt"] = json!(mode);
16998 }
16999 std::fs::write(&path, doc.to_string()).unwrap();
17000 for preview in [true, false] {
17001 let request = ClientControlRequest::SupervisorRescan { preview };
17002 let frame = Frame::build(
17003 FrameType::Request,
17004 control_flags(),
17005 0,
17006 0,
17007 1,
17008 serde_json::to_vec(&request).unwrap(),
17009 )
17010 .unwrap();
17011 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17012 assert_eq!(frames.len(), 1);
17013 let ClientControlResponse::SupervisorRescan { result } =
17014 serde_json::from_slice(&frames[0].body).unwrap()
17015 else {
17016 panic!("expected a rescan response");
17017 };
17018 let expected = if needs_restart {
17019 vec!["privileged_operator_prompt"]
17020 } else {
17021 Vec::new()
17022 };
17023 assert_eq!(result.restart_required, expected);
17024 assert_eq!(handler.privileged_operator_prompt, running);
17025 assert_eq!(
17026 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
17027 running
17028 );
17029 }
17030 }
17031 }
17032 }
17033
17034 #[tokio::test]
17035 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
17036 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
17037 let path = dir.join("subc.jsonc");
17038 let registry = Arc::new(Registry::default());
17039 let handle = SupervisorHandle::new();
17040 let supervisor =
17041 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
17042 let handler = ControlHandler::new(registry)
17043 .with_supervisor(handle)
17044 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
17045 .with_supervisor_rescan(supervisor, &path, None);
17046 handler.forwarding.operator_confirms().configure(
17047 Arc::new(ApproveOperator),
17048 crate::operator_confirm::OperatorLimits::default(),
17049 );
17050 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
17051 for preview in [true, false] {
17052 std::fs::write(
17053 &path,
17054 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
17055 )
17056 .unwrap();
17057 let request = ClientControlRequest::SupervisorRescan { preview };
17058 let frame = Frame::build(
17059 FrameType::Request,
17060 control_flags(),
17061 0,
17062 0,
17063 1,
17064 serde_json::to_vec(&request).unwrap(),
17065 )
17066 .unwrap();
17067 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17068 assert_eq!(frames.len(), 1);
17069 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
17070 assert_eq!(
17071 handler.privileged_operator_prompt,
17072 PrivilegedOperatorPrompt::Enforce
17073 );
17074 }
17075 }
17076
17077 #[tokio::test]
17082 async fn route_open_refusal_names_the_check_that_refused() {
17083 let handler = ControlHandler::new(Arc::new(Registry::default()));
17084 let capture = EventCapture::default();
17085 let _subscriber =
17086 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17087 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17088 let response = handler
17089 .handle_control_frame(
17090 &ctx,
17091 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
17092 )
17093 .await
17094 .unwrap();
17095
17096 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17097 let event = capture
17098 .events()
17099 .into_iter()
17100 .find(|event| {
17101 event.target == "control"
17102 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17103 })
17104 .expect("route.open refusal event");
17105 assert_eq!(
17106 event.fields.get("reason"),
17107 Some(&"\"not_registered\"".to_string())
17108 );
17109 }
17110
17111 #[tokio::test]
17112 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
17113 let registry = Arc::new(Registry::default());
17114 let supervisor_handle = SupervisorHandle::new();
17115 let supervisor =
17116 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17117 .with_handle(supervisor_handle.clone())
17118 .with_connection_file_path(std::env::temp_dir().join(format!(
17119 "subc-route-open-refusal-info-{}",
17120 std::process::id()
17121 )));
17122 let module = supervisor
17123 .supervise_configured(
17124 ModuleSpec {
17125 carry_containment: false,
17126 module_id: "warming".to_string(),
17127 program: fake_aft_stub_path(),
17128 args: Vec::new(),
17129 env: Vec::new(),
17130 reserved: false,
17131 reserved_prefixes: Vec::new(),
17132 protocol: ModuleProtocol::Subc,
17133 overlap: Default::default(),
17134 },
17135 true,
17136 )
17137 .unwrap();
17138 assert_eq!(module.state().unwrap(), ModuleState::Running);
17139
17140 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17141 assert!(handler
17142 .counters()
17143 .snapshot()
17144 .get("route_open_refused_by_code")
17145 .is_none());
17146 let capture = EventCapture::default();
17147 let _subscriber =
17148 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17149 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
17150 let response = handler
17151 .handle_control_frame(
17152 &ctx,
17153 route_open_frame(394, "warming", unique_project_root("refusal-info")),
17154 )
17155 .await
17156 .unwrap();
17157 module.stop().await.unwrap();
17158
17159 assert_eq!(parse_error(&response[0])["code"], "module_warming");
17160 let event = capture
17161 .events()
17162 .into_iter()
17163 .find(|event| {
17164 event.target == "control"
17165 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
17166 })
17167 .expect("route.open refusal event");
17168 assert_eq!(
17169 event.fields.get("module_id"),
17170 Some(&"\"warming\"".to_string())
17171 );
17172 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
17173 assert_eq!(
17174 event.fields.get("reason"),
17175 Some(&"\"supervised_not_registered\"".to_string())
17176 );
17177 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
17178 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
17179 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
17180 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
17181 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
17182 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
17183 assert_eq!(
17184 handler.counters().snapshot()["route_open_refused_by_code"],
17185 json!({ "module_warming": 1 })
17186 );
17187 }
17188
17189 const OUTAGE_START: &str = "route.open refusing module: not serving";
17190 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
17191
17192 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
17193 capture
17194 .events()
17195 .into_iter()
17196 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
17197 .collect()
17198 }
17199
17200 fn supervise_stub(
17201 registry: &Arc<Registry>,
17202 module_id: &str,
17203 enabled: bool,
17204 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
17205 let supervisor_handle = SupervisorHandle::new();
17206 let supervisor =
17207 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
17208 .with_handle(supervisor_handle.clone())
17209 .with_connection_file_path(std::env::temp_dir().join(format!(
17210 "subc-route-outage-{module_id}-{}",
17211 std::process::id()
17212 )));
17213 let module = supervisor
17214 .supervise_configured(
17215 ModuleSpec {
17216 carry_containment: false,
17217 module_id: module_id.to_string(),
17218 program: fake_aft_stub_path(),
17219 args: Vec::new(),
17220 env: Vec::new(),
17221 reserved: false,
17222 reserved_prefixes: Vec::new(),
17223 protocol: ModuleProtocol::Subc,
17224 overlap: Default::default(),
17225 },
17226 enabled,
17227 )
17228 .unwrap();
17229 (supervisor_handle, module)
17230 }
17231
17232 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
17233 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
17234 module_id: module_id.to_string(),
17235 drain_timeout_ms: Some(50),
17236 })
17237 .unwrap();
17238 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
17239 }
17240
17241 #[test]
17245 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
17246 let registry = Arc::new(Registry::default());
17247 let forwarding = Arc::new(ForwardingTable::default());
17248 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17249 let second = ControlHandler::with_forwarding(registry, forwarding);
17250 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
17251 }
17252
17253 #[tokio::test(flavor = "current_thread")]
17257 async fn route_open_unknown_module_refusals_add_no_outage_state() {
17258 let handler = ControlHandler::new(Arc::new(Registry::default()));
17259 let capture = EventCapture::default();
17260 let _subscriber =
17261 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17262 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
17263 for corr in 0..8 {
17264 let response = handler
17265 .handle_control_frame(
17266 &ctx,
17267 route_open_frame(
17268 380 + corr,
17269 &format!("nobody-{corr}"),
17270 unique_project_root("outage-unknown"),
17271 ),
17272 )
17273 .await
17274 .unwrap();
17275 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17276 }
17277
17278 assert_eq!(handler.route_outages.tracked_module_count(), 0);
17279 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
17280 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
17281 }
17282
17283 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17288 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
17289 let registry = Arc::new(Registry::default());
17290 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
17291 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17292 let capture = EventCapture::default();
17293 let _subscriber =
17294 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17295 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
17296 handler.route_outages.record_accepted("outage-restart");
17299
17300 let response = handler
17301 .handle_control_frame(
17302 &ctx,
17303 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
17304 )
17305 .await
17306 .unwrap();
17307 assert_eq!(response[0].header.ty, FrameType::Error);
17308 let starts = outage_lines(&capture, OUTAGE_START);
17309 assert_eq!(starts.len(), 1, "{starts:?}");
17310 assert_eq!(starts[0].level, tracing::Level::WARN);
17311 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
17312 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
17313 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
17314 handler.route_outages.record_accepted("outage-restart");
17315 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
17316
17317 let restart = handler
17318 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
17319 .await
17320 .unwrap();
17321 assert_eq!(
17322 restart[0].header.ty,
17323 FrameType::Response,
17324 "{:?}",
17325 parse_error(&restart[0])
17326 );
17327 handler
17328 .handle_control_frame(
17329 &ctx,
17330 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
17331 )
17332 .await
17333 .unwrap();
17334 module.stop().await.unwrap();
17335
17336 let starts = outage_lines(&capture, OUTAGE_START);
17337 assert_eq!(starts.len(), 2, "{starts:?}");
17338 assert_eq!(starts[1].level, tracing::Level::INFO);
17339 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
17340 }
17341
17342 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17346 async fn failed_operator_restart_leaves_no_operator_mark() {
17347 let registry = Arc::new(Registry::default());
17348 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
17349 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17350 let capture = EventCapture::default();
17351 let _subscriber =
17352 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17353 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
17354 handler.route_outages.record_accepted("outage-disabled");
17355
17356 let restart = handler
17357 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
17358 .await
17359 .unwrap();
17360 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
17361 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
17362
17363 handler
17364 .handle_control_frame(
17365 &ctx,
17366 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
17367 )
17368 .await
17369 .unwrap();
17370 let starts = outage_lines(&capture, OUTAGE_START);
17371 assert_eq!(starts.len(), 1, "{starts:?}");
17372 assert_eq!(starts[0].level, tracing::Level::WARN);
17373 }
17374
17375 #[tokio::test(flavor = "current_thread")]
17376 async fn route_open_unknown_module_escapes_target_module_id() {
17377 let handler = ControlHandler::new(Arc::new(Registry::default()));
17378 let capture = EventCapture::default();
17379 let _subscriber =
17380 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17381 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
17382 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17383 let response = handler
17384 .handle_control_frame(
17385 &ctx,
17386 route_open_frame(
17387 395,
17388 hostile_module_id,
17389 unique_project_root("hostile-target-module-id"),
17390 ),
17391 )
17392 .await
17393 .unwrap();
17394
17395 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17396 let event = capture
17397 .events()
17398 .into_iter()
17399 .find(|event| {
17400 event.target == "control"
17401 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17402 })
17403 .expect("route.open unknown-module refusal event");
17404 let logged = event.fields.get("module_id").expect("module_id field");
17405 assert!(!logged.bytes().any(|byte| byte < 0x20));
17406 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17407 }
17408
17409 #[tokio::test(flavor = "current_thread")]
17412 async fn route_open_refusal_carries_daemon_origin() {
17413 let handler = ControlHandler::new(Arc::new(Registry::default()));
17414 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17415 let response = handler
17416 .handle_control_frame(
17417 &ctx,
17418 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
17419 )
17420 .await
17421 .unwrap();
17422
17423 assert_eq!(response.len(), 1);
17424 assert_eq!(response[0].header.ty, FrameType::Error);
17425 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17426 assert!(
17427 response[0].header.flags.is_daemon_origin(),
17428 "route.open refusal must carry daemon origin: {:?}",
17429 response[0].header.flags
17430 );
17431 }
17432
17433 #[tokio::test(flavor = "current_thread")]
17437 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
17438 let registry = Arc::new(Registry::default());
17439 let forwarding = Arc::new(ForwardingTable::default());
17440 let handler =
17441 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17442 let module_connection = ConnectionId::new(95);
17443 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17444 hello_via_sink(
17445 &handler,
17446 &module_ctx,
17447 &mut module_rx,
17448 hello_frame("aft", PROTOCOL_VERSION, 395),
17449 )
17450 .await;
17451 forwarding
17452 .begin_module_drain("aft", RouteCloseReason::Restart)
17453 .unwrap()
17454 .expect("the module endpoint drains");
17455
17456 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
17457 let response = handler
17458 .handle_control_frame(
17459 &client_ctx,
17460 route_open_frame(396, "aft", unique_project_root("drain-origin")),
17461 )
17462 .await
17463 .unwrap();
17464
17465 assert_eq!(response.len(), 1);
17466 assert_eq!(response[0].header.ty, FrameType::Error);
17467 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17468 assert!(
17469 response[0].header.flags.is_daemon_origin(),
17470 "drain module_reloading must carry daemon origin: {:?}",
17471 response[0].header.flags
17472 );
17473 }
17474
17475 #[tokio::test(flavor = "current_thread")]
17479 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
17480 let registry = Arc::new(Registry::default());
17481 let forwarding = Arc::new(ForwardingTable::default());
17482 let handler =
17483 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17484 let module_connection = ConnectionId::new(95);
17485 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17486 hello_via_sink(
17487 &handler,
17488 &module_ctx,
17489 &mut module_rx,
17490 hello_frame("aft", PROTOCOL_VERSION, 395),
17491 )
17492 .await;
17493
17494 let client_connection = ConnectionId::new(96);
17495 let (client_ctx, _client_rx) = route_ctx(client_connection);
17496 let (route_task, bind) = relay_route_open(
17497 &handler,
17498 client_connection,
17499 &client_ctx.egress,
17500 &mut module_rx,
17501 396,
17502 "aft",
17503 "spoofed-reloading",
17504 )
17505 .await;
17506 let rejection = Frame::build(
17509 FrameType::Error,
17510 crate::router::daemon_error_flags(),
17511 0,
17512 0,
17513 bind.header.corr,
17514 serde_json::to_vec(&ErrorBody::new(
17515 "module_reloading",
17516 "module refused route.bind",
17517 ))
17518 .unwrap(),
17519 )
17520 .unwrap();
17521 handler
17522 .handle_control_frame(&module_ctx, rejection)
17523 .await
17524 .unwrap();
17525
17526 let response = route_task.await.unwrap();
17527 assert_eq!(response.len(), 1);
17528 assert_eq!(response[0].header.ty, FrameType::Error);
17529 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17530 assert!(
17531 !response[0].header.flags.is_daemon_origin(),
17532 "a module's own rejection must not carry daemon origin: {:?}",
17533 response[0].header.flags
17534 );
17535 }
17536
17537 #[tokio::test(flavor = "current_thread")]
17538 async fn route_open_module_rejection_uses_daemon_counter_key() {
17539 let registry = Arc::new(Registry::default());
17540 let forwarding = Arc::new(ForwardingTable::default());
17541 let handler =
17542 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17543 let module_connection = ConnectionId::new(95);
17544 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17545 hello_via_sink(
17546 &handler,
17547 &module_ctx,
17548 &mut module_rx,
17549 hello_frame("aft", PROTOCOL_VERSION, 395),
17550 )
17551 .await;
17552
17553 let client_connection = ConnectionId::new(96);
17554 let (client_ctx, _client_rx) = route_ctx(client_connection);
17555 let capture = EventCapture::default();
17556 static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
17563 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
17564 let _subscriber =
17565 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17566 let (route_task, bind) = relay_route_open(
17567 &handler,
17568 client_connection,
17569 &client_ctx.egress,
17570 &mut module_rx,
17571 396,
17572 "aft",
17573 "hostile-module-code",
17574 )
17575 .await;
17576 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
17577 let rejection = Frame::build(
17578 FrameType::Error,
17579 control_flags(),
17580 0,
17581 0,
17582 bind.header.corr,
17583 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
17584 )
17585 .unwrap();
17586 handler
17587 .handle_control_frame(&module_ctx, rejection)
17588 .await
17589 .unwrap();
17590
17591 let response = route_task.await.unwrap();
17592 assert_eq!(parse_error(&response[0])["code"], hostile_code);
17593 let counters = handler.counters().snapshot();
17594 assert_eq!(
17595 counters["route_open_refused_by_code"],
17596 json!({ "module_rejected": 1 })
17597 );
17598 assert!(counters["route_open_refused_by_code"]
17599 .get(hostile_code)
17600 .is_none());
17601
17602 let event = capture
17603 .events()
17604 .into_iter()
17605 .find(|event| {
17606 event.target == "control"
17607 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
17608 })
17609 .expect("route.open module-rejection refusal event");
17610 let logged = event.fields.get("module_code").expect("module_code field");
17611 assert!(!logged.bytes().any(|byte| byte < 0x20));
17612 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17613 }
17614
17615 #[tokio::test]
17616 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
17617 let registry = Arc::new(Registry::default());
17618 let supervisor_handle = SupervisorHandle::new();
17619 let missing_program = std::env::temp_dir().join(format!(
17620 "subc-route-open-missing-program-{}",
17621 std::process::id()
17622 ));
17623 let supervisor =
17624 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17625 .with_handle(supervisor_handle.clone());
17626 let module = supervisor
17627 .supervise_configured(
17628 ModuleSpec {
17629 carry_containment: false,
17630 module_id: "failed".to_string(),
17631 program: missing_program,
17632 args: Vec::new(),
17633 env: Vec::new(),
17634 reserved: false,
17635 reserved_prefixes: Vec::new(),
17636 protocol: ModuleProtocol::Subc,
17637 overlap: Default::default(),
17638 },
17639 true,
17640 )
17641 .unwrap();
17642 assert_eq!(module.state().unwrap(), ModuleState::Failed);
17643
17644 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17645 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
17646 let response = handler
17647 .handle_control_frame(
17648 &ctx,
17649 route_open_frame(305, "failed", unique_project_root("failed")),
17650 )
17651 .await
17652 .unwrap();
17653
17654 assert_eq!(response[0].header.ty, FrameType::Error);
17655 let error = parse_error(&response[0]);
17656 assert_eq!(error["code"], "target_unavailable");
17657 assert!(error["message"]
17658 .as_str()
17659 .unwrap()
17660 .contains("state=failed, enabled=true, live=false"));
17661 }
17662
17663 #[tokio::test]
17664 async fn route_open_role_mismatch_remains_target_unavailable() {
17665 let registry = Arc::new(Registry::default());
17666 let handler = ControlHandler::new(Arc::clone(®istry));
17667 handler
17668 .handle_control(
17669 ConnectionId::new(41),
17670 non_routable_hello_frame_with_control_ops("health-only", 306, None),
17671 )
17672 .unwrap();
17673
17674 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
17675 let response = handler
17676 .handle_control_frame(
17677 &ctx,
17678 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
17679 )
17680 .await
17681 .unwrap();
17682
17683 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17684 assert!(parse_error(&response[0])["message"]
17685 .as_str()
17686 .unwrap()
17687 .contains("does not provide the requested target"));
17688 }
17689
17690 #[tokio::test]
17691 async fn route_open_inactive_registration_remains_target_unavailable() {
17692 let registry = Arc::new(Registry::default());
17693 let handler = ControlHandler::new(Arc::clone(®istry));
17694 handler
17695 .handle_control(
17696 ConnectionId::new(43),
17697 hello_frame("inactive", PROTOCOL_VERSION, 308),
17698 )
17699 .unwrap();
17700 assert!(registry
17701 .set_module_state_for_test("inactive", ChannelState::Closed)
17702 .unwrap());
17703
17704 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
17705 let response = handler
17706 .handle_control_frame(
17707 &ctx,
17708 route_open_frame(309, "inactive", unique_project_root("inactive")),
17709 )
17710 .await
17711 .unwrap();
17712
17713 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17714 assert!(parse_error(&response[0])["message"]
17715 .as_str()
17716 .unwrap()
17717 .contains("is not active"));
17718 }
17719
17720 #[tokio::test]
17721 async fn late_health_reply_is_recorded_through_the_module_response_path() {
17722 let registry = Arc::new(Registry::default());
17723 let forwarding = Arc::new(ForwardingTable::default());
17724 let supervisor_handle = SupervisorHandle::new();
17725 let supervisor =
17726 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
17727 .with_forwarding(Arc::clone(&forwarding))
17728 .with_handle(supervisor_handle.clone());
17729 let module = supervisor
17730 .supervise_configured(
17731 crate::ModuleSpec {
17732 carry_containment: false,
17733 module_id: "late-health-response".to_string(),
17734 program: PathBuf::from("disabled-module"),
17735 args: Vec::new(),
17736 env: Vec::new(),
17737 reserved: false,
17738 reserved_prefixes: Vec::new(),
17739 protocol: ModuleProtocol::Subc,
17740 overlap: Default::default(),
17741 },
17742 false,
17743 )
17744 .unwrap();
17745 let handler =
17746 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17747 .with_supervisor(supervisor_handle);
17748 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
17749 handler
17750 .handle_control_frame(
17751 &module_ctx,
17752 hello_frame_with_control_ops(
17753 "late-health-response",
17754 PROTOCOL_VERSION,
17755 7,
17756 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17757 ),
17758 )
17759 .await
17760 .unwrap();
17761 let probe_started_at = Instant::now() - Duration::from_millis(80);
17762 let pending = forwarding
17763 .begin_health_probe_rpc_for(
17764 "late-health-response",
17765 MODULE_CONTROL_OP_HEALTH_CHECK,
17766 probe_started_at,
17767 Instant::now() - Duration::from_millis(1),
17768 )
17769 .unwrap();
17770 assert!(forwarding
17771 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
17772 .unwrap());
17773
17774 let responses = handler
17775 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
17776 .await
17777 .unwrap();
17778
17779 assert!(responses.is_empty());
17780 let health = module.status().unwrap().health;
17781 assert_eq!(health.late_answer_count, 1);
17782 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
17783 }
17784
17785 #[tokio::test]
17786 async fn health_probe_timeout_and_module_death_are_typed() {
17787 let registry = Arc::new(Registry::default());
17788 let forwarding = Arc::new(ForwardingTable::default());
17789 let handler =
17790 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17791 .with_health_probe_timeout(Duration::from_millis(50));
17792 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
17793 hello_via_sink(
17794 &handler,
17795 &module_ctx,
17796 &mut module_rx,
17797 hello_frame_with_control_ops(
17798 "aft",
17799 PROTOCOL_VERSION,
17800 7,
17801 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17802 ),
17803 )
17804 .await;
17805
17806 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
17807 let responses = handler
17808 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
17809 .await
17810 .unwrap();
17811 assert_eq!(responses[0].header.ty, FrameType::Error);
17812 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
17813 let _ = module_rx.try_recv();
17814
17815 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
17816 let health_handler = handler.clone();
17817 let death_task = tokio::spawn(async move {
17818 health_handler
17819 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
17820 .await
17821 .unwrap()
17822 });
17823 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
17824 .await
17825 .unwrap()
17826 .unwrap();
17827 handler
17828 .cleanup_connection(module_ctx.connection_id)
17829 .unwrap();
17830 let responses = death_task.await.unwrap();
17831 assert_eq!(responses[0].header.ty, FrameType::Error);
17832 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
17833 }
17834
17835 #[test]
17836 fn hello_requires_exact_protocol_version() {
17837 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
17838 let registry = Arc::new(Registry::default());
17839 let handler = ControlHandler::new(Arc::clone(®istry));
17840 let responses = handler
17841 .handle_control(
17842 ConnectionId::new(connection),
17843 hello_frame("aft", offered, 9),
17844 )
17845 .unwrap();
17846
17847 assert_eq!(responses.len(), 1);
17848 assert_eq!(responses[0].header.ty, FrameType::Error);
17849 let error = parse_error(&responses[0]);
17850 assert_eq!(error["code"], "version_unsupported");
17851 assert!(registry.get_module("aft").unwrap().is_none());
17852 assert_eq!(registry.active_registration_count().unwrap(), 0);
17853 }
17854 }
17855
17856 #[test]
17857 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
17858 let registry = Arc::new(Registry::default());
17859 let forwarding = Arc::new(ForwardingTable::default());
17860 let handler =
17861 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17862 let module_connection = ConnectionId::new(301);
17863 let registration = registry
17864 .register_with_control_ops(
17865 manifest("aft-push", PROTOCOL_VERSION),
17866 PROTOCOL_VERSION,
17867 module_connection,
17868 module_baseline_control_ops(),
17869 )
17870 .unwrap();
17871 let (module_tx, _module_rx) = mpsc::channel(8);
17872 let endpoint = forwarding
17873 .register_module_connection(
17874 module_connection,
17875 "aft-push".to_string(),
17876 PROTOCOL_VERSION,
17877 manifest_concurrency(®istration.manifest),
17878 FrameSink::new(module_tx),
17879 )
17880 .unwrap();
17881
17882 let unknown = Frame::build(
17884 FrameType::Push,
17885 control_flags(),
17886 0,
17887 0,
17888 5,
17889 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
17890 )
17891 .unwrap();
17892 let out = handler.handle_status_update(endpoint, unknown).unwrap();
17893 assert!(
17894 out.is_empty(),
17895 "unknown push op must be ignored, got {out:?}"
17896 );
17897
17898 let malformed = Frame::build(
17900 FrameType::Push,
17901 control_flags(),
17902 0,
17903 0,
17904 6,
17905 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
17906 )
17907 .unwrap();
17908 let out = handler.handle_status_update(endpoint, malformed).unwrap();
17909 assert_eq!(out.len(), 1);
17910 assert_eq!(out[0].header.ty, FrameType::Error);
17911 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
17912 }
17913
17914 #[test]
17915 fn hello_rejected_when_connection_already_owns_client_routes() {
17916 let registry = Arc::new(Registry::default());
17917 let forwarding = Arc::new(ForwardingTable::default());
17918 let handler =
17919 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17920 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
17922 let client_connection = ConnectionId::new(202);
17923
17924 let responses = handler
17927 .handle_control(
17928 client_connection,
17929 hello_frame("aft-second", PROTOCOL_VERSION, 9),
17930 )
17931 .unwrap();
17932 assert_eq!(responses[0].header.ty, FrameType::Error);
17933 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
17934 assert!(registry.get_module("aft-second").unwrap().is_none());
17935 }
17936
17937 #[tokio::test]
17938 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
17939 let registry = Arc::new(Registry::default());
17940 let forwarding = Arc::new(ForwardingTable::default());
17941 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
17942 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
17943 hello_via_sink(
17944 &handler,
17945 &module_ctx,
17946 &mut module_rx,
17947 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
17948 )
17949 .await;
17950 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
17951 let pending = forwarding
17952 .begin_route_bind_relay_for_test(
17953 client_ctx.connection_id,
17954 client_ctx.egress.clone(),
17955 2,
17956 "alpha",
17957 )
17958 .unwrap();
17959 forwarding
17960 .complete_pending_relay(
17961 module_ctx.connection_id,
17962 pending.corr,
17963 RouteBindRelayOutcome::Accepted,
17964 )
17965 .unwrap();
17966 client_rx.try_recv().unwrap();
17967 for module_id in ["beta", "alpha"] {
17968 let replies = handler
17969 .handle_control_frame(
17970 &module_ctx,
17971 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
17972 )
17973 .await
17974 .unwrap();
17975 assert_eq!(replies.len(), 1, "second HELLO must be refused");
17976 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
17977 }
17978 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
17979 assert!(registry.get_module("beta").unwrap().is_none());
17980 assert!(matches!(
17981 forwarding
17982 .lookup_data_route(
17983 client_ctx.connection_id,
17984 pending.client_channel,
17985 pending.client_epoch,
17986 )
17987 .unwrap(),
17988 DataRoute::Client(DataRouteState::Bound(_))
17989 ));
17990 assert!(handler
17991 .hello_launch_nonces
17992 .lock()
17993 .unwrap()
17994 .presented(module_ctx.connection_id, Some("alpha-nonce")));
17995 assert!(module_rx.try_recv().is_err());
17996 }
17997
17998 #[test]
17999 fn reserved_module_hello_requires_matching_launch_nonce() {
18000 let registry = Arc::new(Registry::default());
18001 let supervisor = SupervisorHandle::new();
18002 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
18005 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18006
18007 let no_nonce = handler
18009 .handle_control(
18010 ConnectionId::new(1),
18011 hello_frame("vault", PROTOCOL_VERSION, 1),
18012 )
18013 .unwrap();
18014 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
18015 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
18016 assert!(registry.get_module("vault").unwrap().is_none());
18017
18018 let wrong = handler
18020 .handle_control(
18021 ConnectionId::new(2),
18022 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
18023 )
18024 .unwrap();
18025 assert_eq!(wrong[0].header.ty, FrameType::Error);
18026 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
18027 assert!(registry.get_module("vault").unwrap().is_none());
18028
18029 let ok = handler
18031 .handle_control(
18032 ConnectionId::new(3),
18033 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
18034 )
18035 .unwrap();
18036 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
18037 assert!(registry.get_module("vault").unwrap().is_some());
18038 }
18039
18040 #[test]
18041 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
18042 let registry = Arc::new(Registry::default());
18043 let supervisor = SupervisorHandle::new();
18044 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18045 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18046 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18047
18048 let squat = handler
18049 .handle_control(
18050 ConnectionId::new(1),
18051 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
18052 )
18053 .unwrap();
18054 assert_eq!(squat[0].header.ty, FrameType::Error);
18055 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
18056 assert!(parse_error(&squat[0])["message"]
18057 .as_str()
18058 .unwrap()
18059 .contains("fed:"));
18060
18061 let accepted_peer = handler
18062 .handle_control(
18063 ConnectionId::new(2),
18064 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
18065 )
18066 .unwrap();
18067 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
18068
18069 let accepted_short = handler
18070 .handle_control(
18071 ConnectionId::new(3),
18072 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
18073 )
18074 .unwrap();
18075 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
18076
18077 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
18078 let response = handler
18079 .handle_control(
18080 ConnectionId::new(conn),
18081 hello_frame(module_id, PROTOCOL_VERSION, conn),
18082 )
18083 .unwrap();
18084 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
18085 }
18086 }
18087
18088 #[test]
18089 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
18090 let registry = Arc::new(Registry::default());
18091 let supervisor = SupervisorHandle::new();
18092 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18093 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18094 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
18095 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18096
18097 let owner_nonce = handler
18098 .handle_control(
18099 ConnectionId::new(1),
18100 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
18101 )
18102 .unwrap();
18103 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
18104 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
18105 assert!(registry.get_module("fed:special").unwrap().is_none());
18106
18107 let exact_nonce = handler
18108 .handle_control(
18109 ConnectionId::new(2),
18110 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
18111 )
18112 .unwrap();
18113 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
18114 assert!(registry.get_module("fed:special").unwrap().is_some());
18115 }
18116
18117 #[test]
18118 fn non_reserved_module_ignores_launch_nonce() {
18119 let registry = Arc::new(Registry::default());
18120 let handler = ControlHandler::new(Arc::clone(®istry));
18123 let no_nonce = handler
18124 .handle_control(
18125 ConnectionId::new(1),
18126 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
18127 )
18128 .unwrap();
18129 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
18130 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
18131
18132 let echoed_nonce = handler
18133 .handle_control(
18134 ConnectionId::new(2),
18135 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
18136 )
18137 .unwrap();
18138 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
18139 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
18140 }
18141
18142 #[test]
18143 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
18144 let handler = ControlHandler::default();
18145 let conn = ConnectionId::new(1);
18146 let malformed = Frame::build(
18147 FrameType::Hello,
18148 control_flags(),
18149 0,
18150 0,
18151 3,
18152 b"{not json".to_vec(),
18153 )
18154 .unwrap();
18155
18156 let error = handler.handle_control(conn, malformed).unwrap();
18157 assert_eq!(error[0].header.ty, FrameType::Error);
18158 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
18159
18160 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18161 let pong = handler.handle_control(conn, ping).unwrap();
18162 assert_eq!(pong[0].header.ty, FrameType::Pong);
18163 assert_eq!(pong[0].header.corr, 4);
18164 }
18165
18166 #[test]
18167 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
18168 let registry = Arc::new(Registry::default());
18169 let handler = ControlHandler::new(Arc::clone(®istry));
18170
18171 handler
18172 .handle_control(
18173 ConnectionId::new(1),
18174 hello_frame("aft", PROTOCOL_VERSION, 1),
18175 )
18176 .unwrap();
18177 let duplicate = handler
18178 .handle_control(
18179 ConnectionId::new(2),
18180 hello_frame("aft", PROTOCOL_VERSION, 2),
18181 )
18182 .unwrap();
18183
18184 assert_eq!(duplicate[0].header.ty, FrameType::Error);
18185 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18186 let registration = registry.get_module("aft").unwrap().unwrap();
18187 assert_eq!(registration.connection_id, ConnectionId::new(1));
18188 }
18189
18190 #[test]
18191 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
18192 let registry = Arc::new(Registry::default());
18193 let forwarding = Arc::new(ForwardingTable::default());
18194 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
18195 let handler =
18196 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18197 .with_process_liveness(process_liveness);
18198 let (ctx, route_channel, route_epoch) =
18199 bind_liveness_route(®istry, &forwarding, "aft-dead");
18200 let responses = handler
18201 .handle_route_poll(
18202 &ctx,
18203 route_poll_frame(41, PollKind::Liveness, route_channel),
18204 route_channel,
18205 route_epoch,
18206 PollKind::Liveness,
18207 )
18208 .unwrap();
18209
18210 assert_eq!(responses.len(), 1);
18211 assert_eq!(responses[0].header.ty, FrameType::Response);
18212 assert_route_poll_liveness(&responses[0], false);
18213 }
18214
18215 #[test]
18216 fn liveness_poll_without_process_source_uses_bound_route() {
18217 let registry = Arc::new(Registry::default());
18218 let forwarding = Arc::new(ForwardingTable::default());
18219 let handler =
18220 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18221 let (ctx, route_channel, route_epoch) =
18222 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
18223 let responses = handler
18224 .handle_route_poll(
18225 &ctx,
18226 route_poll_frame(42, PollKind::Liveness, route_channel),
18227 route_channel,
18228 route_epoch,
18229 PollKind::Liveness,
18230 )
18231 .unwrap();
18232
18233 assert_route_poll_liveness(&responses[0], true);
18234 }
18235
18236 #[test]
18237 fn liveness_poll_untracked_process_source_uses_bound_route() {
18238 let registry = Arc::new(Registry::default());
18239 let forwarding = Arc::new(ForwardingTable::default());
18240 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
18241 let handler =
18242 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18243 .with_process_liveness(process_liveness);
18244 let (ctx, route_channel, route_epoch) =
18245 bind_liveness_route(®istry, &forwarding, "aft-untracked");
18246 let responses = handler
18247 .handle_route_poll(
18248 &ctx,
18249 route_poll_frame(43, PollKind::Liveness, route_channel),
18250 route_channel,
18251 route_epoch,
18252 PollKind::Liveness,
18253 )
18254 .unwrap();
18255
18256 assert_route_poll_liveness(&responses[0], true);
18257 }
18258
18259 #[tokio::test]
18260 async fn unknown_op_returns_unknown_control_op() {
18261 let handler = ControlHandler::default();
18262 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
18263 let request = Frame::build(
18264 FrameType::Request,
18265 control_flags(),
18266 0,
18267 0,
18268 55,
18269 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
18270 )
18271 .unwrap();
18272
18273 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18274
18275 assert_eq!(response.len(), 1);
18276 assert_eq!(response[0].header.ty, FrameType::Error);
18277 assert_eq!(response[0].header.corr, 55);
18278 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
18279 }
18280
18281 #[tokio::test]
18282 async fn supervisor_provenance_rejects_unknown_exact_module() {
18283 let handler = ControlHandler::default();
18284 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
18285 let request = Frame::build(
18286 FrameType::Request,
18287 control_flags(),
18288 0,
18289 0,
18290 57,
18291 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
18292 )
18293 .unwrap();
18294
18295 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18296
18297 assert_eq!(response.len(), 1);
18298 assert_eq!(response[0].header.ty, FrameType::Error);
18299 assert_eq!(response[0].header.corr, 57);
18300 let error = parse_error(&response[0]);
18301 assert_eq!(error["code"], "unknown_module");
18302 assert_eq!(error["message"], "module_id 'missing' is not supervised");
18303 }
18304
18305 #[test]
18306 fn provenance_probe_override_keeps_handler_tests_deterministic() {
18307 let expected = subc_control::RunningImageAgreement::Unavailable {
18308 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18309 };
18310 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
18311 assert_eq!(handler.provenance_probe_override, Some(expected));
18312 }
18313
18314 #[test]
18315 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
18316 let verdict = reload_verdict(
18317 std::path::Path::new("/bin/new"),
18318 Some(std::path::Path::new("/bin/old")),
18319 subc_control::RunningImageAgreement::Unavailable {
18320 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18321 },
18322 );
18323 assert!(matches!(
18324 verdict.path,
18325 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
18326 if configured == std::path::Path::new("/bin/new")
18327 && spawned_from == std::path::Path::new("/bin/old")
18328 ));
18329 }
18330
18331 #[test]
18332 fn launch_spec_without_running_digest_is_unavailable_not_match() {
18333 use subc_control::{
18334 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
18335 };
18336 let configured = crate::supervise::ModuleSpec {
18337 carry_containment: false,
18338 module_id: "legacy".into(),
18339 program: "/module".into(),
18340 args: Vec::new(),
18341 env: Vec::new(),
18342 reserved: false,
18343 reserved_prefixes: Vec::new(),
18344 protocol: subc_control::ModuleProtocol::Subc,
18345 overlap: Default::default(),
18346 };
18347 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
18348 assert_eq!(
18349 launch_spec_agreement(&configured, false, Some(&digest)),
18350 Agreement::Unavailable {
18351 reason: Reason::NotRunning
18352 }
18353 );
18354 assert_eq!(
18355 launch_spec_agreement(&configured, true, None),
18356 Agreement::Unavailable {
18357 reason: Reason::SpawnedSpecUnavailable
18358 }
18359 );
18360 assert_eq!(
18361 launch_spec_agreement(&configured, true, Some(&digest)),
18362 Agreement::Match
18363 );
18364 }
18365
18366 #[cfg(windows)]
18367 #[test]
18368 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
18369 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
18370 let configured = dir.join("module.exe");
18371 std::fs::write(&configured, b"executable path fixture").unwrap();
18372 let spawned = std::fs::canonicalize(&configured).unwrap();
18373 assert_ne!(
18374 configured, spawned,
18375 "fixture must exercise different path spellings"
18376 );
18377 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
18378 let lower_case = configured.to_string_lossy().to_lowercase();
18379 for spelling in [
18380 &configured,
18381 Path::new(&forward_slashes),
18382 Path::new(&lower_case),
18383 ] {
18384 let verdict = reload_verdict(
18385 spelling,
18386 Some(&spawned),
18387 subc_control::RunningImageAgreement::Unavailable {
18388 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18389 },
18390 );
18391 assert_eq!(
18392 verdict.path,
18393 ReloadPathAgreement::Match,
18394 "configured {spelling:?}, spawned {spawned:?}"
18395 );
18396 }
18397 }
18398
18399 #[cfg(windows)]
18400 #[test]
18401 fn windows_reload_path_keeps_different_existing_program_pending() {
18402 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
18403 let old = dir.join("old.exe");
18404 let new = dir.join("new.exe");
18405 std::fs::write(&old, b"same executable content").unwrap();
18406 std::fs::copy(&old, &new).unwrap();
18407 let spawned = std::fs::canonicalize(&old).unwrap();
18408 let image = subc_control::RunningImageAgreement::Unavailable {
18409 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18410 };
18411 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
18412 assert!(
18413 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
18414 if configured == new && spawned_from == spawned)
18415 );
18416 let missing = dir.join("missing.exe");
18417 assert!(matches!(
18418 reload_verdict(&missing, Some(&spawned), image).path,
18419 ReloadPathAgreement::Mismatch { .. }
18420 ));
18421 }
18422
18423 #[test]
18424 fn reload_verdict_detects_replaced_image_at_same_path() {
18425 let image = subc_control::RunningImageAgreement::Mismatch {
18426 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
18427 digest: "old".into(),
18428 },
18429 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
18430 digest: "new".into(),
18431 },
18432 };
18433 let verdict = reload_verdict(
18434 std::path::Path::new("/bin/same"),
18435 Some(std::path::Path::new("/bin/same")),
18436 image.clone(),
18437 );
18438 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18439 assert_eq!(verdict.image, image);
18440 }
18441
18442 #[test]
18443 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
18444 let image = subc_control::RunningImageAgreement::Unavailable {
18445 reason: subc_control::RunningImageUnavailableReason::NotRunning,
18446 };
18447 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
18448 assert_eq!(
18449 verdict.path,
18450 subc_control::ReloadPathAgreement::Unavailable {
18451 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
18452 }
18453 );
18454 assert_eq!(verdict.image, image);
18455
18456 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
18457 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18458 };
18459 let verdict = reload_verdict(
18460 std::path::Path::new("/bin/same"),
18461 Some(std::path::Path::new("/bin/same")),
18462 unconfirmed.clone(),
18463 );
18464 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18465 assert_eq!(verdict.image, unconfirmed);
18466 }
18467
18468 #[test]
18469 fn reload_verdict_preserves_each_image_unavailability_reason() {
18470 use subc_control::RunningImageUnavailableReason as Reason;
18471
18472 for reason in [
18473 Reason::NotRunning,
18474 Reason::UnsupportedPlatform,
18475 Reason::RunningExecutableUnreadable,
18476 Reason::SpawnedPathUnreadable,
18477 Reason::HashFailed,
18478 Reason::ProcessIdentityUnconfirmed,
18479 Reason::Unknown("future_probe_reason".to_string()),
18480 ] {
18481 let image = subc_control::RunningImageAgreement::Unavailable {
18482 reason: reason.clone(),
18483 };
18484 let verdict = reload_verdict(
18485 std::path::Path::new("/bin/same"),
18486 Some(std::path::Path::new("/bin/same")),
18487 image.clone(),
18488 );
18489 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18490 assert_eq!(verdict.image, image, "{reason:?}");
18491 }
18492 }
18493
18494 #[tokio::test]
18495 async fn malformed_control_bodies_return_invalid_control_body() {
18496 let handler = ControlHandler::default();
18497 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
18498
18499 for (corr, body) in [
18500 (56, br#"{"route_channel":1}"#.as_slice()),
18501 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
18502 (
18503 58,
18504 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
18505 ),
18506 ] {
18507 let request = Frame::build(
18508 FrameType::Request,
18509 control_flags(),
18510 0,
18511 0,
18512 corr,
18513 body.to_vec(),
18514 )
18515 .unwrap();
18516 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18517
18518 assert_eq!(response.len(), 1);
18519 assert_eq!(response[0].header.ty, FrameType::Error);
18520 assert_eq!(response[0].header.corr, corr);
18521 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
18522 }
18523 }
18524
18525 #[tokio::test]
18526 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
18527 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18528 let registry = Arc::new(Registry::default());
18529 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18530 let router = Router::with_control_handler(Arc::clone(&control));
18531 let connection = router.begin_connection();
18532 let (ctx, mut rx) = route_ctx(connection.id());
18533
18534 router
18535 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
18536 .await
18537 .unwrap();
18538 let response = rx.recv().await.unwrap();
18539 let ack = parse_ack(&response);
18540 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18541 let channel = 1;
18542
18543 let goodbye =
18544 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
18545 router.route_for_connection(&ctx, goodbye).await.unwrap();
18546 assert!(rx.try_recv().is_err());
18547 assert!(registry.get_module("aft").unwrap().is_none());
18548
18549 router
18550 .route_for_connection(&ctx, channel_request(channel, 13))
18551 .await
18552 .unwrap();
18553 let error_frame = rx.recv().await.unwrap();
18554 assert_eq!(error_frame.header.ty, FrameType::Error);
18555 assert_eq!(error_frame.header.channel, channel);
18556 let captured = crate::router::test_log::captured_logs(&logs);
18557 assert_eq!(
18558 captured
18559 .lines()
18560 .filter(|line| {
18561 line.contains("module_id=aft")
18562 && line.contains("reason=explicit_goodbye")
18563 && line.contains("module registration ended")
18564 })
18565 .count(),
18566 1,
18567 "unexpected GOODBYE registry log: {captured}"
18568 );
18569 }
18570
18571 #[tokio::test]
18572 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
18573 let registry = Arc::new(Registry::default());
18574 let handler = ControlHandler::new(registry).with_capability_config(
18575 [("prov".to_string(), true), ("cons".to_string(), true)],
18576 BTreeMap::new(),
18577 );
18578 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
18579 register_capability_manifest(
18580 &handler,
18581 &provider_ctx,
18582 &mut provider_rx,
18583 capability_manifest("prov", &["thing/v1"], &[]),
18584 1,
18585 )
18586 .await;
18587 let mut consumer = capability_manifest("cons", &[], &[]);
18588 consumer.capabilities.as_mut().unwrap().requires.push(
18589 subc_protocol::manifest::CapabilityRequirement {
18590 capability: "thing/v1".to_string(),
18591 need: subc_protocol::manifest::CapabilityNeed::Required,
18592 },
18593 );
18594 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
18595 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
18596 assert_eq!(
18597 handler.capability_evaluator.verdict("cons", "thing/v1"),
18598 Some(CapabilityVerdict::Provided)
18599 );
18600 let (mut client_rx, _) = open_route_for_capability_test(
18601 &handler,
18602 &provider_ctx,
18603 &mut provider_rx,
18604 703,
18605 3,
18606 "prov",
18607 None,
18608 )
18609 .await;
18610 let goodbye =
18611 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18612 handler
18613 .handle_control_frame(&provider_ctx, goodbye)
18614 .await
18615 .unwrap();
18616 assert_eq!(
18617 handler.capability_evaluator.verdict("cons", "thing/v1"),
18618 Some(CapabilityVerdict::NeverProvided)
18619 );
18620 let closed = client_rx
18621 .try_recv()
18622 .expect("GOODBYE pushes route.closed before route GOODBYE");
18623 assert!(
18624 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
18625 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
18626 );
18627 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
18628 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
18629 }
18630
18631 struct RouteClosedFixture {
18632 handler: ControlHandler,
18633 #[cfg(unix)]
18634 supervisor: Supervisor,
18635 module_ctx: RouteCtx,
18636 #[cfg(unix)]
18637 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18638 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18639 route: Arc<crate::forwarding::RouteBinding>,
18640 }
18641
18642 async fn route_closed_fixture() -> RouteClosedFixture {
18643 let registry = Arc::new(Registry::default());
18644 let handler = ControlHandler::new(Arc::clone(®istry));
18645 #[cfg(unix)]
18646 let supervisor = Supervisor::new(registry, RestartPolicy::default())
18647 .with_forwarding(Arc::clone(&handler.forwarding));
18648 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
18649 register_capability_manifest(
18650 &handler,
18651 &module_ctx,
18652 &mut module_rx,
18653 capability_manifest("prov", &[], &[]),
18654 1,
18655 )
18656 .await;
18657 let (client_rx, bind) = open_route_for_capability_test(
18658 &handler,
18659 &module_ctx,
18660 &mut module_rx,
18661 702,
18662 2,
18663 "prov",
18664 None,
18665 )
18666 .await;
18667 let ModuleControlRequest::RouteBind {
18668 route_channel: channel,
18669 epoch: route_epoch,
18670 ..
18671 } = bind
18672 else {
18673 panic!("expected route.bind");
18674 };
18675 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
18676 handler
18677 .forwarding
18678 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
18679 .unwrap()
18680 else {
18681 panic!("the acknowledged route must be bound");
18682 };
18683 RouteClosedFixture {
18684 handler,
18685 #[cfg(unix)]
18686 supervisor,
18687 module_ctx,
18688 #[cfg(unix)]
18689 module_rx,
18690 client_rx,
18691 route,
18692 }
18693 }
18694
18695 impl RouteClosedFixture {
18696 #[cfg(unix)]
18697 async fn cut(&mut self) {
18698 let notices = async {
18699 let mut draining = self.module_rx.recv().await.unwrap();
18700 assert!(matches!(
18701 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
18702 &draining.body
18703 )
18704 .unwrap(),
18705 subc_protocol::session::ModuleControlCommand::Draining {
18706 reason: RouteCloseReason::Disable,
18707 ..
18708 }
18709 ));
18710 draining.flushed.take().unwrap().send(()).unwrap();
18711 let mut closing = self.client_rx.recv().await.unwrap();
18712 assert!(matches!(
18713 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
18714 ClientControlPush::RouteClosing {
18715 reason: RouteCloseReason::Restart,
18716 ..
18717 }
18718 ));
18719 closing.flushed.take().unwrap().send(()).unwrap();
18720 };
18721 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
18722 result.unwrap();
18723 }
18724
18725 fn disconnect(&mut self) -> ClientControlPush {
18726 self.handler
18727 .cleanup_connection(self.module_ctx.connection_id)
18728 .unwrap();
18729 let closed = self.client_rx.try_recv().unwrap();
18730 assert_eq!(closed.header.ty, FrameType::Push);
18731 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
18732 assert!(matches!(
18733 &push,
18734 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
18735 if channels == &[self.route.client_channel]
18736 ));
18737 assert_eq!(
18738 self.client_rx.try_recv().unwrap().header.ty,
18739 FrameType::Goodbye
18740 );
18741 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
18742 push
18743 }
18744 }
18745
18746 #[cfg(unix)]
18747 #[tokio::test(start_paused = true)]
18748 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
18749 let mut fixture = route_closed_fixture().await;
18750 fixture.cut().await;
18751 assert!(matches!(
18752 fixture.disconnect(),
18753 ClientControlPush::RouteClosed {
18754 reason: RouteCloseReason::Restart,
18755 drained: true,
18756 ..
18757 }
18758 ));
18759 }
18760
18761 #[cfg(unix)]
18762 #[tokio::test(start_paused = true)]
18763 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
18764 let mut fixture = route_closed_fixture().await;
18765 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
18766 let started = tokio::time::Instant::now();
18767 fixture.cut().await;
18768 assert!(started.elapsed() >= Duration::from_secs(2));
18769 assert_eq!(fixture.route.flow.in_flight(), 1);
18770 assert!(matches!(
18771 fixture.disconnect(),
18772 ClientControlPush::RouteClosed {
18773 reason: RouteCloseReason::Restart,
18774 drained: false,
18775 ..
18776 }
18777 ));
18778 }
18779
18780 #[tokio::test]
18781 async fn module_connection_crash_reports_crash_and_not_drained() {
18782 let mut fixture = route_closed_fixture().await;
18783 assert!(matches!(
18787 fixture.disconnect(),
18788 ClientControlPush::RouteClosed {
18789 reason: RouteCloseReason::Crash,
18790 drained: false,
18791 ..
18792 }
18793 ));
18794 }
18795
18796 #[tokio::test]
18797 async fn dropping_router_connection_releases_registration() {
18798 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18799 let registry = Arc::new(Registry::default());
18800 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18801 let router = Router::with_control_handler(Arc::clone(&control));
18802 let connection = router.begin_connection();
18803 let connection_id = connection.id();
18804 let (ctx, mut rx) = route_ctx(connection_id);
18805
18806 router
18807 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
18808 .await
18809 .unwrap();
18810 let response = rx.recv().await.unwrap();
18811 let ack = parse_ack(&response);
18812 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18813 assert!(registry.get_module("aft").unwrap().is_some());
18814
18815 drop(connection);
18816
18817 assert!(registry.get_module("aft").unwrap().is_none());
18818 assert_eq!(registry.active_registration_count().unwrap(), 0);
18819
18820 control
18821 .cleanup_connection(ConnectionId::new(u64::MAX))
18822 .unwrap();
18823 let captured = crate::router::test_log::captured_logs(&logs);
18824 println!("captured registry lifecycle logs:\n{captured}");
18825 let events: Vec<_> = captured
18826 .lines()
18827 .filter(|line| {
18828 line.contains("module registered module_id=aft ")
18829 || line.contains("module registration ended")
18830 })
18831 .collect();
18832 assert_eq!(
18833 events.len(),
18834 2,
18835 "unexpected registry lifecycle logs: {captured}"
18836 );
18837 assert!(events[0].contains("module registered module_id=aft "));
18838 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
18839 assert!(events[1].contains(&format!(
18840 "module_id=aft connection_id={}",
18841 connection_id.get()
18842 )));
18843 assert!(events[1].contains("reason=connection_closed"));
18844 assert!(events[1].contains("module registration ended"));
18845 assert_eq!(
18846 captured
18847 .lines()
18848 .filter(|line| line.contains("module registered module_id=aft "))
18849 .count(),
18850 1,
18851 "legacy registration admission line must appear once: {captured}"
18852 );
18853 }
18854
18855 #[tokio::test]
18856 async fn hello_registration_keeps_legacy_module_registered_line_once() {
18857 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18858 let registry = Arc::new(Registry::default());
18859 let control = ControlHandler::new(Arc::clone(®istry));
18860 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
18861 hello_via_sink(
18862 &control,
18863 &ctx,
18864 &mut rx,
18865 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
18866 )
18867 .await;
18868
18869 let captured = crate::router::test_log::captured_logs(&logs);
18870 let admissions: Vec<_> = captured
18871 .lines()
18872 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
18873 .collect();
18874 assert_eq!(
18875 admissions.len(),
18876 1,
18877 "legacy admission line must remain exactly once: {captured}"
18878 );
18879 assert!(admissions[0].contains("routable_provider=true"));
18880 assert!(admissions[0].contains("connection_id=777"));
18881 }
18882
18883 fn capability_manifest(
18884 module_id: &str,
18885 provides: &[&str],
18886 must_never_reach: &[&str],
18887 ) -> ModuleManifest {
18888 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
18889 manifest.capabilities = Some(CapabilityDeclarations {
18890 provides: provides
18891 .iter()
18892 .map(|capability| (*capability).to_string())
18893 .collect(),
18894 requires: Vec::new(),
18895 must_never_reach: must_never_reach
18896 .iter()
18897 .map(|capability| (*capability).to_string())
18898 .collect(),
18899 });
18900 manifest
18901 }
18902
18903 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
18904 Frame::build(
18905 FrameType::Hello,
18906 control_flags(),
18907 0,
18908 0,
18909 corr,
18910 serde_json::to_vec(&ModuleHelloBody {
18911 protocol_ver: manifest.protocol_ver,
18912 manifest,
18913 control_ops: None,
18914 launch_nonce: None,
18915 })
18916 .expect("capability test HELLO serializes"),
18917 )
18918 .expect("capability test HELLO frame builds")
18919 }
18920
18921 fn catalog_update_with_capabilities_frame(
18922 corr: u64,
18923 capabilities: CapabilityDeclarations,
18924 ) -> Frame {
18925 Frame::build(
18926 FrameType::Request,
18927 control_flags(),
18928 0,
18929 0,
18930 corr,
18931 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
18932 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
18933 capabilities: Some(capabilities),
18934 ready: None,
18935 })
18936 .expect("capability catalog.update serializes"),
18937 )
18938 .expect("capability catalog.update frame builds")
18939 }
18940
18941 async fn register_capability_manifest(
18942 handler: &ControlHandler,
18943 ctx: &RouteCtx,
18944 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
18945 manifest: ModuleManifest,
18946 corr: u64,
18947 ) {
18948 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
18949 }
18950
18951 async fn open_route_for_capability_test(
18952 handler: &ControlHandler,
18953 target_ctx: &RouteCtx,
18954 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
18955 client_connection_id: u64,
18956 corr: u64,
18957 target_module_id: &str,
18958 consumer_identity: Option<ConsumerIdentity>,
18959 ) -> (
18960 mpsc::Receiver<crate::router::OutboundFrame>,
18961 ModuleControlRequest,
18962 ) {
18963 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
18964 let route_handler = handler.clone();
18965 let target_module_id = target_module_id.to_string();
18966 let route_task = tokio::spawn(async move {
18967 route_handler
18968 .handle_control_frame(
18969 &client_ctx,
18970 route_open_frame_with_admission_facts(
18971 corr,
18972 &target_module_id,
18973 unique_project_root("admission-facts"),
18974 consumer_identity,
18975 None,
18976 ),
18977 )
18978 .await
18979 .expect("capability test route.open succeeds")
18980 });
18981 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
18982 .await
18983 .expect("capability test route.open must reach route.bind")
18984 .expect("target control receiver stays open");
18985 let bind_request: ModuleControlRequest =
18986 serde_json::from_slice(&bind.body).expect("route.bind decodes");
18987 handler
18988 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
18989 .await
18990 .expect("capability test route.bind ACK succeeds");
18991 assert!(route_task.await.expect("route.open task joins").is_empty());
18992 let opened = client_rx
18993 .recv()
18994 .await
18995 .expect("successful route.open publishes a response");
18996 assert!(matches!(
18997 serde_json::from_slice::<ClientControlResponse>(&opened.body),
18998 Ok(ClientControlResponse::RouteOpen { .. })
18999 ));
19000 (client_rx, bind_request)
19001 }
19002
19003 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
19004 assert_eq!(frame.header.ty, FrameType::Push);
19005 assert_eq!(frame.header.channel, 0);
19006 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
19007 .expect("route.closed control push decodes");
19008 let ClientControlPush::RouteClosed { channels, .. } = &push else {
19009 panic!("expected route.closed");
19010 };
19011 assert_eq!(channels.len(), 1, "exactly one violating route closed");
19012 let closed_channel = channels[0];
19013 let channels = channels.clone();
19014 assert_eq!(
19015 push,
19016 ClientControlPush::RouteClosed {
19017 module_id: target_module_id.to_string(),
19018 channels,
19019 reason: RouteCloseReason::CapabilityDenied,
19020 drained: false,
19021 abandoned: 0,
19022 excluded_subscriptions: 0,
19023 terminal: Some(false),
19024 }
19025 );
19026 closed_channel
19027 }
19028
19029 #[tokio::test]
19030 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
19031 let registry = Arc::new(Registry::default());
19032 let forwarding = Arc::new(ForwardingTable::default());
19033 let supervisor = SupervisorHandle::new();
19034 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19035 let handler =
19036 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19037 .with_supervisor(supervisor);
19038 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
19039 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
19040 register_capability_manifest(
19041 &handler,
19042 &target_ctx,
19043 &mut target_rx,
19044 capability_manifest("target", &["credentials-provider/v1"], &[]),
19045 1,
19046 )
19047 .await;
19048 register_capability_manifest(
19049 &handler,
19050 &opener_ctx,
19051 &mut opener_rx,
19052 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19053 2,
19054 )
19055 .await;
19056
19057 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
19058 let replies = handler
19059 .handle_control_frame(
19060 &client_ctx,
19061 route_open_frame_with_admission_facts(
19062 3,
19063 "target",
19064 unique_project_root("admission-facts"),
19065 Some(ConsumerIdentity {
19066 module_id: "opener".to_string(),
19067 launch_nonce: "opener-nonce".to_string(),
19068 }),
19069 None,
19070 ),
19071 )
19072 .await
19073 .expect("denied route.open returns a typed frame");
19074 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19075 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19076 assert!(
19077 target_rx.try_recv().is_err(),
19078 "forbidden route.open must not relay route.bind"
19079 );
19080 }
19081
19082 #[tokio::test]
19083 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
19084 let registry = Arc::new(Registry::default());
19085 let forwarding = Arc::new(ForwardingTable::default());
19086 let supervisor = SupervisorHandle::new();
19087 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19088 let handler =
19089 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19090 .with_supervisor(supervisor);
19091 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
19092 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
19093 register_capability_manifest(
19094 &handler,
19095 &target_ctx,
19096 &mut target_rx,
19097 capability_manifest("target", &["credentials-provider/v1"], &[]),
19098 1,
19099 )
19100 .await;
19101 register_capability_manifest(
19102 &handler,
19103 &old_opener_ctx,
19104 &mut old_opener_rx,
19105 capability_manifest("opener", &[], &[]),
19106 2,
19107 )
19108 .await;
19109 let (mut client_rx, _) = open_route_for_capability_test(
19110 &handler,
19111 &target_ctx,
19112 &mut target_rx,
19113 712,
19114 3,
19115 "target",
19116 Some(ConsumerIdentity {
19117 module_id: "opener".to_string(),
19118 launch_nonce: "opener-nonce".to_string(),
19119 }),
19120 )
19121 .await;
19122 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19123
19124 handler
19125 .cleanup_connection(old_opener_ctx.connection_id)
19126 .expect("old opener registration cleans up");
19127 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
19128 register_capability_manifest(
19129 &handler,
19130 &new_opener_ctx,
19131 &mut new_opener_rx,
19132 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19133 4,
19134 )
19135 .await;
19136
19137 let closed_channel = assert_capability_denied_push(
19138 client_rx
19139 .try_recv()
19140 .expect("HELLO deny addition must emit route.closed")
19141 .frame,
19142 "target",
19143 );
19144 let goodbye = client_rx
19145 .try_recv()
19146 .expect("HELLO deny addition must send the client a route GOODBYE")
19147 .frame;
19148 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19149 assert_eq!(goodbye.header.channel, closed_channel);
19150 assert_ne!(goodbye.header.channel, 0);
19151 assert_ne!(goodbye.header.epoch, 0);
19152 assert!(client_rx.try_recv().is_err());
19153 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19154 assert!(matches!(
19155 target_rx.try_recv(),
19156 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19157 ));
19158 }
19159
19160 #[tokio::test]
19161 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
19162 let registry = Arc::new(Registry::default());
19163 let forwarding = Arc::new(ForwardingTable::default());
19164 let supervisor = SupervisorHandle::new();
19165 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19166 let handler =
19167 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19168 .with_supervisor(supervisor);
19169 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
19170 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
19171 register_capability_manifest(
19172 &handler,
19173 &target_ctx,
19174 &mut target_rx,
19175 capability_manifest("target", &[], &[]),
19176 1,
19177 )
19178 .await;
19179 register_capability_manifest(
19180 &handler,
19181 &opener_ctx,
19182 &mut opener_rx,
19183 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19184 2,
19185 )
19186 .await;
19187 let (mut client_rx, _) = open_route_for_capability_test(
19188 &handler,
19189 &target_ctx,
19190 &mut target_rx,
19191 722,
19192 3,
19193 "target",
19194 Some(ConsumerIdentity {
19195 module_id: "opener".to_string(),
19196 launch_nonce: "opener-nonce".to_string(),
19197 }),
19198 )
19199 .await;
19200 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19201
19202 let replies = handler
19203 .handle_control_frame(
19204 &target_ctx,
19205 catalog_update_with_capabilities_frame(
19206 4,
19207 CapabilityDeclarations {
19208 provides: vec!["credentials-provider/v1".to_string()],
19209 requires: Vec::new(),
19210 must_never_reach: Vec::new(),
19211 },
19212 ),
19213 )
19214 .await
19215 .expect("claim catalog.update succeeds");
19216 assert!(matches!(
19217 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
19218 Ok(ModuleControlResponseToModule::CatalogUpdate {})
19219 ));
19220 let closed_channel = assert_capability_denied_push(
19221 client_rx
19222 .try_recv()
19223 .expect("claim addition must emit route.closed")
19224 .frame,
19225 "target",
19226 );
19227 let goodbye = client_rx
19228 .try_recv()
19229 .expect("claim addition must send the client a route GOODBYE")
19230 .frame;
19231 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19232 assert_eq!(goodbye.header.channel, closed_channel);
19233 assert_ne!(goodbye.header.channel, 0);
19234 assert_ne!(goodbye.header.epoch, 0);
19235 assert!(client_rx.try_recv().is_err());
19236 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19237 assert!(matches!(
19238 target_rx.try_recv(),
19239 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19240 ));
19241 }
19242
19243 #[tokio::test]
19244 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
19245 let registry = Arc::new(Registry::default());
19246 let forwarding = Arc::new(ForwardingTable::default());
19247 let supervisor = SupervisorHandle::new();
19248 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19249 let handler =
19250 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19251 .with_supervisor(supervisor);
19252 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
19253 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
19254 register_capability_manifest(
19255 &handler,
19256 &target_ctx,
19257 &mut target_rx,
19258 capability_manifest("target", &["credentials-provider/v1"], &[]),
19259 1,
19260 )
19261 .await;
19262 register_capability_manifest(
19263 &handler,
19264 &opener_ctx,
19265 &mut opener_rx,
19266 capability_manifest("opener", &[], &[]),
19267 2,
19268 )
19269 .await;
19270 let (mut client_rx, _) = open_route_for_capability_test(
19271 &handler,
19272 &target_ctx,
19273 &mut target_rx,
19274 732,
19275 3,
19276 "target",
19277 Some(ConsumerIdentity {
19278 module_id: "opener".to_string(),
19279 launch_nonce: "opener-nonce".to_string(),
19280 }),
19281 )
19282 .await;
19283 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19284
19285 handler
19286 .handle_control_frame(
19287 &target_ctx,
19288 catalog_update_with_capabilities_frame(
19289 4,
19290 CapabilityDeclarations {
19291 provides: Vec::new(),
19292 requires: Vec::new(),
19293 must_never_reach: Vec::new(),
19294 },
19295 ),
19296 )
19297 .await
19298 .expect("claim removal catalog.update succeeds");
19299 assert_eq!(
19300 forwarding.active_binding_count().unwrap(),
19301 1,
19302 "removing an attested target claim must leave the route census unchanged"
19303 );
19304 assert!(
19305 client_rx.try_recv().is_err(),
19306 "claim removal must not emit route.closed or a client route GOODBYE"
19307 );
19308 assert!(
19309 target_rx.try_recv().is_err(),
19310 "claim removal must not send the target a route GOODBYE"
19311 );
19312 }
19313
19314 #[tokio::test]
19317 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
19318 let registry = Arc::new(Registry::default());
19319 let forwarding = Arc::new(ForwardingTable::default());
19320 let supervisor = SupervisorHandle::new();
19321 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19322 let handler =
19323 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19324 .with_supervisor(supervisor);
19325 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
19326 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
19327 register_capability_manifest(
19328 &handler,
19329 &target_ctx,
19330 &mut target_rx,
19331 capability_manifest("target", &["credentials-provider/v1"], &[]),
19332 1,
19333 )
19334 .await;
19335 register_capability_manifest(
19336 &handler,
19337 &opener_ctx,
19338 &mut opener_rx,
19339 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19340 2,
19341 )
19342 .await;
19343
19344 let (_client_rx, bind) = open_route_for_capability_test(
19345 &handler,
19346 &target_ctx,
19347 &mut target_rx,
19348 742,
19349 3,
19350 "target",
19351 None,
19352 )
19353 .await;
19354 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
19355 panic!("direct scope-honesty route must bind");
19356 };
19357 assert_eq!(principal, Some(Principal::Direct));
19358 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19359 }
19360
19361 #[tokio::test]
19364 async fn must_never_reach_self_route_is_capability_forbidden() {
19365 let registry = Arc::new(Registry::default());
19366 let forwarding = Arc::new(ForwardingTable::default());
19367 let supervisor = SupervisorHandle::new();
19368 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
19369 let handler =
19370 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19371 .with_supervisor(supervisor);
19372 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
19373 register_capability_manifest(
19374 &handler,
19375 &self_ctx,
19376 &mut self_rx,
19377 capability_manifest(
19378 "self-provider",
19379 &["credentials-provider/v1"],
19380 &["credentials-provider/v1"],
19381 ),
19382 1,
19383 )
19384 .await;
19385
19386 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
19387 let replies = handler
19388 .handle_control_frame(
19389 &client_ctx,
19390 route_open_frame_with_admission_facts(
19391 2,
19392 "self-provider",
19393 unique_project_root("admission-facts"),
19394 Some(ConsumerIdentity {
19395 module_id: "self-provider".to_string(),
19396 launch_nonce: "self-nonce".to_string(),
19397 }),
19398 None,
19399 ),
19400 )
19401 .await
19402 .expect("self-route refusal returns a typed frame");
19403 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19404 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19405 assert!(
19406 self_rx.try_recv().is_err(),
19407 "self denial must not relay route.bind"
19408 );
19409 }
19410
19411 #[test]
19412 fn unsupported_channel_zero_frame_returns_error() {
19413 let handler = ControlHandler::default();
19414 let request = Frame::build(
19415 FrameType::Request,
19416 control_flags(),
19417 0,
19418 0,
19419 21,
19420 b"opaque".to_vec(),
19421 )
19422 .unwrap();
19423
19424 let response = handler
19425 .handle_control(ConnectionId::new(1), request)
19426 .unwrap();
19427
19428 assert_eq!(response[0].header.ty, FrameType::Error);
19429 assert_eq!(
19430 parse_error(&response[0])["code"],
19431 "unsupported_control_frame"
19432 );
19433 }
19434
19435 mod swap {
19440 use super::*;
19441
19442 const INCUMBENT: ConnectionId = ConnectionId::new(30);
19443 const CANDIDATE: ConnectionId = ConnectionId::new(40);
19444
19445 struct Swap {
19446 registry: Arc<Registry>,
19447 forwarding: Arc<ForwardingTable>,
19448 handler: ControlHandler,
19449 incumbent_ctx: RouteCtx,
19450 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19451 candidate_ctx: RouteCtx,
19452 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19453 }
19454
19455 async fn swap_with_incumbent() -> Swap {
19456 let registry = Arc::new(Registry::default());
19457 let forwarding = Arc::new(ForwardingTable::default());
19458 let handler =
19459 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
19460 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
19461 hello_via_sink(
19462 &handler,
19463 &incumbent_ctx,
19464 &mut incumbent_rx,
19465 hello_frame("aft", PROTOCOL_VERSION, 7),
19466 )
19467 .await;
19468 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
19469 Swap {
19470 registry,
19471 forwarding,
19472 handler,
19473 incumbent_ctx,
19474 incumbent_rx,
19475 candidate_ctx,
19476 candidate_rx,
19477 }
19478 }
19479
19480 fn register_candidate(swap: &Swap, ready: Option<bool>) {
19481 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
19482 candidate_manifest.ready = ready;
19483 let registration = swap
19484 .registry
19485 .register_candidate_with_control_ops(
19486 candidate_manifest,
19487 PROTOCOL_VERSION,
19488 CANDIDATE,
19489 module_baseline_control_ops(),
19490 )
19491 .unwrap();
19492 swap.forwarding
19493 .register_candidate_module_connection(
19494 CANDIDATE,
19495 "aft".to_string(),
19496 PROTOCOL_VERSION,
19497 manifest_concurrency(®istration.manifest),
19498 swap.candidate_ctx.egress.clone(),
19499 )
19500 .unwrap();
19501 }
19502
19503 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
19504 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
19505 swap.registry.promote_candidate("aft").unwrap().unwrap();
19506 cutover.incumbent.unwrap()
19507 }
19508
19509 #[tokio::test]
19518 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
19519 let mut swap = swap_with_incumbent().await;
19520 let hold = Arc::new(std::sync::Barrier::new(2));
19521 let handler = swap
19522 .handler
19523 .clone()
19524 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
19525 handler
19526 .supervisor
19527 .open_swap("aft", "candidate-nonce".to_string());
19528 let hello = {
19529 let handler = handler.clone();
19530 let candidate_ctx = swap.candidate_ctx.clone();
19531 std::thread::spawn(move || {
19532 handler
19533 .handle_hello(
19534 candidate_ctx.connection_id,
19535 Some(candidate_ctx.egress.clone()),
19536 hello_frame_with_nonce(
19537 "aft",
19538 PROTOCOL_VERSION,
19539 8,
19540 Some("candidate-nonce"),
19541 ),
19542 )
19543 .unwrap()
19544 })
19545 };
19546
19547 hold.wait();
19548 assert!(swap
19551 .registry
19552 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
19553 .unwrap()
19554 .is_some());
19555 assert!(swap
19556 .registry
19557 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19558 .unwrap()
19559 .is_none());
19560 assert!(swap
19561 .forwarding
19562 .module_endpoint_for_connection(CANDIDATE)
19563 .unwrap()
19564 .is_none());
19565
19566 hold.wait();
19567 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
19568 assert_eq!(
19569 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19570 FrameType::HelloAck
19571 );
19572 let published = swap
19573 .registry
19574 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19575 .unwrap()
19576 .expect("candidate is published once its HELLO completes");
19577 assert_eq!(published.connection_id, CANDIDATE);
19578 assert!(swap
19579 .forwarding
19580 .module_endpoint_for_connection(CANDIDATE)
19581 .unwrap()
19582 .is_some());
19583 }
19584
19585 #[tokio::test]
19586 async fn candidate_lookup_after_hello_ack_waits_for_publication() {
19587 let mut swap = swap_with_incumbent().await;
19588 let hold = Arc::new(std::sync::Barrier::new(2));
19589 swap.handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
19590 swap.handler
19591 .supervisor
19592 .open_swap("aft", "candidate-nonce".to_string());
19593 let hello = {
19594 let handler = swap.handler.clone();
19595 let candidate_ctx = swap.candidate_ctx.clone();
19596 std::thread::spawn(move || {
19597 handler
19598 .handle_hello(
19599 CANDIDATE,
19600 Some(candidate_ctx.egress),
19601 hello_frame_with_nonce(
19602 "aft",
19603 PROTOCOL_VERSION,
19604 8,
19605 Some("candidate-nonce"),
19606 ),
19607 )
19608 .unwrap()
19609 })
19610 };
19611 hold.wait();
19612 assert_eq!(
19613 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19614 FrameType::HelloAck
19615 );
19616 let reader_registry = Arc::clone(&swap.registry);
19617 let published = lookup_during_hello_publication(&swap.registry, &hold, move || {
19618 reader_registry
19619 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19620 .unwrap()
19621 });
19622 assert!(hello.join().unwrap().is_empty());
19623 assert_eq!(
19624 published
19625 .expect("ACK makes candidate publication visible")
19626 .connection_id,
19627 CANDIDATE
19628 );
19629 assert_eq!(
19630 swap.registry.generation().unwrap(),
19631 1,
19632 "candidate is not a second catalog entry"
19633 );
19634 }
19635
19636 #[tokio::test]
19637 async fn audit4_ungated_removed_candidate_publication_rolls_back_admission() {
19638 let mut swap = swap_with_incumbent().await;
19639 let hold = Arc::new(std::sync::Barrier::new(2));
19640 let handler = swap
19641 .handler
19642 .clone()
19643 .with_hello_registry_to_forwarding_hold(hold.clone());
19644 handler
19645 .supervisor
19646 .open_swap("aft", "candidate-nonce".into());
19647 let hello = {
19648 let handler = handler.clone();
19649 let ctx = swap.candidate_ctx.clone();
19650 std::thread::spawn(move || {
19651 handler
19652 .handle_hello(
19653 CANDIDATE,
19654 Some(ctx.egress),
19655 hello_frame_with_nonce(
19656 "aft",
19657 PROTOCOL_VERSION,
19658 8,
19659 Some("candidate-nonce"),
19660 ),
19661 )
19662 .unwrap()
19663 })
19664 };
19665 hold.wait();
19666 assert_eq!(
19667 swap.registry
19668 .deregister_connection(CANDIDATE)
19669 .unwrap()
19670 .len(),
19671 1
19672 );
19673 hold.wait();
19674 assert!(hello.join().unwrap().is_empty());
19675 assert_eq!(
19676 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19677 FrameType::HelloAck
19678 );
19679 assert!(swap.registry.get_candidate("aft").unwrap().is_none());
19680 assert_eq!(
19681 handler
19682 .supervisor
19683 .swap_hello_admission("aft", Some("candidate-nonce")),
19684 SwapHelloAdmission::Candidate,
19685 "failed publication must roll back candidate_admitted for an ungated nonce swap"
19686 );
19687 handler.supervisor.close_swap("aft");
19688 }
19689
19690 fn keyed_total(counters: &Value, key: &str) -> u64 {
19691 counters[key]
19692 .as_object()
19693 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
19694 .unwrap_or(0)
19695 }
19696
19697 #[tokio::test]
19698 async fn replacement_logs_old_end_and_new_admission_once() {
19699 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19700 let mut swap = swap_with_incumbent().await;
19701 swap.handler
19702 .supervisor
19703 .open_swap("aft", "candidate-nonce".to_string());
19704 hello_via_sink(
19705 &swap.handler,
19706 &swap.candidate_ctx,
19707 &mut swap.candidate_rx,
19708 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
19709 )
19710 .await;
19711 cutover(&swap);
19712 swap.handler.cleanup_connection(INCUMBENT).unwrap();
19713
19714 let captured = crate::router::test_log::captured_logs(&logs);
19715 println!("captured replacement registry logs:\n{captured}");
19716 let new_admissions: Vec<_> = captured
19717 .lines()
19718 .filter(|line| {
19719 line.contains("connection_id=40")
19720 && line.contains("swap candidate registered; not routable until cutover")
19721 })
19722 .collect();
19723 assert_eq!(
19724 new_admissions.len(),
19725 1,
19726 "unexpected admission logs: {captured}"
19727 );
19728 assert!(new_admissions[0].contains("module_id=aft"));
19729 assert!(new_admissions[0].contains("connection_id=40"));
19730 assert!(
19731 new_admissions[0].contains("swap candidate registered; not routable until cutover")
19732 );
19733 assert!(new_admissions[0].contains("ready=true"));
19734
19735 let old_admissions: Vec<_> = captured
19736 .lines()
19737 .filter(|line| {
19738 line.contains("module registered module_id=aft ")
19739 && line.contains("connection_id=30")
19740 })
19741 .collect();
19742 assert_eq!(
19743 old_admissions.len(),
19744 1,
19745 "unexpected incumbent admission: {captured}"
19746 );
19747
19748 let old_ends: Vec<_> = captured
19749 .lines()
19750 .filter(|line| {
19751 line.contains("connection_id=30") && line.contains("module registration ended")
19752 })
19753 .collect();
19754 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
19755 assert!(old_ends[0].contains("module_id=aft"));
19756 assert!(old_ends[0].contains("connection_id=30"));
19757 assert!(old_ends[0].contains("reason=replaced"));
19758 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
19759 assert_eq!(
19760 captured
19761 .lines()
19762 .filter(|line| line.contains("module registration promoted"))
19763 .count(),
19764 1,
19765 "unexpected promotion logs: {captured}"
19766 );
19767 }
19768
19769 #[tokio::test]
19775 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
19776 let mut swap = swap_with_incumbent().await;
19777 let handler = swap.handler.clone();
19778
19779 let cotenant = ConnectionId::new(31);
19781 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
19782 let (cotenant_task, cotenant_bind) = relay_route_open(
19783 &handler,
19784 cotenant,
19785 &cotenant_ctx.egress,
19786 &mut swap.incumbent_rx,
19787 100,
19788 "aft",
19789 "swap-cotenant",
19790 )
19791 .await;
19792 handler
19793 .handle_control_frame(
19794 &swap.incumbent_ctx,
19795 route_bind_ack(cotenant_bind.header.corr),
19796 )
19797 .await
19798 .unwrap();
19799 assert!(cotenant_task.await.unwrap().is_empty());
19800 let (cotenant_channel, cotenant_epoch) =
19801 published_route(&cotenant_rx.recv().await.unwrap());
19802
19803 let caller = ConnectionId::new(32);
19805 let (caller_ctx, mut caller_rx) = route_ctx(caller);
19806 let (caller_task, caller_bind) = relay_route_open(
19807 &handler,
19808 caller,
19809 &caller_ctx.egress,
19810 &mut swap.incumbent_rx,
19811 101,
19812 "aft",
19813 "swap-caller",
19814 )
19815 .await;
19816 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
19817
19818 register_candidate(&swap, None);
19819 cutover(&swap);
19820
19821 let ack = handler
19823 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
19824 .await;
19825 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
19826 if module_loop_error.is_some() {
19827 handler.cleanup_connection(INCUMBENT).unwrap();
19830 }
19831
19832 assert!(
19834 cotenant_rx.try_recv().is_err(),
19835 "the co-tenant route on the incumbent was torn down by one late ack: \
19836 {module_loop_error:?}"
19837 );
19838 assert!(matches!(
19839 swap.forwarding
19840 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
19841 .unwrap(),
19842 DataRoute::Client(DataRouteState::Bound(_))
19843 ));
19844 assert_eq!(module_loop_error, None);
19845 assert!(swap
19846 .registry
19847 .get_module_by_connection(INCUMBENT)
19848 .unwrap()
19849 .is_some());
19850
19851 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
19853 .await
19854 .expect("the incumbent is told to drop the abandoned binding")
19855 .unwrap()
19856 .frame;
19857 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19858 assert_eq!(goodbye.header.channel, abandoned_channel);
19859 assert_eq!(goodbye.header.epoch, abandoned_epoch);
19860 assert!(swap.incumbent_rx.try_recv().is_err());
19861
19862 let response = caller_task.await.unwrap();
19864 assert_eq!(response.len(), 1);
19865 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
19866 assert!(caller_rx.try_recv().is_err());
19867
19868 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
19872 let counters = handler.counters().snapshot();
19873 assert_eq!(
19874 keyed_total(&counters, "route_open_accepted_by_principal"),
19875 1
19876 );
19877 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
19878 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
19879 }
19880
19881 #[tokio::test]
19885 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
19886 let mut swap = swap_with_incumbent().await;
19887 register_candidate(&swap, None);
19888 let incumbent = cutover(&swap);
19889 swap.forwarding
19890 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
19891 .unwrap()
19892 .expect("the incumbent is still registered");
19893
19894 let client = ConnectionId::new(33);
19895 let (client_ctx, mut client_rx) = route_ctx(client);
19896 let route_handler = swap.handler.clone();
19897 let open_ctx = RouteCtx {
19898 connection_id: client,
19899 egress: client_ctx.egress.clone(),
19900 };
19901 let mut route_task = tokio::spawn(async move {
19902 route_handler
19903 .handle_control_frame(
19904 &open_ctx,
19905 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
19906 )
19907 .await
19908 .unwrap()
19909 });
19910 let bind = tokio::select! {
19911 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
19912 response = &mut route_task => {
19913 let response = response.unwrap();
19914 panic!(
19915 "post-cutover route.open was refused instead of relayed to the candidate: {}",
19916 parse_error(&response[0])["code"]
19917 );
19918 }
19919 };
19920 swap.handler
19921 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
19922 .await
19923 .unwrap();
19924 assert!(route_task.await.unwrap().is_empty());
19925 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
19926 match swap
19927 .forwarding
19928 .lookup_data_route(client, channel, epoch)
19929 .unwrap()
19930 {
19931 DataRoute::Client(DataRouteState::Bound(route)) => {
19932 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
19933 }
19934 other => panic!("expected a bound route on the candidate, got {other:?}"),
19935 }
19936 assert!(swap.incumbent_rx.try_recv().is_err());
19937 }
19938
19939 #[tokio::test]
19944 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
19945 let swap = swap_with_incumbent().await;
19946 register_candidate(&swap, Some(false));
19947 let update = Frame::build(
19948 FrameType::Request,
19949 control_flags(),
19950 0,
19951 0,
19952 55,
19953 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
19954 provides: manifest("aft", PROTOCOL_VERSION).provides,
19955 capabilities: None,
19956 ready: Some(true),
19957 })
19958 .unwrap(),
19959 )
19960 .unwrap();
19961
19962 let replies = swap
19963 .handler
19964 .handle_control_frame(&swap.candidate_ctx, update)
19965 .await
19966 .unwrap();
19967
19968 assert_eq!(replies.len(), 1);
19969 assert_eq!(
19970 replies[0].header.ty,
19971 FrameType::Response,
19972 "candidate catalog.update was refused: {:?}",
19973 serde_json::from_slice::<Value>(&replies[0].body).ok()
19974 );
19975 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
19976 assert_eq!(
19977 swap.registry
19978 .get_module("aft")
19979 .unwrap()
19980 .unwrap()
19981 .connection_id,
19982 INCUMBENT
19983 );
19984 }
19985 }
19986
19987 mod swap_admission {
19991 use super::*;
19992
19993 const INCUMBENT_NONCE: &str = "incumbent-nonce";
19994 const CANDIDATE_NONCE: &str = "candidate-nonce";
19995
19996 fn handler_with_incumbent(
19997 module_id: &str,
19998 reserved: bool,
19999 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
20000 let registry = Arc::new(Registry::default());
20001 let supervisor = SupervisorHandle::new();
20002 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
20003 if reserved {
20004 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
20005 }
20006 let handler =
20007 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
20008 let incumbent = handler
20009 .handle_control(
20010 ConnectionId::new(1),
20011 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
20012 )
20013 .unwrap();
20014 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
20015 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
20016 (registry, supervisor, handler)
20017 }
20018
20019 #[test]
20025 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
20026 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
20027
20028 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
20029 let replies = handler
20030 .handle_control(
20031 ConnectionId::new(connection),
20032 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
20033 )
20034 .unwrap();
20035 assert_eq!(replies[0].header.ty, FrameType::Error);
20036 assert_eq!(
20037 parse_error(&replies[0])["code"],
20038 "swap_token_invalid",
20039 "nonce {nonce:?}"
20040 );
20041 }
20042 assert!(registry.get_candidate("aft").unwrap().is_none());
20043 assert_eq!(
20044 registry.get_module("aft").unwrap().unwrap().connection_id,
20045 ConnectionId::new(1)
20046 );
20047
20048 let admitted = handler
20051 .handle_control(
20052 ConnectionId::new(4),
20053 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
20054 )
20055 .unwrap();
20056 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
20057 assert_eq!(
20058 registry
20059 .get_candidate("aft")
20060 .unwrap()
20061 .unwrap()
20062 .connection_id,
20063 ConnectionId::new(4)
20064 );
20065 assert_eq!(
20066 registry.get_module("aft").unwrap().unwrap().connection_id,
20067 ConnectionId::new(1),
20068 "the candidate must not take the active slot"
20069 );
20070 let replayed = handler
20071 .handle_control(
20072 ConnectionId::new(5),
20073 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
20074 )
20075 .unwrap();
20076 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
20077
20078 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
20082 let squatter = handler
20083 .handle_control(
20084 ConnectionId::new(6),
20085 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
20086 )
20087 .unwrap();
20088 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
20089 assert!(
20090 registry.get_module("aft").unwrap().is_none(),
20091 "a squatter took the active slot of an id being swapped"
20092 );
20093 }
20094
20095 #[test]
20100 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
20101 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
20102
20103 let replies = handler
20104 .handle_control(
20105 ConnectionId::new(2),
20106 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20107 )
20108 .unwrap();
20109
20110 assert_eq!(
20111 replies[0].header.ty,
20112 FrameType::HelloAck,
20113 "reserved candidate refused: {:?}",
20114 serde_json::from_slice::<Value>(&replies[0].body).ok()
20115 );
20116 assert_eq!(
20117 registry
20118 .get_candidate("vault")
20119 .unwrap()
20120 .unwrap()
20121 .connection_id,
20122 ConnectionId::new(2)
20123 );
20124 }
20125
20126 #[test]
20129 fn without_an_open_swap_the_ordinary_gates_decide() {
20130 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
20131 supervisor.close_swap("vault");
20132
20133 let candidate = handler
20134 .handle_control(
20135 ConnectionId::new(2),
20136 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20137 )
20138 .unwrap();
20139 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
20140 let duplicate = handler
20141 .handle_control(
20142 ConnectionId::new(3),
20143 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
20144 )
20145 .unwrap();
20146 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
20147 assert!(registry.get_candidate("vault").unwrap().is_none());
20148 }
20149 }
20150
20151 mod scopes {
20155 mod close_reasons {
20156 include!("route_close_reason_tests.rs");
20157 }
20158 mod idle_tests {
20159 use super::*;
20160 include!("scope_idle_tests.rs");
20161 }
20162 mod route_open_timings {
20163 use super::*;
20164 include!("route_open_timing_tests.rs");
20165 }
20166 mod route_open_refusal_log {
20167 use super::*;
20168 include!("route_open_refusal_log_tests.rs");
20169 }
20170 mod activity {
20171 include!("activity_scope_tests.rs");
20172 }
20173 mod delta {
20174 include!("scope_handler_delta_tests.rs");
20175 }
20176 mod call_modes {
20177 include!("scope_call_mode_tests.rs");
20178 }
20179 mod federated {
20180 include!("scope_federated_tests.rs");
20181 }
20182 mod authorize {
20183 include!("route_authorize_tests.rs");
20184 }
20185 #[tokio::test]
20186 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
20187 let rig = rig().await;
20188 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
20189 "s".into(),
20190 1,
20191 "tool".into(),
20192 ))
20193 .unwrap();
20194 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
20195 let (direct, _rx) = wide_ctx(999);
20196 for (ctx, expected) in [
20197 (&rig.owner, FrameType::Response),
20198 (&direct, FrameType::Error),
20199 ] {
20200 let frame =
20201 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
20202 .unwrap();
20203 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
20204 assert_eq!(replies[0].header.ty, expected);
20205 if expected == FrameType::Response {
20206 let reply: subc_control::ScopeServedByReply =
20207 serde_json::from_slice(&replies[0].body).unwrap();
20208 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
20209 } else {
20210 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
20211 }
20212 }
20213 let reply = call(
20214 &rig.handler,
20215 &rig.owner,
20216 &ModuleControlRequestFromModule::ScopeDescribe {
20217 owner: Principal::Reserved {
20218 module_id: OWNER.into(),
20219 },
20220 scope_ref: "s".into(),
20221 },
20222 )
20223 .await;
20224 assert!(matches!(
20225 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
20226 ModuleControlResponseToModule::ScopeDescribe { .. }
20227 ));
20228 }
20229
20230 use subc_protocol::scope::{
20231 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
20232 ScopeStatus,
20233 };
20234
20235 use super::*;
20236
20237 const OWNER: &str = "prefrontal-core";
20238
20239 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
20240 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
20241 }
20242
20243 async fn call(
20244 handler: &ControlHandler,
20245 ctx: &RouteCtx,
20246 request: &ModuleControlRequestFromModule,
20247 ) -> Frame {
20248 let body = serde_json::to_vec(request).unwrap();
20249 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
20250 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
20251 assert_eq!(replies.len(), 1, "{replies:?}");
20252 replies.pop().unwrap()
20253 }
20254
20255 async fn sync(
20256 handler: &ControlHandler,
20257 ctx: &RouteCtx,
20258 generation: u64,
20259 scopes: Vec<ScopeRecord>,
20260 ) -> Result<ModuleControlResponseToModule, String> {
20261 let reply = call(
20262 handler,
20263 ctx,
20264 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
20265 )
20266 .await;
20267 match reply.header.ty {
20268 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
20269 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
20270 }
20271 }
20272
20273 async fn describe(
20274 handler: &ControlHandler,
20275 ctx: &RouteCtx,
20276 owner: &str,
20277 scope_ref: &str,
20278 ) -> ModuleControlResponseToModule {
20279 let reply = call(
20280 handler,
20281 ctx,
20282 &ModuleControlRequestFromModule::ScopeDescribe {
20283 owner: Principal::Reserved {
20284 module_id: owner.to_string(),
20285 },
20286 scope_ref: scope_ref.to_string(),
20287 },
20288 )
20289 .await;
20290 assert_eq!(
20291 reply.header.ty,
20292 FrameType::Response,
20293 "{:?}",
20294 parse_error(&reply)
20295 );
20296 serde_json::from_slice(&reply.body).unwrap()
20297 }
20298
20299 #[tokio::test]
20300 async fn scope_describe_never_echoes_bind_proof_or_hash() {
20301 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
20302 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
20303 let mut rig = rig().await;
20304 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
20305 .await;
20306 let _bound = rig
20307 .bound(
20308 Some(AFT),
20309 PLEXUS,
20310 Some(
20311 rig.selector("s", Some(1))
20312 .with_bind_proof(Some(PROOF.into())),
20313 ),
20314 )
20315 .await;
20316 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
20317 let output = serde_json::to_string(&reply).unwrap();
20318 assert!(output.contains("\"status\":\"live\""));
20319 assert!(!output.contains(PROOF));
20320 assert!(!output.contains(HASH));
20321 }
20322
20323 async fn module(
20325 handler: &ControlHandler,
20326 connection: u64,
20327 module_id: &str,
20328 nonce: Option<&str>,
20329 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20330 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
20331 hello_via_sink(
20332 handler,
20333 &ctx,
20334 &mut rx,
20335 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
20336 )
20337 .await;
20338 (ctx, rx)
20339 }
20340
20341 #[tokio::test]
20344 async fn a_client_connection_cannot_sync_or_describe() {
20345 let handler = ControlHandler::new(Arc::new(Registry::default()));
20346 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
20347 for request in [
20348 ModuleControlRequestFromModule::ScopeSync {
20349 generation: 1,
20350 scopes: vec![head("s", 1)],
20351 },
20352 ModuleControlRequestFromModule::ScopeDescribe {
20353 owner: Principal::Direct,
20354 scope_ref: "s".to_string(),
20355 },
20356 ] {
20357 let reply = call(&handler, &ctx, &request).await;
20358 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
20359 }
20360 assert!(
20361 !handler
20362 .scopes
20363 .read()
20364 .unwrap()
20365 .describe(
20366 &Principal::Reserved {
20367 module_id: OWNER.to_string()
20368 },
20369 "s"
20370 )
20371 .owner_synced
20372 );
20373 }
20374
20375 #[tokio::test]
20378 async fn a_module_without_a_supervised_launch_cannot_sync() {
20379 let handler = ControlHandler::new(Arc::new(Registry::default()));
20380 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
20381 assert_eq!(
20382 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
20383 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20384 );
20385 }
20386
20387 #[tokio::test]
20388 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
20389 let supervisor = SupervisorHandle::new();
20390 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20391 let handler = ControlHandler::new(Arc::new(Registry::default()))
20392 .with_supervisor(supervisor.clone());
20393 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20394 sync(&handler, &incumbent, 1, vec![head("s", 1)])
20395 .await
20396 .expect("the current launch syncs");
20397
20398 supervisor.open_swap(OWNER, "n2".to_string());
20401 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
20402 assert_eq!(
20403 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20404 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20405 );
20406 sync(&handler, &incumbent, 2, vec![head("s", 1)])
20407 .await
20408 .expect("the serving owner syncs during the swap");
20409
20410 supervisor.close_swap(OWNER);
20413 assert_eq!(
20414 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20415 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20416 );
20417 sync(&handler, &incumbent, 3, vec![head("s", 1)])
20418 .await
20419 .expect("the serving owner syncs after the rollback");
20420 handler.cleanup_connection(candidate.connection_id).unwrap();
20421
20422 supervisor.open_swap(OWNER, "n3".to_string());
20427 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
20428 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
20429 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
20430 .await
20431 .expect("the promoted launch takes authority");
20432 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20433 panic!("unexpected reply {reply:?}");
20434 };
20435 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
20436 assert_eq!(
20437 sync(&handler, &incumbent, 4, Vec::new()).await,
20438 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20439 );
20440 }
20441
20442 #[tokio::test]
20445 async fn closing_the_authority_connection_frees_sync_authority() {
20446 let supervisor = SupervisorHandle::new();
20447 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20448 let handler = ControlHandler::new(Arc::new(Registry::default()))
20449 .with_supervisor(supervisor.clone());
20450 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20451 sync(&handler, &first, 10, vec![head("s", 1)])
20452 .await
20453 .unwrap();
20454 handler.cleanup_connection(first.connection_id).unwrap();
20455
20456 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
20457 sync(&handler, &second, 1, vec![head("s", 1)])
20458 .await
20459 .expect("the next connection takes the released authority");
20460 }
20461
20462 #[tokio::test]
20463 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
20464 let supervisor = SupervisorHandle::new();
20465 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20466 let handler =
20467 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
20468 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
20469 sync(&handler, &first, 10, vec![head("s", 1)])
20470 .await
20471 .unwrap();
20472 handler
20473 .handle_control_frame(
20474 &first,
20475 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
20476 )
20477 .await
20478 .unwrap();
20479 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
20480 sync(&handler, &second, 1, vec![head("s", 1)])
20481 .await
20482 .expect("GOODBYE releases authority even if the old socket remains open");
20483 }
20484
20485 #[tokio::test]
20486 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
20487 let registry = Arc::new(Registry::default());
20488 let supervisor_handle = SupervisorHandle::new();
20489 let supervisor =
20490 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20491 .with_handle(supervisor_handle.clone())
20492 .with_daemon_incarnation("incarnation-7".to_string());
20493 supervisor
20496 .supervise_configured(
20497 ModuleSpec {
20498 carry_containment: false,
20499 module_id: OWNER.to_string(),
20500 program: PathBuf::from("/nonexistent/prefrontal-core"),
20501 args: Vec::new(),
20502 env: Vec::new(),
20503 reserved: false,
20504 reserved_prefixes: Vec::new(),
20505 protocol: ModuleProtocol::Subc,
20506 overlap: Default::default(),
20507 },
20508 false,
20509 )
20510 .unwrap();
20511 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
20512 let handler =
20513 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
20514 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
20515
20516 let ModuleControlResponseToModule::ScopeDescribe {
20518 status,
20519 daemon_incarnation,
20520 owner_synced,
20521 owner_configured,
20522 scope,
20523 ..
20524 } = describe(&handler, &reader, OWNER, "s").await
20525 else {
20526 panic!("not a describe reply");
20527 };
20528 assert_eq!(status, ScopeStatus::NotLive);
20529 assert_eq!(daemon_incarnation, "incarnation-7");
20530 assert!(!owner_synced);
20531 assert!(owner_configured);
20532 assert!(scope.is_none());
20533
20534 let ModuleControlResponseToModule::ScopeDescribe {
20537 status,
20538 owner_configured,
20539 ..
20540 } = describe(&handler, &reader, "ghost", "s").await
20541 else {
20542 panic!("not a describe reply");
20543 };
20544 assert_eq!(status, ScopeStatus::NotLive);
20545 assert!(!owner_configured);
20546
20547 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
20549 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
20550 let ModuleControlResponseToModule::ScopeDescribe {
20551 status,
20552 scope_epoch,
20553 owner_synced,
20554 scope,
20555 ..
20556 } = describe(&handler, &reader, OWNER, "s").await
20557 else {
20558 panic!("not a describe reply");
20559 };
20560 assert_eq!(status, ScopeStatus::Live);
20561 assert_eq!(scope_epoch, Some(4));
20562 assert!(owner_synced);
20563 let stamp = scope.expect("a live scope carries its stamp");
20564 assert!(
20565 stamp.owner_authorized,
20566 "prefrontal-core is the default authority"
20567 );
20568 assert_eq!(stamp.kind, ScopeKind::Head);
20569 }
20570
20571 #[tokio::test]
20572 async fn scope_authority_owners_decides_owner_authorized() {
20573 let supervisor = SupervisorHandle::new();
20574 supervisor.set_spawn_nonce("broca", "b1".to_string());
20575 let handler = ControlHandler::new(Arc::new(Registry::default()))
20576 .with_supervisor(supervisor)
20577 .with_scope_authority_owners(vec!["broca".to_string()]);
20578 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
20579 let mut gated = head("s", 1);
20580 gated.attributes.agent_id = Some("agent".to_string());
20581 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
20582 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
20583 describe(&handler, &broca, "broca", "s").await
20584 else {
20585 panic!("not a describe reply");
20586 };
20587 assert!(scope.unwrap().owner_authorized);
20588 }
20589
20590 #[tokio::test]
20594 async fn scope_ops_and_the_scopes_capability_are_advertised() {
20595 let handler = ControlHandler::new(Arc::new(Registry::default()));
20596 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
20597 let ack = hello_via_sink(
20598 &handler,
20599 &ctx,
20600 &mut rx,
20601 hello_frame("m", PROTOCOL_VERSION, 1),
20602 )
20603 .await;
20604 let ack = parse_ack(&ack);
20605 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
20606 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
20607 }
20608 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
20609
20610 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
20611 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
20612 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
20613 let reply = handler
20614 .handle_control_frame(&client, frame)
20615 .await
20616 .unwrap()
20617 .pop()
20618 .unwrap();
20619 let ClientControlResponse::ServerDescribe { capabilities, .. } =
20620 serde_json::from_slice(&reply.body).unwrap()
20621 else {
20622 panic!("not a server.describe reply");
20623 };
20624 assert!(
20625 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
20626 "{capabilities:?}"
20627 );
20628 }
20629
20630 const PLEXUS: &str = "plexus";
20633 const OTHER: &str = "other";
20634 const AFT: &str = "aft";
20635 const BROCA: &str = "broca";
20636 const MAGIC: &str = "magic-context";
20637
20638 fn nonce(module_id: &str) -> String {
20639 format!("nonce-{module_id}")
20640 }
20641
20642 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20643 let (tx, rx) = mpsc::channel(64);
20644 (
20645 RouteCtx {
20646 connection_id: ConnectionId::new(connection),
20647 egress: FrameSink::new(tx),
20648 },
20649 rx,
20650 )
20651 }
20652
20653 struct Rig {
20657 handler: ControlHandler,
20658 forwarding: Arc<ForwardingTable>,
20659 owner: RouteCtx,
20660 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20661 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
20662 generation: u64,
20663 next_connection: u64,
20664 _supervisor: Supervisor,
20665 }
20666
20667 async fn rig() -> Rig {
20668 rig_with_flow_support(true).await
20669 }
20670
20671 async fn rig_with_flow_support(flow_support: bool) -> Rig {
20672 rig_with_scope_capabilities(flow_support, false).await
20673 }
20674
20675 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
20676 let mut provides = Vec::new();
20677 if flow_support {
20678 provides.push("flow-scopes/v1");
20679 }
20680 if run_support {
20681 provides.push("agent-run-scopes/v1");
20682 }
20683 rig_with_target_capabilities(&provides).await
20684 }
20685
20686 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
20687 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
20688 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20689 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20692 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
20693 Frame::build(
20694 FrameType::Hello,
20695 control_flags(),
20696 0,
20697 0,
20698 connection,
20699 serde_json::to_vec(&body).unwrap(),
20700 )
20701 .unwrap()
20702 }
20703
20704 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
20705 let registry = Arc::new(Registry::default());
20706 let forwarding = Arc::new(ForwardingTable::default());
20707 let supervisor_handle = SupervisorHandle::new();
20708 let supervisor =
20709 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20710 .with_handle(supervisor_handle.clone());
20711 supervisor
20712 .supervise_configured(
20713 ModuleSpec {
20714 carry_containment: false,
20715 module_id: OWNER.to_string(),
20716 program: PathBuf::from("/nonexistent/prefrontal-core"),
20717 args: Vec::new(),
20718 env: Vec::new(),
20719 reserved: false,
20720 reserved_prefixes: Vec::new(),
20721 protocol: ModuleProtocol::Subc,
20722 overlap: Default::default(),
20723 },
20724 false,
20725 )
20726 .unwrap();
20727 for module_id in [OWNER, AFT, BROCA, MAGIC] {
20728 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
20729 }
20730 let handler =
20731 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
20732 .with_supervisor(supervisor_handle);
20733 let (owner, mut owner_rx) = wide_ctx(1);
20734 hello_via_sink(
20735 &handler,
20736 &owner,
20737 &mut owner_rx,
20738 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
20739 )
20740 .await;
20741 let mut modules = BTreeMap::new();
20742 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
20743 let (ctx, mut rx) = wide_ctx(connection);
20744 let hello = hello_with_capabilities(module_id, connection, provides);
20745 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
20746 modules.insert(module_id.to_string(), (ctx, rx));
20747 }
20748 Rig {
20749 handler,
20750 forwarding,
20751 owner,
20752 _owner_rx: owner_rx,
20753 modules,
20754 generation: 0,
20755 next_connection: 100,
20756 _supervisor: supervisor,
20757 }
20758 }
20759
20760 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
20761 ScopeCarrier::new(Principal::Reserved {
20762 module_id: module_id.to_string(),
20763 })
20764 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
20765 }
20766
20767 fn session(scope_epoch: u64) -> ScopeRecord {
20770 let mut record = head("s", scope_epoch);
20771 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
20772 record.attributes.agent_id = Some("agent-1".to_string());
20773 record.attributes.delegates = true;
20774 record
20775 }
20776
20777 impl Rig {
20778 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
20779 self.generation += 1;
20780 sync(&self.handler, &self.owner, self.generation, scopes)
20781 .await
20782 .expect("the owner's sync is accepted");
20783 }
20784
20785 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
20786 ScopeSelector::new(
20787 Principal::Reserved {
20788 module_id: OWNER.to_string(),
20789 },
20790 scope_ref.to_string(),
20791 scope_epoch,
20792 )
20793 }
20794
20795 fn open_frame(
20796 &mut self,
20797 opener: Option<&str>,
20798 target: &str,
20799 scope: Option<ScopeSelector>,
20800 ) -> (
20801 RouteCtx,
20802 mpsc::Receiver<crate::router::OutboundFrame>,
20803 Frame,
20804 ) {
20805 self.next_connection += 1;
20806 let (ctx, rx) = wide_ctx(self.next_connection);
20807 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
20808 busy_refusals: None,
20809 target: RouteTarget::ToolProvider {
20810 module_id: target.to_string(),
20811 },
20812 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
20813 consumer_identity: opener.map(|module_id| ConsumerIdentity {
20814 module_id: module_id.to_string(),
20815 launch_nonce: nonce(module_id),
20816 }),
20817 consumer_capabilities: None,
20818 role_versions: None,
20819 admission_facts: None,
20820 scope,
20821 })
20822 .unwrap();
20823 let frame = Frame::build(
20824 FrameType::Request,
20825 control_flags(),
20826 0,
20827 0,
20828 self.next_connection,
20829 body,
20830 )
20831 .unwrap();
20832 (ctx, rx, frame)
20833 }
20834
20835 async fn refused(
20837 &mut self,
20838 opener: Option<&str>,
20839 target: &str,
20840 scope: Option<ScopeSelector>,
20841 ) -> String {
20842 self.refusal_body(opener, target, scope).await["code"]
20843 .as_str()
20844 .unwrap()
20845 .to_string()
20846 }
20847
20848 async fn refusal_body(
20849 &mut self,
20850 opener: Option<&str>,
20851 target: &str,
20852 scope: Option<ScopeSelector>,
20853 ) -> Value {
20854 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
20855 let replies = tokio::time::timeout(
20856 Duration::from_secs(2),
20857 self.handler.handle_control_frame(&ctx, frame),
20858 )
20859 .await
20860 .expect("the open must be refused before waiting for a bind ack")
20861 .unwrap();
20862 assert_eq!(replies.len(), 1, "{replies:?}");
20863 assert_eq!(replies[0].header.ty, FrameType::Error);
20864 let (_, module_rx) = self.modules.get_mut(target).unwrap();
20865 assert!(
20866 module_rx.try_recv().is_err(),
20867 "a refused open relays nothing"
20868 );
20869 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
20870 parse_error(&replies[0])
20871 }
20872
20873 async fn relayed(
20875 &mut self,
20876 opener: Option<&str>,
20877 target: &str,
20878 scope: Option<ScopeSelector>,
20879 ) -> Relayed {
20880 self.relayed_with_legacy_root(opener, target, scope, None)
20881 .await
20882 }
20883
20884 async fn relayed_with_legacy_root(
20885 &mut self,
20886 opener: Option<&str>,
20887 target: &str,
20888 scope: Option<ScopeSelector>,
20889 legacy_root: Option<&std::path::Path>,
20890 ) -> Relayed {
20891 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
20892 if let Some(root) = legacy_root {
20893 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
20894 body["identity"]["project_root"] = serde_json::json!(root);
20895 frame.body = serde_json::to_vec(&body).unwrap();
20896 }
20897 let handler = self.handler.clone();
20898 let task_ctx = ctx.clone();
20899 let task = tokio::spawn(async move {
20900 handler
20901 .handle_control_frame(&task_ctx, frame)
20902 .await
20903 .unwrap()
20904 });
20905 let (_, module_rx) = self.modules.get_mut(target).unwrap();
20906 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
20907 .await
20908 .expect("the target receives the relayed route.bind")
20909 .unwrap()
20910 .frame;
20911 Relayed {
20912 target: target.to_string(),
20913 client: ctx,
20914 client_rx: rx,
20915 task,
20916 bind,
20917 }
20918 }
20919
20920 async fn ack(&self, relayed: &Relayed) {
20921 let (module, _) = &self.modules[&relayed.target];
20922 self.handler
20923 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
20924 .await
20925 .unwrap();
20926 }
20927
20928 async fn bound(
20930 &mut self,
20931 opener: Option<&str>,
20932 target: &str,
20933 scope: Option<ScopeSelector>,
20934 ) -> Bound {
20935 let relayed = self.relayed(opener, target, scope).await;
20936 self.commit_relayed(relayed).await
20937 }
20938
20939 async fn bound_with_legacy_root(
20940 &mut self,
20941 opener: Option<&str>,
20942 target: &str,
20943 scope: Option<ScopeSelector>,
20944 root: &std::path::Path,
20945 ) -> Bound {
20946 let relayed = self
20947 .relayed_with_legacy_root(opener, target, scope, Some(root))
20948 .await;
20949 self.commit_relayed(relayed).await
20950 }
20951
20952 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
20953 self.ack(&relayed).await;
20954 let Relayed {
20955 target,
20956 client,
20957 mut client_rx,
20958 task,
20959 bind,
20960 } = relayed;
20961 assert!(
20962 task.await.unwrap().is_empty(),
20963 "the open is answered by commit"
20964 );
20965 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
20966 Bound {
20967 target,
20968 client,
20969 client_rx,
20970 channel,
20971 epoch,
20972 bind,
20973 }
20974 }
20975
20976 fn live(&self, route: &Bound) -> bool {
20977 matches!(
20978 self.forwarding
20979 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
20980 .unwrap(),
20981 DataRoute::Client(DataRouteState::Bound(_))
20982 )
20983 }
20984 }
20985
20986 struct Relayed {
20987 target: String,
20988 client: RouteCtx,
20989 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20990 task: tokio::task::JoinHandle<Vec<Frame>>,
20991 bind: Frame,
20992 }
20993
20994 struct Bound {
20995 target: String,
20996 client: RouteCtx,
20997 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20998 channel: u16,
20999 epoch: u32,
21000 bind: Frame,
21001 }
21002
21003 impl Bound {
21004 fn closed_reason(&mut self) -> RouteCloseReason {
21007 let mut reason = None;
21008 let mut goodbye = false;
21009 while let Ok(outbound) = self.client_rx.try_recv() {
21010 let frame = outbound.frame;
21011 match frame.header.ty {
21012 FrameType::Goodbye => {
21013 assert_eq!(
21014 (frame.header.channel, frame.header.epoch),
21015 (self.channel, self.epoch)
21016 );
21017 goodbye = true;
21018 }
21019 FrameType::Push => {
21020 let ClientControlPush::RouteClosed {
21021 reason: r,
21022 module_id,
21023 ..
21024 } = serde_json::from_slice(&frame.body).unwrap()
21025 else {
21026 panic!("unexpected push");
21027 };
21028 assert_eq!(module_id, self.target);
21029 reason = Some(r);
21030 }
21031 other => panic!("unexpected frame {other:?}"),
21032 }
21033 }
21034 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
21035 reason.expect("the client is told why the route closed")
21036 }
21037
21038 fn untouched(&mut self) -> bool {
21039 self.client_rx.try_recv().is_err()
21040 }
21041
21042 fn stamp(&self) -> Option<ScopeStamp> {
21043 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
21044 ModuleControlRequest::RouteBind { scope, .. } => scope,
21045 other => panic!("expected a route.bind, got {other:?}"),
21046 }
21047 }
21048 }
21049
21050 #[tokio::test]
21051 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
21052 ) {
21053 let mut rig = rig().await;
21054 let mut record = session(1);
21055 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
21056 record.child_owners = vec![Principal::Reserved {
21057 module_id: MAGIC.to_string(),
21058 }];
21059 rig.sync(vec![record]).await;
21060 let scope = || Some(rig_selector("s", Some(1)));
21061
21062 rig.bound(Some(OWNER), PLEXUS, scope()).await;
21065 rig.bound(Some(AFT), OTHER, scope()).await;
21066 rig.bound(Some(BROCA), PLEXUS, scope()).await;
21067
21068 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
21072 assert_eq!(
21073 rig.refused(opener, target, scope()).await,
21074 error_codes::SCOPE_NOT_CARRIER,
21075 "{opener:?} -> {target}"
21076 );
21077 }
21078 }
21079
21080 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
21081 ScopeSelector::new(
21082 Principal::Reserved {
21083 module_id: OWNER.to_string(),
21084 },
21085 scope_ref.to_string(),
21086 scope_epoch,
21087 )
21088 }
21089
21090 #[tokio::test]
21091 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
21092 let mut rig = rig().await;
21093 rig.sync(vec![session(1)]).await;
21094 for opener in [OWNER, AFT] {
21095 assert_eq!(
21096 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
21097 .await,
21098 error_codes::SCOPE_EPOCH_REQUIRED,
21099 "{opener}"
21100 );
21101 }
21102 }
21103
21104 #[tokio::test]
21105 async fn admission_separates_not_synced_not_live_and_ended() {
21106 let mut rig = rig().await;
21107 let code = rig
21109 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21110 .await;
21111 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
21112 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
21113
21114 let ghost = ScopeSelector::new(
21116 Principal::Reserved {
21117 module_id: "ghost".to_string(),
21118 },
21119 "s".to_string(),
21120 Some(1),
21121 );
21122 assert_eq!(
21123 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
21124 error_codes::SCOPE_NOT_LIVE
21125 );
21126
21127 rig.sync(vec![session(2)]).await;
21128 assert_eq!(
21129 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
21130 .await,
21131 error_codes::SCOPE_NOT_LIVE
21132 );
21133 for epoch in [1, 3] {
21134 assert_eq!(
21135 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
21136 .await,
21137 error_codes::SCOPE_ENDED,
21138 "epoch {epoch}"
21139 );
21140 }
21141 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
21143 .await;
21144 }
21145
21146 #[tokio::test]
21147 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
21148 let mut rig = rig().await;
21149 rig.sync(vec![session(1)]).await;
21150 let route = rig
21151 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21152 .await;
21153 let stamp = route.stamp().expect("a scoped bind carries the stamp");
21154 assert_eq!(stamp.scope_ref, "s");
21155 assert_eq!(stamp.scope_epoch, 1);
21156 assert_eq!(stamp.kind, ScopeKind::Head);
21157 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
21158 assert!(stamp.attributes.delegates);
21159 assert!(stamp.owner_authorized);
21160 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21161 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
21162
21163 let (broca, mut broca_rx) = wide_ctx(50);
21166 hello_via_sink(
21167 &rig.handler,
21168 &broca,
21169 &mut broca_rx,
21170 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
21171 )
21172 .await;
21173 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
21174 .await
21175 .unwrap();
21176 let own = ScopeSelector::new(
21177 Principal::Reserved {
21178 module_id: BROCA.to_string(),
21179 },
21180 "b".to_string(),
21181 Some(1),
21182 );
21183 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
21184 assert!(!route.stamp().unwrap().owner_authorized);
21185 }
21186
21187 #[tokio::test]
21188 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
21189 let mut rig = rig().await;
21190 let mut record = head("s", 1);
21191 record.carriers = vec![carrier(AFT, None)];
21192 let flow_id = "Flow:run-7/step_2!~";
21193 record.attributes.flow_id = Some(flow_id.to_string());
21194 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
21195 .await
21196 .unwrap();
21197 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
21198 panic!("not a sync reply");
21199 };
21200 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
21201 let route = rig
21202 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21203 .await;
21204 assert!(rig.live(&route), "the stamped bind committed");
21205 let stamp = route.stamp().expect("a flow scope carries a stamp");
21206 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
21207 assert_eq!(stamp.attributes.agent_id, None);
21208 assert!(!stamp.attributes.delegates);
21209 assert!(stamp.owner_authorized);
21210 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21211 assert_eq!(unscoped.stamp(), None);
21212 }
21213
21214 #[tokio::test]
21215 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
21216 let mut rig = rig_with_flow_support(false).await;
21217 let mut record = session(1);
21218 record.attributes.flow_id = Some("flow:7".to_string());
21219 rig.sync(vec![record]).await;
21220 for opener in [OWNER, AFT] {
21221 let body = rig
21222 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21223 .await;
21224 assert_eq!(body["code"], "target_flow_unsupported");
21225 let message = body["message"].as_str().unwrap();
21226 for required in [PLEXUS, "flow-scopes/v1"] {
21227 assert!(message.contains(required), "{message}");
21228 }
21229 }
21230 }
21231
21232 #[tokio::test]
21233 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
21234 let mut rig = rig_with_flow_support(true).await;
21235 let mut record = session(1);
21236 record.attributes.flow_id = Some("flow:7".to_string());
21237 rig.sync(vec![record]).await;
21238 for opener in [OWNER, AFT] {
21239 let route = rig
21240 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21241 .await;
21242 assert!(rig.live(&route));
21243 assert_eq!(
21244 route.stamp().unwrap().attributes.flow_id.as_deref(),
21245 Some("flow:7")
21246 );
21247 }
21248 }
21249
21250 #[tokio::test]
21251 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
21252 use std::future::Future;
21253
21254 let mut rig = rig_with_flow_support(true).await;
21255 let mut record = session(1);
21256 record.attributes.flow_id = Some("flow:7".to_string());
21257 rig.sync(vec![record]).await;
21258 let (client, mut client_rx, frame) =
21259 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
21260 for _ in 0..64 {
21263 client.egress.try_send(route_bind_ack(1)).unwrap();
21264 }
21265 let handler = rig.handler.clone();
21266 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21267 std::future::poll_fn(|cx| {
21268 assert!(open.as_mut().poll(cx).is_pending());
21269 std::task::Poll::Ready(())
21270 })
21271 .await;
21272 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21273
21274 let old_connection = rig.modules[PLEXUS].0.connection_id;
21275 rig.handler.cleanup_connection(old_connection).unwrap();
21276 let (replacement, mut replacement_rx) = wide_ctx(200);
21277 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
21278 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
21279 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
21280 let hello = Frame::build(
21281 FrameType::Hello,
21282 control_flags(),
21283 0,
21284 0,
21285 200,
21286 serde_json::to_vec(&body).unwrap(),
21287 )
21288 .unwrap();
21289 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
21290
21291 client_rx.try_recv().unwrap();
21292 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21293 .await
21294 .expect("the replacement is refused without waiting for a bind ack")
21295 .unwrap();
21296 assert_eq!(replies.len(), 1);
21297 let body = parse_error(&replies[0]);
21298 assert_eq!(body["code"], "target_flow_unsupported");
21299 for required in [PLEXUS, "flow-scopes/v1"] {
21300 assert!(body["message"].as_str().unwrap().contains(required));
21301 }
21302 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
21303 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21304 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21305 assert!(rig
21306 .handler
21307 .registry
21308 .get_module_by_connection(replacement.connection_id)
21309 .unwrap()
21310 .is_some());
21311 }
21312
21313 #[tokio::test]
21314 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
21315 use std::future::Future;
21316 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
21317 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21318 const NEXT_HASH: &str =
21319 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21320 let mut rig = rig().await;
21321 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
21322 record.carriers.push(ScopeCarrier::new(Principal::Direct));
21323 rig.sync(vec![record.clone()]).await;
21324 let (client, mut rx, frame) = rig.open_frame(
21325 None,
21326 PLEXUS,
21327 Some(
21328 rig.selector("s", Some(1))
21329 .with_bind_proof(Some(PROOF.into())),
21330 ),
21331 );
21332 for _ in 0..64 {
21335 client.egress.try_send(route_bind_ack(1)).unwrap();
21336 }
21337 let handler = rig.handler.clone();
21338 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21339 std::future::poll_fn(|cx| {
21340 assert!(open.as_mut().poll(cx).is_pending());
21341 std::task::Poll::Ready(())
21342 })
21343 .await;
21344 record.bind_proof_sha256 = Some(NEXT_HASH.into());
21345 rig.sync(vec![record]).await;
21346 rx.try_recv().unwrap();
21347 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21348 .await
21349 .unwrap()
21350 .unwrap();
21351 assert_eq!(
21352 parse_error(&replies[0])["code"],
21353 error_codes::SCOPE_BIND_PROOF_MISMATCH
21354 );
21355 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
21356 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21357 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21358 }
21359
21360 #[tokio::test]
21361 async fn scope_bind_proof_reserved_openers_ignore_proof() {
21362 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21363 let mut rig = rig().await;
21364 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21365 .await;
21366 for opener in [OWNER, AFT] {
21367 for proof in [None, Some("malformed+proof=")] {
21368 let route = rig
21369 .bound(
21370 Some(opener),
21371 PLEXUS,
21372 Some(
21373 rig.selector("s", Some(1))
21374 .with_bind_proof(proof.map(str::to_string)),
21375 ),
21376 )
21377 .await;
21378 assert!(rig.live(&route));
21379 }
21380 }
21381 }
21382
21383 #[tokio::test]
21384 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
21385 use std::future::Future;
21386 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21387 const NEXT_HASH: &str =
21388 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21389 let mut rig = rig().await;
21390 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21391 .await;
21392 let (client, mut rx, frame) =
21393 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
21394 for _ in 0..64 {
21395 client.egress.try_send(route_bind_ack(1)).unwrap();
21396 }
21397 let handler = rig.handler.clone();
21398 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21399 std::future::poll_fn(|cx| {
21400 assert!(open.as_mut().poll(cx).is_pending());
21401 std::task::Poll::Ready(())
21402 })
21403 .await;
21404 rig.sync(vec![
21405 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
21406 ])
21407 .await;
21408 rx.try_recv().unwrap();
21409 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21410 let bind = std::future::poll_fn(|cx| {
21411 assert!(open.as_mut().poll(cx).is_pending());
21412 module_rx.poll_recv(cx)
21413 })
21414 .await
21415 .unwrap()
21416 .frame;
21417 handler
21418 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
21419 .await
21420 .unwrap();
21421 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21422 .await
21423 .unwrap()
21424 .unwrap();
21425 assert_eq!(replies.len(), 1);
21426 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
21429 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21430 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
21431 let route = rig
21432 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
21433 .await;
21434 assert!(rig.live(&route));
21435 }
21436
21437 #[tokio::test]
21438 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
21439 {
21440 let mut rig = rig_with_flow_support(false).await;
21441 rig.sync(vec![session(1)]).await;
21442 let route = rig
21443 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21444 .await;
21445 assert!(rig.live(&route));
21446 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
21447 let mut record = session(1);
21448 record.attributes.flow_id = Some("flow:7".to_string());
21449 rig.sync(vec![record]).await;
21450 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
21451 assert!(rig.live(&unscoped));
21452 assert_eq!(unscoped.stamp(), None);
21453 }
21454
21455 #[tokio::test]
21456 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
21457 let mut rig = rig().await;
21458 let mut record = session(1);
21459 record.attributes.flow_id = Some("flow:7".to_string());
21460 rig.sync(vec![record.clone()]).await;
21461 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21462 let mut owner_route = rig
21463 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21464 .await;
21465 let mut carrier_route = rig
21466 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
21467 .await;
21468 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21469 rig.sync(vec![record.clone()]).await;
21470 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21471 assert!(rig.live(&owner_route) && owner_route.untouched());
21472 assert!(rig.live(&carrier_route) && carrier_route.untouched());
21473
21474 record.attributes.flow_id = Some("flow:8".to_string());
21475 rig.sync(vec![record]).await;
21476 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
21477 let before = before.unwrap();
21478 assert_eq!(after.scope_epoch, before.scope_epoch);
21479 assert!(after.version > before.version);
21480 for route in [&mut owner_route, &mut carrier_route] {
21481 assert!(!rig.live(route));
21482 assert_eq!(
21483 route.closed_reason(),
21484 RouteCloseReason::ScopeDelegationChanged
21485 );
21486 }
21487 assert!(rig.live(&unscoped) && unscoped.untouched());
21488 for target in [PLEXUS, OTHER] {
21491 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
21492 let goodbye = module_rx
21493 .try_recv()
21494 .expect("the provider sees the drain")
21495 .frame;
21496 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21497 assert!(module_rx.try_recv().is_err());
21498 }
21499 let rebound = rig
21500 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21501 .await;
21502 assert_eq!(
21503 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
21504 Some("flow:8")
21505 );
21506 }
21507
21508 #[tokio::test]
21513 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
21514 let mut rig = rig().await;
21515 rig.sync(vec![session(1)]).await;
21516 let mut cotenant = rig
21517 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21518 .await;
21519
21520 let mut changed = session(1);
21521 changed.child_owners.push(Principal::Reserved {
21522 module_id: MAGIC.to_string(),
21523 });
21524 let mut ended = None;
21525 for (code, next) in [
21526 (error_codes::SCOPE_CHANGED, vec![changed]),
21527 (error_codes::SCOPE_ENDED, Vec::new()),
21528 ] {
21529 let relayed = rig
21530 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21531 .await;
21532 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
21533 ended = Some(next.is_empty());
21534 rig.sync(next).await;
21535 rig.ack(&relayed).await;
21536 let replies = relayed.task.await.unwrap();
21537 assert_eq!(replies.len(), 1, "{replies:?}");
21538 assert_eq!(parse_error(&replies[0])["code"], code);
21539 assert_eq!(
21540 subc_protocol::error_codes::is_retryable_route_open(code),
21541 code == error_codes::SCOPE_CHANGED
21542 );
21543 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21544 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21546 let mut goodbyes = Vec::new();
21549 while let Ok(outbound) = plexus_rx.try_recv() {
21550 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
21551 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
21552 }
21553 assert!(
21554 goodbyes.contains(&(bind_channel, bind_epoch)),
21555 "{goodbyes:?}"
21556 );
21557 assert!(rig
21558 .handler
21559 .registry
21560 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
21561 .unwrap()
21562 .is_some());
21563 }
21564 assert_eq!(ended, Some(true));
21565 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
21568 }
21569
21570 #[tokio::test]
21573 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
21574 struct Case {
21575 name: &'static str,
21576 change: fn(&mut ScopeRecord),
21577 closed: [Option<RouteCloseReason>; 4],
21580 }
21581 use RouteCloseReason::*;
21582 let cases = [
21583 Case {
21584 name: "a carrier entry removed",
21585 change: |r| {
21586 r.carriers.retain(|c| {
21587 c.principal
21588 != Principal::Reserved {
21589 module_id: AFT.to_string(),
21590 }
21591 })
21592 },
21593 closed: [None, Some(ScopeCarrierRemoved), None, None],
21594 },
21595 Case {
21596 name: "a target removed from a carrier",
21597 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
21598 closed: [None, None, None, Some(ScopeCarrierRemoved)],
21599 },
21600 Case {
21601 name: "a bare carrier narrowed to targets",
21602 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
21603 closed: [None, Some(ScopeCarrierRemoved), None, None],
21604 },
21605 Case {
21606 name: "delegates turned off",
21607 change: |r| r.attributes.delegates = false,
21608 closed: [Some(ScopeDelegationChanged); 4],
21609 },
21610 Case {
21611 name: "agent_id changed",
21612 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
21613 closed: [Some(ScopeDelegationChanged); 4],
21614 },
21615 Case {
21616 name: "a carrier added, child owners changed, the record re-sent",
21617 change: |r| {
21618 r.carriers.push(carrier(MAGIC, None));
21619 r.child_owners.push(Principal::Reserved {
21620 module_id: MAGIC.to_string(),
21621 });
21622 },
21623 closed: [None; 4],
21624 },
21625 Case {
21626 name: "a target added",
21627 change: |r| {
21628 r.carriers[1]
21629 .targets
21630 .as_mut()
21631 .unwrap()
21632 .push("third".to_string())
21633 },
21634 closed: [None; 4],
21635 },
21636 Case {
21637 name: "delegates turned on",
21638 change: |r| r.attributes.delegates = true,
21639 closed: [None; 4],
21640 },
21641 ];
21642 for case in cases {
21643 let mut rig = rig().await;
21644 rig.sync(vec![session(1)]).await;
21645 let scope = || Some(rig_selector("s", Some(1)));
21646 let mut routes = [
21647 rig.bound(Some(OWNER), PLEXUS, scope()).await,
21648 rig.bound(Some(AFT), PLEXUS, scope()).await,
21649 rig.bound(Some(BROCA), PLEXUS, scope()).await,
21650 rig.bound(Some(BROCA), OTHER, scope()).await,
21651 ];
21652 let mut record = session(1);
21653 (case.change)(&mut record);
21654 rig.sync(vec![record]).await;
21655 for (index, expected) in case.closed.iter().enumerate() {
21656 let route = &mut routes[index];
21657 match expected {
21658 Some(reason) => {
21659 assert!(!rig.live(route), "{}: route {index} still live", case.name);
21660 assert_eq!(
21661 route.closed_reason(),
21662 *reason,
21663 "{}: route {index}",
21664 case.name
21665 );
21666 }
21667 None => {
21668 assert!(rig.live(route), "{}: route {index} closed", case.name);
21669 assert!(
21670 route.untouched(),
21671 "{}: route {index} was told something",
21672 case.name
21673 );
21674 }
21675 }
21676 }
21677 }
21678 }
21679
21680 #[tokio::test]
21681 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
21682 for next in [Vec::new(), vec![session(2)]] {
21684 let mut rig = rig().await;
21685 rig.sync(vec![session(1)]).await;
21686 let mut route = rig
21687 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21688 .await;
21689 rig.sync(next).await;
21690 assert!(!rig.live(&route));
21691 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
21692 }
21693
21694 let mut rig = rig().await;
21697 let mut child = session(1);
21698 child.scope_ref = "child".to_string();
21699 child.kind = ScopeKind::Worker;
21700 child.parent = Some(ScopeParent::new(
21701 Principal::Reserved {
21702 module_id: OWNER.to_string(),
21703 },
21704 "s".to_string(),
21705 1,
21706 ));
21707 rig.sync(vec![session(1), child.clone()]).await;
21708 let mut child_route = rig
21709 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
21710 .await;
21711 assert_eq!(
21712 child_route.stamp().unwrap().parent_state,
21713 Some(ParentState::Linked)
21714 );
21715 rig.sync(vec![child]).await;
21716 assert!(!rig.live(&child_route));
21717 assert_eq!(
21718 child_route.closed_reason(),
21719 RouteCloseReason::ScopeParentEnded
21720 );
21721 }
21722
21723 #[tokio::test]
21724 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
21725 ) {
21726 let mut rig = rig().await;
21727 rig.sync(vec![session(1)]).await;
21728 let mut route = rig
21729 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21730 .await;
21731 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21732 rig.sync(vec![session(1)]).await;
21733 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21734 assert!(rig.live(&route) && route.untouched());
21735
21736 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
21740 .forwarding
21741 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21742 .unwrap()
21743 else {
21744 panic!("the route is bound");
21745 };
21746 binding.flow.acquire_tagged(9, false).await.unwrap();
21747 let mut widened = session(1);
21748 widened.carriers.push(carrier(MAGIC, None));
21749 rig.sync(vec![widened]).await;
21750 assert!(rig.live(&route) && route.untouched());
21751 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21752 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
21753 assert_eq!(binding.flow.in_flight(), 1);
21756 binding
21757 .flow
21758 .acquire_tagged(10, false)
21759 .await
21760 .expect("the flow is still open");
21761 }
21762
21763 #[tokio::test]
21766 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
21767 let mut rig = rig().await;
21768 rig.sync(vec![session(1)]).await;
21769 let mut on_incumbent = rig
21770 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21771 .await;
21772
21773 let (candidate, _candidate_rx) = wide_ctx(9);
21776 let registration = rig
21777 .handler
21778 .registry
21779 .register_candidate_with_control_ops(
21780 manifest(PLEXUS, PROTOCOL_VERSION),
21781 PROTOCOL_VERSION,
21782 candidate.connection_id,
21783 module_baseline_control_ops(),
21784 )
21785 .unwrap();
21786 rig.forwarding
21787 .register_candidate_module_connection(
21788 candidate.connection_id,
21789 PLEXUS.to_string(),
21790 PROTOCOL_VERSION,
21791 manifest_concurrency(®istration.manifest),
21792 candidate.egress.clone(),
21793 )
21794 .unwrap();
21795 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
21796 rig.handler
21797 .registry
21798 .promote_candidate(PLEXUS)
21799 .unwrap()
21800 .unwrap();
21801 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
21802
21803 rig.sync(Vec::new()).await;
21804 assert!(!rig.live(&on_incumbent));
21805 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
21806 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21807 let goodbye = incumbent_rx
21808 .try_recv()
21809 .expect("the superseded endpoint is told")
21810 .frame;
21811 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21812 }
21813 }
21814}
21815
21816#[cfg(test)]
21817mod concurrency_default_exposure_tests {
21818 use super::*;
21819
21820 fn hello_body(role_json: &str) -> Vec<u8> {
21821 format!(
21822 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
21823 )
21824 .into_bytes()
21825 }
21826
21827 fn manifest_from(body: &[u8]) -> ModuleManifest {
21828 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
21829 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
21830 .expect("manifest parses")
21831 }
21832
21833 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
21834
21835 #[test]
21836 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
21837 let body = hello_body(&format!(
21838 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
21839 ));
21840 let manifest = manifest_from(&body);
21841 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21844 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
21845 }
21846
21847 #[test]
21848 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
21849 let body = hello_body(&format!(
21850 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
21851 ));
21852 let manifest = manifest_from(&body);
21853 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21854 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21855 }
21856
21857 #[test]
21858 fn non_management_roles_are_never_reported() {
21859 let body = hello_body(
21860 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
21861 );
21862 let manifest = manifest_from(&body);
21863 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21864 }
21865}