1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26 error_codes,
27 manifest::{
28 validate_hello_capability_grammar, validate_hello_confirm_operations,
29 validate_hello_display_name, validate_hello_event_declarations,
30 validate_hello_flow_operations, validate_hello_operation_deferral,
31 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32 validate_hello_self_signal_declarations, validate_role_declarations_value,
33 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34 ProviderRole,
35 },
36 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
37 provider_declarations::ProviderDeclarations,
38 scope::{
39 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
40 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
41 SCOPE_SYNC_OP,
42 },
43 session::{
44 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
45 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
46 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
47 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
48 },
49 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
50 Priority, RouteTarget, PROTOCOL_VERSION,
51};
52use tokio::time::{timeout_at, Instant};
53use tracing::{debug, info, instrument::WithSubscriber, warn};
54
55use crate::{
56 caller_identity::CallerIdentity,
57 capability_requirements::{
58 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
59 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
60 RequirementStatus, RuntimeModule,
61 },
62 child_roster::DaemonShutdownFlag,
63 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
64 forwarding::{
65 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
66 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
67 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
68 },
69 observability::{
70 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
71 },
72 privileged_prompt::{prompt_text, PrivilegedOperation},
73 provenance::{
74 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
75 },
76 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
77 router::{daemon_error_flags, RouteCtx, RouterError},
78 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
79 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
80 stderr_tail::{CaptureState, TailEntry},
81 supervise::{
82 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
83 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
84 },
85 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
86};
87
88pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
94
95const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
96const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
97const CAP_PING_PONG: &str = "ping_pong_v1";
98const CAP_SESSION_ATTACH: &str = "session_attach_v1";
99const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
100
101const SUBC_CONTROL_OPS: &[&str] = &[
102 "claim.mint",
103 "claim.present",
104 ops::DAEMON_SHUTDOWN_INTENT,
105 ops::ACTIVITY_RECORD,
106 ops::ACTIVITY_QUERY,
107 ops::ACTIVITY_SUBSCRIBE,
108 ops::SERVER_DESCRIBE,
109 ops::CATALOG_LIST,
110 ops::CATALOG_WATCH,
111 ops::ROUTE_OPEN,
112 ops::ROUTE_POLL,
113 ops::ROUTE_CLOSING,
114 ops::ROUTE_CLOSED,
115 ops::SUPERVISOR_LIST,
116 ops::SUPERVISOR_RESTART,
117 ops::SUPERVISOR_SWAP,
118 ops::SUPERVISOR_RELOAD,
119 ops::SUPERVISOR_RESCAN,
120 ops::SUPERVISOR_RELEASE_RESERVED,
121 ops::SUPERVISOR_SET_ENABLED,
122 ops::SUPERVISOR_HEALTH_PROBE,
123 ops::SUPERVISOR_HEALTH,
124 ops::SUPERVISOR_STDERR_TAIL,
125 ops::SUPERVISOR_TERMINALS,
126 ops::SUPERVISOR_ROUTES,
127 ops::SUPERVISOR_PROVENANCE,
128 ops::SUPERVISOR_SPAWN_SNAPSHOT,
129 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
130];
131
132const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[
139 ops::SUPERVISOR_RESCAN_CHECKED,
140 "config.status",
141 "config.plan",
142 "config.apply",
143 "config.stamp",
144 "config.unstamp",
145 "config.history",
146 "config.rollback",
147];
148
149mod caller_attestation;
150mod channel_operations;
151#[cfg(all(test, unix, feature = "test-support"))]
152#[path = "control/channel_operations_tests.rs"]
153mod channel_operations_tests;
154pub(crate) mod claims;
155#[cfg(test)]
156#[path = "control/config_advertisement_tests.rs"]
157mod config_advertisement_tests;
158mod config_gate;
159use caller_attestation::{caller_for_target, scope_acts_for_another};
160#[cfg(all(test, unix, feature = "test-support"))]
161#[path = "control/caller_attestation_tests.rs"]
162mod caller_attestation_tests;
163mod scope_expiry;
164
165mod agent_commands;
166pub(crate) mod catalog;
167mod rescan_checked;
168use rescan_checked::{RescanMode, RescanPlan};
169
170const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
171 "claim.mint",
172 "claim.present",
173 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
174 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
175 "supervisor.live_roots",
176 "supervisor.agent_commands",
177 SCOPE_SYNC_OP,
178 SCOPE_APPLY_OP,
179 SCOPE_DESCRIBE_OP,
180 subc_protocol::scope::ROUTE_AUTHORIZE_OP,
181 "operator.confirm",
182];
183
184const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
188
189const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
190
191pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
199
200pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
215
216pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
234
235const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
236const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
237
238fn reload_verdict(
239 configured: &Path,
240 spawned_from: Option<&Path>,
241 image: subc_control::RunningImageAgreement,
242) -> PendingReloadVerdict {
243 let path = match spawned_from {
244 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
245 ReloadPathAgreement::Match
246 }
247 Some(spawned_from) => ReloadPathAgreement::Mismatch {
248 configured: configured.to_path_buf(),
249 spawned_from: spawned_from.to_path_buf(),
250 },
251 None => ReloadPathAgreement::Unavailable {
252 reason: if matches!(
253 image,
254 subc_control::RunningImageAgreement::Unavailable {
255 reason: subc_control::RunningImageUnavailableReason::NotRunning
256 }
257 ) {
258 ReloadPathUnavailableReason::NotRunning
259 } else {
260 ReloadPathUnavailableReason::SpawnedPathUnavailable
261 },
262 },
263 };
264 PendingReloadVerdict::new(path, image)
265}
266
267fn launch_spec_agreement(
268 configured: &crate::supervise::ModuleSpec,
269 running: bool,
270 spawned: Option<&crate::supervise::LaunchSpecDigest>,
271) -> subc_control::LaunchSpecAgreement {
272 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
273 if !running {
274 return Agreement::Unavailable {
275 reason: Reason::NotRunning,
276 };
277 }
278 match spawned {
279 None => Agreement::Unavailable {
280 reason: Reason::SpawnedSpecUnavailable,
281 },
282 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
283 Agreement::Match
284 }
285 Some(_) => Agreement::Mismatch,
286 }
287}
288
289fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
290 if configured == spawned_from {
291 return true;
292 }
293 #[cfg(windows)]
294 {
295 match (
301 std::fs::canonicalize(configured),
302 std::fs::canonicalize(spawned_from),
303 ) {
304 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
305 _ => false,
306 }
307 }
308 #[cfg(not(windows))]
309 {
310 false
311 }
312}
313
314#[derive(Clone)]
315struct DaemonProvenanceFacts {
316 build: DaemonBuildProvenance,
317 pid: Option<u32>,
318 started_at_ms: Option<u64>,
319 start_clock: Option<crate::clock::StartClock>,
320 executable_path: Option<PathBuf>,
321 executable_identity: Option<SpawnedFileIdentity>,
322 process_start_time: Option<u64>,
323 probe: ExecutableIdentityProbe,
324}
325
326impl Default for DaemonProvenanceFacts {
327 fn default() -> Self {
328 Self {
329 build: DaemonBuildProvenance {
330 build_git_sha: None,
331 build_lock_digest: None,
332 },
333 pid: None,
334 started_at_ms: None,
335 start_clock: None,
336 executable_path: None,
337 executable_identity: None,
338 process_start_time: None,
339 probe: ExecutableIdentityProbe::default(),
340 }
341 }
342}
343
344#[derive(Debug, Clone)]
345struct SupervisorRescanContext {
346 supervisor: Supervisor,
347 config_path: PathBuf,
348 configured_port: Option<u16>,
349 storage_config: Option<crate::daemon_config::StorageConfig>,
350 admission_facts_carrier_module_id: Option<String>,
351 admission_facts_targets: Option<Vec<String>>,
352 scope_authority_owners: Vec<String>,
353 federated_scope_owner: String,
354 privileged_operator_prompt: PrivilegedOperatorPrompt,
355 generation: Arc<std::sync::atomic::AtomicU64>,
361 module_entries: Arc<Mutex<rescan_checked::ModuleEntries>>,
362}
363
364const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
385 "reloading",
386 "supervisor_not_live",
387 "registration_not_active",
388 "no_forwarding_connection",
389 "relay_send_failed",
390];
391
392#[cfg(test)]
394struct ActivityRequestPause {
395 connection: ConnectionId,
396 started: tokio::sync::Notify,
397 cancelled: tokio::sync::Notify,
398 cleanup_started: std::sync::atomic::AtomicBool,
399 cancelled_after_cleanup: std::sync::atomic::AtomicBool,
400}
401
402#[cfg(test)]
403struct ActivityRequestCancellation(Arc<ActivityRequestPause>);
404
405#[cfg(test)]
406impl Drop for ActivityRequestCancellation {
407 fn drop(&mut self) {
408 self.0.cancelled_after_cleanup.store(
409 self.0
410 .cleanup_started
411 .load(std::sync::atomic::Ordering::Acquire),
412 std::sync::atomic::Ordering::Release,
413 );
414 self.0.cancelled.notify_one();
415 }
416}
417
418#[cfg(test)]
420type TestPause = (
421 tokio::sync::oneshot::Sender<()>,
422 std::sync::mpsc::Receiver<()>,
423);
424
425#[cfg(test)]
427type TestHook = Box<dyn FnOnce() + Send>;
428
429#[cfg(test)]
431type AsyncTestPause = (
432 tokio::sync::oneshot::Sender<()>,
433 tokio::sync::oneshot::Receiver<()>,
434);
435
436#[derive(Clone)]
438pub struct ControlHandler {
439 #[cfg(test)]
440 activity_request_pause: Arc<Mutex<Option<Arc<ActivityRequestPause>>>>,
441 #[cfg(test)]
445 cleanup_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
446 #[cfg(test)]
449 after_nonce_attestation: Arc<Mutex<Option<TestHook>>>,
450 #[cfg(test)]
453 hello_ingress_pause: Arc<Mutex<Option<AsyncTestPause>>>,
454 #[cfg(test)]
456 agent_commands_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
457 claims: Arc<claims::Claims>,
458 registry: Arc<Registry>,
459 config_runtime: Option<Arc<crate::config_gate::runtime::RuntimeOwner>>,
460 config_review_admission: Arc<dyn crate::config_gate::ReviewAdmission>,
461 config_review_admission_is_default: bool,
462 activity: Option<crate::activity::ActivityLedger>,
463 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
464 catalog_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
465 forwarding: Arc<ForwardingTable>,
466 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
467 supervisor: SupervisorHandle,
468 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
469 agent_declarations: agent_commands::Queues,
470 list_admission: Arc<tokio::sync::Semaphore>,
471 subc_capabilities: Arc<[String]>,
472 route_bind_relay_timeout: Duration,
476 route_bind_relay_timeouts: BTreeMap<String, Duration>,
480 route_bind_breakers: RouteBindBreakers,
483 route_bind_concurrency: RouteBindConcurrency,
486 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
491 route_bind_breaker_threshold: u32,
493 route_bind_breaker_cooldown: Duration,
495 health_probe_timeout: Duration,
496 storage_config: Option<crate::daemon_config::StorageConfig>,
499 machine_id: Option<crate::machine_id::MachineId>,
503 admission_facts_carrier_module_id: Option<String>,
504 admission_facts_targets: Option<Vec<String>>,
505 scopes: Arc<RwLock<ScopeTable>>,
509 scope_deadline_added: Arc<tokio::sync::Notify>,
513 scope_authority_owners: Vec<String>,
516 federated_scope_owner: String,
517 privileged_operator_prompt: PrivilegedOperatorPrompt,
519 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
522 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
523 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
524 captured_peers: Arc<Mutex<HashMap<ConnectionId, subc_os::local_peer::CapturedPeer>>>,
525 config_desktop_stamps:
526 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
527 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
528 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
529 pub(crate) operator_apps:
530 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
531 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
536 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
542 #[cfg(test)]
544 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
545 #[cfg(test)]
546 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
547 #[cfg(test)]
548 recovery_exit_test_after_step: Option<fn(usize) -> Result<(), crate::config_gate::GateError>>,
549 #[cfg(test)]
550 recovery_exit_test_completed: Option<fn(u64)>,
551 #[cfg(test)]
552 apply_reconcile_pause: Arc<Mutex<Option<AsyncTestPause>>>,
553 #[cfg(test)]
554 recovery_exit_test_quiet_retry: bool,
555 #[cfg(test)]
556 recovery_exit_test_retry_pause: Option<fn(u64) -> Option<PathBuf>>,
557 #[cfg(test)]
558 config_delivery_test_wakes:
559 Option<tokio::sync::mpsc::UnboundedSender<(&'static str, tokio::time::Instant)>>,
560 #[cfg(all(test, unix))]
561 config_delivery_test_idle:
562 Option<tokio::sync::mpsc::UnboundedSender<tokio::sync::oneshot::Sender<()>>>,
563 #[allow(dead_code)]
567 #[cfg(windows)]
568 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
569 #[allow(dead_code)]
570 #[cfg(windows)]
571 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
572 #[allow(dead_code)]
576 #[cfg(windows)]
577 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
578 #[cfg(all(test, windows))]
581 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
582 #[cfg(test)]
583 caller_lookup: Option<TestCallerLookup>,
584 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
587 rescan: Option<SupervisorRescanContext>,
588 connected_clients: ConnectedClients,
589 counters: DaemonCounters,
590 local_endpoint_unavailable: Option<String>,
593 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
594 daemon_shutdown: DaemonShutdownFlag,
599 daemon_provenance: DaemonProvenanceFacts,
600 #[cfg(test)]
601 control_dispatch_delay: Option<Duration>,
602 #[cfg(test)]
603 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
604 #[cfg(test)]
608 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
609 #[cfg(test)]
615 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
616 #[cfg(test)]
617 hello_ack_to_publication_hold: Option<Arc<std::sync::Barrier>>,
618 #[cfg(test)]
619 hello_before_publication_hold: Option<Arc<std::sync::Barrier>>,
620 #[cfg(test)]
621 hello_after_launch_check_hold: Option<Arc<std::sync::Barrier>>,
622 #[cfg(test)]
623 hello_inside_publication_hold: Option<Arc<std::sync::Barrier>>,
624}
625
626impl fmt::Debug for ControlHandler {
627 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
628 f.debug_struct("ControlHandler")
629 .field("registry", &self.registry)
630 .field("forwarding", &self.forwarding)
631 .field("process_liveness", &self.process_liveness.is_some())
632 .field("supervisor", &self.supervisor)
633 .field("subc_capabilities", &self.subc_capabilities)
634 .finish()
635 }
636}
637
638struct RouteOpenRequest {
639 target: RouteTarget,
640 identity: BindIdentity,
641 consumer_identity: Option<ConsumerIdentity>,
642 consumer_capabilities: Option<Vec<String>>,
643 role_versions: Option<BTreeMap<String, String>>,
644 busy_refusals: Option<bool>,
645 admission_facts: Option<serde_json::Value>,
646 scope: Option<ScopeSelector>,
647 accepts_close_reasons: Option<Vec<String>>,
648}
649
650#[derive(Clone, Copy)]
651enum RouteOpenPhase {
652 Admission,
653 ProjectRoot,
654 EgressReservation,
655 BindSend,
656 BindAckWait,
657}
658
659struct RouteOpenTimings {
663 module_id: String,
664 principal: String,
665 admission: Duration,
666 project_root: Duration,
667 egress_reservation: Duration,
668 bind_send: Duration,
669 bind_ack_wait: Duration,
670 phase: Option<RouteOpenPhase>,
671 phase_started_at: Instant,
672}
673
674impl RouteOpenTimings {
675 fn new() -> Self {
676 Self {
677 module_id: String::new(),
678 principal: "unresolved".to_string(),
680 admission: Duration::ZERO,
681 project_root: Duration::ZERO,
682 egress_reservation: Duration::ZERO,
683 bind_send: Duration::ZERO,
684 bind_ack_wait: Duration::ZERO,
685 phase: Some(RouteOpenPhase::Admission),
686 phase_started_at: Instant::now(),
687 }
688 }
689
690 fn finish(&mut self) {
691 let elapsed = self.phase_started_at.elapsed();
692 match self.phase.take() {
693 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
694 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
695 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
696 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
697 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
698 None => {}
699 }
700 }
701
702 fn enter(&mut self, phase: RouteOpenPhase) {
703 self.finish();
704 self.phase = Some(phase);
705 self.phase_started_at = Instant::now();
706 }
707}
708
709struct RouteBindReservationGuard {
710 forwarding: Arc<ForwardingTable>,
711 endpoint: ModuleEndpointId,
712 relay_corr: u64,
713 armed: bool,
714}
715
716struct ModuleControlRpcGuard {
717 forwarding: Arc<ForwardingTable>,
718 endpoint: ModuleEndpointId,
719 corr: u64,
720 armed: bool,
721}
722
723impl ModuleControlRpcGuard {
724 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
725 Self {
726 forwarding,
727 endpoint,
728 corr,
729 armed: true,
730 }
731 }
732
733 fn disarm(&mut self) {
734 self.armed = false;
735 }
736}
737
738impl Drop for ModuleControlRpcGuard {
739 fn drop(&mut self) {
740 if self.armed {
741 let _ = self
742 .forwarding
743 .cancel_module_control_rpc(self.endpoint, self.corr);
744 }
745 }
746}
747
748impl RouteBindReservationGuard {
749 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
750 Self {
751 forwarding,
752 endpoint,
753 relay_corr,
754 armed: true,
755 }
756 }
757
758 fn release_and_disarm(&mut self) {
759 if !self.armed {
760 return;
761 }
762 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
763 self.endpoint,
764 self.relay_corr,
765 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
766 ) {
767 send_goodbye_target_best_effort(
768 &self.forwarding.counters(),
769 &target,
770 "canceled route.bind",
771 );
772 }
773 self.armed = false;
774 }
775
776 fn disarm(&mut self) {
777 self.armed = false;
778 }
779}
780
781impl Drop for RouteBindReservationGuard {
782 fn drop(&mut self) {
783 self.release_and_disarm();
784 }
785}
786
787#[derive(Debug, Clone, Default)]
817pub(crate) struct RouteBindBreakers {
818 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
819}
820
821#[derive(Debug, Clone, Default)]
822pub(crate) struct RouteBindConcurrency {
823 modules: Arc<Mutex<HashMap<String, usize>>>,
824}
825
826struct RouteBindConcurrencyGuard {
827 concurrency: RouteBindConcurrency,
828 module_id: String,
829}
830
831impl RouteBindConcurrency {
832 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
835 let mut modules = self
836 .modules
837 .lock()
838 .expect("route.bind concurrency mutex poisoned");
839 let in_flight = modules.entry(module_id.to_string()).or_default();
840 if *in_flight >= limit {
841 return Err(*in_flight);
842 }
843 *in_flight += 1;
844 Ok(RouteBindConcurrencyGuard {
845 concurrency: self.clone(),
846 module_id: module_id.to_string(),
847 })
848 }
849}
850
851impl Drop for RouteBindConcurrencyGuard {
852 fn drop(&mut self) {
853 let mut modules = self
854 .concurrency
855 .modules
856 .lock()
857 .expect("route.bind concurrency mutex poisoned");
858 let remove = {
859 let in_flight = modules
860 .get_mut(&self.module_id)
861 .expect("admitted route.bind has a concurrency entry");
862 *in_flight -= 1;
863 *in_flight == 0
864 };
865 if remove {
866 modules.remove(&self.module_id);
867 }
868 }
869}
870
871#[derive(Debug, Default)]
872struct ModuleBreakerState {
873 consecutive_timeouts: u32,
875 cooldown_until: Option<Instant>,
878 probe_in_flight: Option<Arc<()>>,
883}
884
885enum RouteBindAdmission<'a> {
887 Admitted {
888 guard: RouteBindBreakerGuard<'a>,
889 probe: bool,
892 },
893 Refused {
894 consecutive_timeouts: u32,
895 retry_in: Duration,
899 probe_in_flight: bool,
900 },
901}
902
903struct RouteBindBreakerGuard<'a> {
910 breakers: RouteBindBreakers,
911 module_id: &'a str,
912 probe_token: Option<Arc<()>>,
913 settled: bool,
914}
915
916impl RouteBindBreakerGuard<'_> {
917 fn record_accepted(&mut self) -> bool {
921 self.settled = true;
922 self.breakers.record_accepted(self.module_id)
923 }
924
925 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
928 self.settled = true;
929 self.breakers.record_timeout(
930 self.module_id,
931 self.probe_token.as_ref(),
932 threshold,
933 cooldown,
934 )
935 }
936
937 fn record_inconclusive(&mut self) {
946 self.settled = true;
947 self.breakers
948 .record_inconclusive(self.module_id, self.probe_token.as_ref());
949 }
950}
951
952impl Drop for RouteBindBreakerGuard<'_> {
953 fn drop(&mut self) {
954 if !self.settled {
955 self.breakers
956 .record_inconclusive(self.module_id, self.probe_token.as_ref());
957 }
958 }
959}
960
961struct BreakerOpened {
965 consecutive_timeouts: u32,
966 reopened_after_probe: bool,
969}
970
971impl RouteBindBreakers {
972 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
973 self.modules
974 .lock()
975 .expect("route.bind breaker mutex poisoned")
976 }
977
978 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
981 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
982 probe: probe_token.is_some(),
983 guard: RouteBindBreakerGuard {
984 breakers: self.clone(),
985 module_id,
986 probe_token,
987 settled: false,
988 },
989 };
990
991 let mut modules = self.lock();
992 let Some(state) = modules.get_mut(module_id) else {
993 return admitted(None);
994 };
995 let Some(cooldown_until) = state.cooldown_until else {
996 return admitted(None);
997 };
998 if state.probe_in_flight.is_some() {
999 return RouteBindAdmission::Refused {
1000 consecutive_timeouts: state.consecutive_timeouts,
1001 retry_in: Duration::ZERO,
1002 probe_in_flight: true,
1003 };
1004 }
1005 let now = Instant::now();
1006 if now < cooldown_until {
1007 return RouteBindAdmission::Refused {
1008 consecutive_timeouts: state.consecutive_timeouts,
1009 retry_in: cooldown_until - now,
1010 probe_in_flight: false,
1011 };
1012 }
1013 let token = Arc::new(());
1014 state.probe_in_flight = Some(Arc::clone(&token));
1015 admitted(Some(token))
1016 }
1017
1018 fn record_accepted(&self, module_id: &str) -> bool {
1019 self.lock()
1020 .remove(module_id)
1021 .is_some_and(|state| state.cooldown_until.is_some())
1022 }
1023
1024 fn record_timeout(
1025 &self,
1026 module_id: &str,
1027 probe_token: Option<&Arc<()>>,
1028 threshold: u32,
1029 cooldown: Duration,
1030 ) -> Option<BreakerOpened> {
1031 let mut modules = self.lock();
1032 let state = modules.entry(module_id.to_string()).or_default();
1033 let was_open = state.cooldown_until.is_some();
1034 let was_probe = Self::owns_probe(state, probe_token);
1035 if was_probe {
1036 state.probe_in_flight = None;
1037 }
1038 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
1039 if state.consecutive_timeouts < threshold {
1040 return None;
1041 }
1042 state.cooldown_until = Some(Instant::now() + cooldown);
1043 Some(BreakerOpened {
1044 consecutive_timeouts: state.consecutive_timeouts,
1045 reopened_after_probe: was_open && was_probe,
1046 })
1047 }
1048
1049 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
1050 state
1054 .probe_in_flight
1055 .as_ref()
1056 .zip(token)
1057 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
1058 }
1059
1060 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
1061 if let Some(state) = self.lock().get_mut(module_id) {
1062 if Self::owns_probe(state, probe_token) {
1063 state.probe_in_flight = None;
1064 }
1065 }
1066 }
1067
1068 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
1083 self.lock()
1084 .remove(module_id)
1085 .map(|state| state.consecutive_timeouts)
1086 .filter(|discarded| *discarded > 0)
1087 }
1088
1089 fn open_snapshot(&self) -> Option<serde_json::Value> {
1096 let now = Instant::now();
1097 let modules = self.lock();
1098 let open = modules
1099 .iter()
1100 .filter_map(|(module_id, state)| {
1101 let cooldown_until = state.cooldown_until?;
1102 Some((
1103 module_id.clone(),
1104 serde_json::json!({
1105 "consecutive_timeouts": state.consecutive_timeouts,
1106 "cooldown_remaining_ms":
1107 cooldown_until.saturating_duration_since(now).as_millis() as u64,
1108 "probe_in_flight": state.probe_in_flight.is_some(),
1109 }),
1110 ))
1111 })
1112 .collect::<serde_json::Map<String, serde_json::Value>>();
1113 (!open.is_empty()).then_some(serde_json::Value::Object(open))
1114 }
1115}
1116
1117impl ControlHandler {
1118 pub(crate) fn install_registration_service(
1119 &self,
1120 router: std::sync::Weak<crate::Router>,
1121 auth: crate::server::ServerAuth,
1122 ) {
1123 self.supervisor.install_registration_service(router, auth);
1124 }
1125 pub fn new(registry: Arc<Registry>) -> Self {
1126 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
1127 }
1128
1129 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
1130 let counters = forwarding.counters();
1131 let route_bind_breakers = forwarding.route_bind_breakers();
1135 let route_bind_concurrency = forwarding.route_bind_concurrency();
1136 let route_outages = forwarding.route_outages();
1137 Self {
1138 registry,
1139 #[cfg(test)]
1140 activity_request_pause: Arc::new(Mutex::new(None)),
1141 #[cfg(test)]
1142 cleanup_pause: Arc::new(Mutex::new(None)),
1143 #[cfg(test)]
1144 after_nonce_attestation: Arc::new(Mutex::new(None)),
1145 #[cfg(test)]
1146 hello_ingress_pause: Arc::new(Mutex::new(None)),
1147 #[cfg(test)]
1148 agent_commands_pause: Arc::new(Mutex::new(None)),
1149 claims: Arc::new(claims::Claims::default()),
1150 config_runtime: None,
1151 config_review_admission: Arc::new(crate::config_gate::MissingReviewAdmission),
1152 config_review_admission_is_default: true,
1153 activity: None,
1154 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1155 catalog_streams: Arc::new(Mutex::new(HashMap::new())),
1156 forwarding,
1157 process_liveness: None,
1158 supervisor: SupervisorHandle::new(),
1159 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1160 env!("CARGO_PKG_VERSION").to_owned(),
1161 )),
1162 agent_declarations: agent_commands::Queues::default(),
1163 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1164 subc_capabilities: Arc::from([
1165 CAP_MANIFEST_REGISTRATION.to_string(),
1166 CAP_CHANNEL_LIFECYCLE.to_string(),
1167 CAP_PING_PONG.to_string(),
1168 CAP_SESSION_ATTACH.to_string(),
1169 CAP_ADMISSION_FACTS_RELAY.to_string(),
1170 CAP_SCOPES_V1.to_string(),
1171 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1172 subc_protocol::peer::CALLER_ATTESTATION_CAPABILITY.to_string(),
1173 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1174 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1175 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1176 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1177 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1178 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1179 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1180 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1181 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1182 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1183 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1184 subc_protocol::ROUTE_CLOSE_REASONS_CAPABILITY.to_string(),
1185 ]),
1186 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1187 route_bind_relay_timeouts: BTreeMap::new(),
1188 route_bind_breakers,
1189 route_bind_concurrency,
1190 route_outages,
1191 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1192 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1193 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1194 storage_config: None,
1195 machine_id: None,
1196 admission_facts_carrier_module_id: None,
1197 admission_facts_targets: None,
1198 scopes: Arc::new(RwLock::new(ScopeTable::new(
1199 crate::daemon_config::default_scope_authority_owners(),
1200 ))),
1201 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1202 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1203 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1204 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1205 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1206 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1207 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1208 captured_peers: Arc::new(Mutex::new(HashMap::new())),
1209 config_desktop_stamps: Arc::new(Mutex::new(HashMap::new())),
1210 local_connections: Arc::new(Mutex::new(HashSet::new())),
1211 operator_app_policy: Default::default(),
1212 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1213 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1214 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1215 #[cfg(test)]
1216 app_open_test_platform: None,
1217 #[cfg(test)]
1218 app_open_test_launch: None,
1219 #[cfg(test)]
1220 recovery_exit_test_after_step: None,
1221 #[cfg(test)]
1222 recovery_exit_test_completed: None,
1223 #[cfg(test)]
1224 apply_reconcile_pause: Arc::new(Mutex::new(None)),
1225 #[cfg(test)]
1226 recovery_exit_test_quiet_retry: false,
1227 #[cfg(test)]
1228 recovery_exit_test_retry_pause: None,
1229 #[cfg(test)]
1230 config_delivery_test_wakes: None,
1231 #[cfg(all(test, unix))]
1232 config_delivery_test_idle: None,
1233 #[cfg(windows)]
1234 windows_verify: None,
1235 #[cfg(windows)]
1236 windows_flash: None,
1237 #[cfg(windows)]
1238 windows_track: None,
1239 #[cfg(all(test, windows))]
1240 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1241 #[cfg(test)]
1242 caller_lookup: None,
1243 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1244 rescan: None,
1245 connected_clients: ConnectedClients::new(),
1246 local_endpoint_unavailable: None,
1247 counters,
1248 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1249 daemon_shutdown: DaemonShutdownFlag::default(),
1250 daemon_provenance: DaemonProvenanceFacts::default(),
1251 #[cfg(test)]
1252 control_dispatch_delay: None,
1253 #[cfg(test)]
1254 provenance_probe_override: None,
1255 #[cfg(test)]
1256 route_bind_relay_sent_hold: None,
1257 #[cfg(test)]
1258 hello_registry_to_forwarding_hold: None,
1259 #[cfg(test)]
1260 hello_ack_to_publication_hold: None,
1261 #[cfg(test)]
1262 hello_before_publication_hold: None,
1263 #[cfg(test)]
1264 hello_after_launch_check_hold: None,
1265 #[cfg(test)]
1266 hello_inside_publication_hold: None,
1267 }
1268 }
1269
1270 pub fn with_storage_config(
1273 mut self,
1274 storage_config: Option<crate::daemon_config::StorageConfig>,
1275 ) -> Self {
1276 self.storage_config = storage_config;
1277 self
1278 }
1279
1280 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1283 self.activity = Some(crate::activity::ActivityLedger::open(
1284 run_dir.join("activity.jsonl"),
1285 incarnation,
1286 ));
1287 self.supervisor.set_activity(self.activity.clone());
1288 self
1289 }
1290
1291 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1294 self.machine_id = machine_id;
1295 self
1296 }
1297
1298 pub fn with_admission_facts_config(
1302 mut self,
1303 carrier_module_id: Option<String>,
1304 targets: Option<Vec<String>>,
1305 ) -> Self {
1306 self.admission_facts_carrier_module_id = carrier_module_id;
1307 self.admission_facts_targets = targets;
1308 self
1309 }
1310
1311 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1315 self.scopes = Arc::new(RwLock::new(
1316 ScopeTable::new(owners.iter().cloned())
1317 .with_federated_owner(self.federated_scope_owner.clone()),
1318 ));
1319 self.scope_authority_owners = owners;
1320 self
1321 }
1322
1323 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1325 self.scopes = Arc::new(RwLock::new(
1326 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1327 .with_federated_owner(owner.clone()),
1328 ));
1329 self.federated_scope_owner = owner;
1330 self
1331 }
1332
1333 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1335 self.privileged_operator_prompt = mode;
1336 self
1337 }
1338
1339 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1342 self.route_bind_relay_timeout = timeout;
1343 self
1344 }
1345
1346 pub fn with_route_bind_relay_timeouts(
1352 mut self,
1353 timeouts: impl IntoIterator<Item = (String, Duration)>,
1354 ) -> Self {
1355 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1356 self
1357 }
1358
1359 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1365 self.route_bind_relay_timeouts
1366 .get(module_id)
1367 .copied()
1368 .unwrap_or(self.route_bind_relay_timeout)
1369 }
1370
1371 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1379 self.route_bind_breaker_threshold = threshold.max(1);
1380 self.route_bind_breaker_cooldown = cooldown;
1381 self
1382 }
1383
1384 #[cfg(test)]
1385 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1386 self.health_probe_timeout = timeout;
1387 self
1388 }
1389
1390 #[cfg(test)]
1391 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1392 self.control_dispatch_delay = Some(delay);
1393 self
1394 }
1395
1396 #[cfg(test)]
1400 pub(crate) fn with_route_bind_relay_sent_hold(
1401 mut self,
1402 hold: Arc<tokio::sync::Barrier>,
1403 ) -> Self {
1404 self.route_bind_relay_sent_hold = Some(hold);
1405 self
1406 }
1407
1408 #[cfg(test)]
1413 pub(crate) fn with_hello_registry_to_forwarding_hold(
1414 mut self,
1415 hold: Arc<std::sync::Barrier>,
1416 ) -> Self {
1417 self.hello_registry_to_forwarding_hold = Some(hold);
1418 self
1419 }
1420
1421 #[cfg(test)]
1422 fn hold_between_registry_and_forwarding(&self) {
1423 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1424 hold.wait();
1425 hold.wait();
1426 }
1427 }
1428
1429 #[cfg(test)]
1432 fn hold_between_ack_and_publication(&self) {
1433 if let Some(hold) = &self.hello_ack_to_publication_hold {
1434 hold.wait();
1435 hold.wait();
1436 }
1437 }
1438
1439 #[cfg(all(test, unix))]
1440 pub(crate) fn with_hello_before_publication_hold(
1441 mut self,
1442 hold: Arc<std::sync::Barrier>,
1443 ) -> Self {
1444 self.hello_before_publication_hold = Some(hold);
1445 self
1446 }
1447
1448 #[cfg(all(test, unix))]
1449 pub(crate) fn with_hello_after_launch_check_hold(
1450 mut self,
1451 hold: Arc<std::sync::Barrier>,
1452 ) -> Self {
1453 self.hello_after_launch_check_hold = Some(hold);
1454 self
1455 }
1456
1457 #[cfg(all(test, unix))]
1458 pub(crate) fn with_hello_inside_publication_hold(
1459 mut self,
1460 hold: Arc<std::sync::Barrier>,
1461 ) -> Self {
1462 self.hello_inside_publication_hold = Some(hold);
1463 self
1464 }
1465
1466 pub fn with_process_liveness(
1467 mut self,
1468 process_liveness: Arc<dyn ModuleProcessLiveness>,
1469 ) -> Self {
1470 self.process_liveness = Some(process_liveness);
1471 self
1472 }
1473
1474 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1475 self.supervisor = supervisor;
1476 if let Some(owner) = &self.config_runtime {
1477 self.supervisor.install_binding_retirement(owner);
1478 }
1479 self.supervisor.set_activity(self.activity.clone());
1480 self
1481 }
1482
1483 pub fn with_daemon_provenance(
1484 mut self,
1485 pid: u32,
1486 started_at_ms: u64,
1487 executable_path: Option<PathBuf>,
1488 build_git_sha: Option<String>,
1489 build_lock_digest: Option<String>,
1490 ) -> Self {
1491 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1492 let process_start_time = process_start_time(pid);
1493 self.daemon_provenance = DaemonProvenanceFacts {
1494 build: DaemonBuildProvenance {
1495 build_git_sha,
1496 build_lock_digest,
1497 },
1498 pid: Some(pid),
1499 started_at_ms: Some(started_at_ms),
1500 start_clock: None,
1501 executable_path,
1502 executable_identity,
1503 process_start_time,
1504 probe: ExecutableIdentityProbe::default(),
1505 };
1506 self
1507 }
1508
1509 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1510 self.daemon_provenance.start_clock = Some(clock);
1511 self
1512 }
1513
1514 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1516 self.scopes
1517 .write()
1518 .unwrap_or_else(|poisoned| poisoned.into_inner())
1519 .set_wall_clock(Arc::new(clock));
1520 self
1521 }
1522
1523 #[cfg(test)]
1524 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1525 self.provenance_probe_override = Some(result);
1526 self
1527 }
1528
1529 pub fn with_capability_config(
1533 self,
1534 modules: impl IntoIterator<Item = (String, bool)>,
1535 reserved_capabilities: BTreeMap<String, String>,
1536 ) -> Self {
1537 self.capability_evaluator
1538 .configure(modules, reserved_capabilities);
1539 self
1540 }
1541
1542 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1546 self.daemon_shutdown = flag;
1547 self
1548 }
1549
1550 pub fn with_supervisor_rescan(
1551 mut self,
1552 supervisor: Supervisor,
1553 config_path: impl Into<PathBuf>,
1554 configured_port: Option<u16>,
1555 ) -> Self {
1556 self.rescan = Some(SupervisorRescanContext {
1557 supervisor,
1558 config_path: config_path.into(),
1559 configured_port,
1560 storage_config: self.storage_config.clone(),
1561 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1562 admission_facts_targets: self.admission_facts_targets.clone(),
1563 scope_authority_owners: self.scope_authority_owners.clone(),
1564 federated_scope_owner: self.federated_scope_owner.clone(),
1565 privileged_operator_prompt: self.privileged_operator_prompt,
1566 generation: Arc::default(),
1567 module_entries: Arc::default(),
1568 });
1569 self
1570 }
1571
1572 pub(crate) fn with_supervisor_rescan_entries(
1573 self,
1574 modules: &[crate::daemon_config::ConfiguredModule],
1575 ) -> Self {
1576 if let Some(context) = &self.rescan {
1577 let mut entries = context
1578 .module_entries
1579 .lock()
1580 .unwrap_or_else(|e| e.into_inner());
1581 entries.seed(modules);
1582 }
1583 self
1584 }
1585
1586 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1587 self.connected_clients = connected_clients;
1588 self
1589 }
1590
1591 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1594 self.local_endpoint_unavailable = reason;
1595 self
1596 }
1597
1598 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1599 Arc::clone(&self.forwarding)
1600 }
1601
1602 pub(crate) fn counters(&self) -> DaemonCounters {
1603 self.counters.clone()
1604 }
1605
1606 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1609 let handler = Arc::downgrade(&self);
1610 let evaluator = self.capability_evaluator.clone();
1611 tokio::spawn(async move {
1612 loop {
1613 evaluator.wait_for_change_or_deadline().await;
1614 let Some(handler) = handler.upgrade() else {
1615 break;
1616 };
1617 handler.refresh_capability_requirements();
1618 }
1619 });
1620 }
1621
1622 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1628 let context = scope_expiry::ScopeExpiryContext::new(&self);
1629 let deadline_added = self.scope_deadline_added.clone();
1630 tokio::spawn(async move {
1631 loop {
1632 if !context.has_deadlines() {
1633 deadline_added.notified().await;
1634 continue;
1635 }
1636 tokio::time::sleep(Duration::from_secs(1)).await;
1637 if let Err(error) = context.sweep() {
1638 warn!(%error, "scope expiry sweep failed");
1639 }
1640 }
1641 });
1642 }
1643
1644 #[cfg(test)]
1645 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1646 scope_expiry::ScopeExpiryContext::new(self).sweep()
1647 }
1648
1649 fn publish_scope_changes(
1650 &self,
1651 changes: &[crate::scopes::ScopeTagChange],
1652 expired: &[crate::scopes::ScopeExpired],
1653 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1654 scope_expiry::ScopeExpiryContext::new(self).publish_scope_changes(changes, expired)
1655 }
1656
1657 fn runtime_capability_snapshot(
1658 &self,
1659 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1660 let runtime = self
1661 .supervisor
1662 .list()
1663 .into_iter()
1664 .map(|module| {
1665 let status = module.status().map_err(|err| {
1666 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1667 })?;
1668 Ok(RuntimeModule {
1669 module_id: status.module_id,
1670 state: status.state,
1671 enabled: status.enabled,
1672 })
1673 })
1674 .collect::<Result<Vec<_>, RouterError>>()?;
1675 let registrations = self.registry.list_registered_modules().map_err(|err| {
1681 RouterError::backend(
1682 0,
1683 0,
1684 format!("failed to list capability registrations: {err}"),
1685 )
1686 })?;
1687 let registrations = registrations
1688 .into_iter()
1689 .map(|registration| RegisteredModule {
1690 module_id: registration.manifest.module_id,
1691 module_version: registration.manifest.module_version,
1692 capabilities: registration.manifest.capabilities,
1693 })
1694 .collect();
1695 Ok((runtime, registrations))
1696 }
1697
1698 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1705 let cached_registration = RegisteredModule {
1706 module_id: registration.manifest.module_id.clone(),
1707 module_version: registration.manifest.module_version.clone(),
1708 capabilities: registration.manifest.capabilities.clone(),
1709 };
1710 let change = self.capability_evaluator.record_hello(&cached_registration);
1711 log_manifest_change(&cached_registration.module_id, &change);
1712 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1713 self.enforce_capability_denies();
1714 }
1715 self.refresh_capability_requirements();
1716 }
1717
1718 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1722 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1723 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1724 self.supervisor.set_swap_promotion_observer(observer);
1725 }
1726
1727 pub fn refresh_capability_requirements(&self) {
1728 match self.runtime_capability_snapshot() {
1729 Ok((runtime, registrations)) => {
1730 log_requirement_events(
1731 self.capability_evaluator
1732 .evaluate_now(&runtime, ®istrations),
1733 &self.daemon_shutdown,
1734 );
1735 }
1736 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1737 }
1738 }
1739
1740 fn enforce_capability_denies(&self) {
1744 let registrations = match self.registry.list_registered_modules() {
1745 Ok(snapshot) => snapshot,
1746 Err(err) => {
1747 warn!(error = %err, "failed to read registrations for capability deny census");
1748 return;
1749 }
1750 };
1751 let manifests = registrations
1752 .into_iter()
1753 .map(|registration| {
1754 (
1755 registration.manifest.module_id.clone(),
1756 registration.manifest,
1757 )
1758 })
1759 .collect::<BTreeMap<_, _>>();
1760 let census = match self.forwarding.route_census(None) {
1761 Ok(census) => census,
1762 Err(err) => {
1763 warn!(error = %err, "failed to read route census for capability deny enforcement");
1764 return;
1765 }
1766 };
1767
1768 for (target_module_id, routes) in census {
1769 let Some(target_manifest) = manifests.get(&target_module_id) else {
1770 continue;
1771 };
1772 let mut closed_routes = Vec::new();
1773 let mut goodbyes = Vec::new();
1774 for route in routes {
1775 let Principal::Reserved {
1776 module_id: opening_module_id,
1777 } = &route.principal
1778 else {
1779 continue;
1780 };
1781 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1782 continue;
1783 };
1784 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1785 continue;
1786 };
1787
1788 match self.forwarding.release_client_route(
1789 route.goodbye_target.connection_id,
1790 route.goodbye_target.channel,
1791 route.goodbye_target.epoch,
1792 ) {
1793 Ok(RouteRelease::Removed(module_goodbye)) => {
1794 warn!(
1795 opening_module_id,
1796 target_module_id,
1797 capability,
1798 "force-closing route because an attested capability deny edge now matches"
1799 );
1800 goodbyes.push(module_goodbye);
1801 goodbyes.push(route.goodbye_target.clone());
1802 closed_routes.push(route);
1803 }
1804 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1805 Err(err) => warn!(
1806 opening_module_id,
1807 target_module_id,
1808 capability,
1809 error = %err,
1810 "failed to force-close capability-denied route"
1811 ),
1812 }
1813 }
1814
1815 if closed_routes.is_empty() {
1816 continue;
1817 }
1818 send_route_control_pushes(
1819 &self.forwarding,
1820 closed_routes,
1821 ClientControlPush::RouteClosed {
1822 module_id: target_module_id,
1823 channels: Vec::new(),
1824 reason: RouteCloseReason::CapabilityDenied,
1825 drained: false,
1826 abandoned: 0,
1827 excluded_subscriptions: 0,
1828 terminal: Some(false),
1829 },
1830 );
1831 self.emit_route_goodbyes(goodbyes);
1834 }
1835 }
1836
1837 fn not_ready_reason(
1843 &self,
1844 registration: &crate::registry::ModuleRegistration,
1845 ) -> Option<NotReadyReason> {
1846 if !registration.ready {
1847 return Some(NotReadyReason {
1848 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1849 capability: None,
1850 });
1851 }
1852 self.first_unprovided_required_capability(registration)
1853 .map(|capability| NotReadyReason {
1854 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1855 capability: Some(capability),
1856 })
1857 }
1858
1859 fn first_unprovided_required_capability(
1877 &self,
1878 registration: &crate::registry::ModuleRegistration,
1879 ) -> Option<String> {
1880 let required = registration
1881 .manifest
1882 .capabilities
1883 .iter()
1884 .flat_map(|declarations| declarations.requires.iter())
1885 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1886 .map(|requirement| requirement.capability.as_str())
1887 .collect::<BTreeSet<_>>();
1888 if required.is_empty() {
1889 return None;
1890 }
1891 let module_id = registration.manifest.module_id.as_str();
1892 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1893 if required
1894 .iter()
1895 .any(|capability| verdict(capability).is_none())
1896 {
1897 self.refresh_capability_requirements();
1898 }
1899 required
1900 .into_iter()
1901 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1902 .map(str::to_string)
1903 }
1904
1905 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1906 self.capability_evaluator
1907 .statuses()
1908 .into_iter()
1909 .map(capability_requirement_status)
1910 .collect()
1911 }
1912
1913 fn deregister_connection(
1920 &self,
1921 connection_id: ConnectionId,
1922 reason: RegistrationEndReason,
1923 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1924 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
1925 spawn.end_primary_registration();
1926 }
1927 self.registry
1928 .deregister_connection_with_reason(connection_id, reason)
1929 }
1930
1931 fn rollback_unpublished_hello(
1936 &self,
1937 connection: ConnectionId,
1938 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1939 self.registry.deregister_connection_with_reason(
1940 connection,
1941 RegistrationEndReason::RegistrationFailed,
1942 )
1943 }
1944
1945 fn request_refusal_close(
1946 &self,
1947 connection: ConnectionId,
1948 sink: Option<&crate::FrameSink>,
1949 reason: CloseReason,
1950 ) {
1951 if let Some(sink) = sink {
1952 sink.terminate_ingress();
1953 }
1954 if self.claims.close_notifications_enabled() {
1955 self.forwarding.request_connection_close(connection, reason);
1956 }
1957 }
1958
1959 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1960 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1961 return None;
1962 }
1963 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1964 parse_client_control_request(&frame.body)
1965 else {
1966 return None;
1967 };
1968 Some(target_module_id(&target).to_string())
1969 }
1970
1971 pub(crate) fn route_open_capacity_refusal(
1972 &self,
1973 ctx: &RouteCtx,
1974 frame: &Frame,
1975 target_module_id: &str,
1976 in_flight: usize,
1977 limit: usize,
1978 ) -> Result<Frame, RouterError> {
1979 let caller = self.route_open_caller_from_frame(ctx, frame);
1982 self.route_open_admission_refusal_frame(
1983 ctx,
1984 &caller,
1985 frame,
1986 target_module_id,
1987 "open_admission_full",
1988 (in_flight, limit),
1989 format!(
1990 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1991 ),
1992 )
1993 }
1994
1995 fn route_open_target_capacity_refusal(
1996 &self,
1997 ctx: &RouteCtx,
1998 caller: &RouteOpenCaller,
1999 frame: &Frame,
2000 target_module_id: &str,
2001 in_flight: usize,
2002 ) -> Result<Frame, RouterError> {
2003 self.route_open_admission_refusal_frame(
2004 ctx,
2005 caller,
2006 frame,
2007 target_module_id,
2008 "target_binds_full",
2009 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
2010 format!(
2011 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2012 ),
2013 )
2014 }
2015
2016 #[allow(clippy::too_many_arguments)]
2024 fn route_open_admission_refusal_frame(
2025 &self,
2026 ctx: &RouteCtx,
2027 caller: &RouteOpenCaller,
2028 frame: &Frame,
2029 target_module_id: &str,
2030 reason: &'static str,
2031 (in_flight, limit): (usize, usize),
2032 message: impl Into<String>,
2033 ) -> Result<Frame, RouterError> {
2034 let code = error_codes::TARGET_UNAVAILABLE;
2035 self.counters.increment_route_open_refused(code);
2036 info!(
2037 target: "control",
2038 code,
2039 reason,
2040 module_id = ?target_module_id,
2041 connection_id = ctx.connection_id.get(),
2042 principal = %caller.principal,
2043 scope = %caller.scope,
2044 harness = %caller.harness,
2045 in_flight,
2046 limit,
2047 "route.open refused"
2048 );
2049 control_error_frame(frame, code, message.into())
2050 }
2051
2052 #[cfg(test)]
2058 pub fn handle_control(
2059 &self,
2060 connection_id: ConnectionId,
2061 frame: Frame,
2062 ) -> Result<Vec<Frame>, RouterError> {
2063 match frame.header.ty {
2064 FrameType::Ping => Ok(vec![pong(&frame)?]),
2065 FrameType::Hello => self.handle_hello(connection_id, None, frame),
2066 FrameType::Goodbye => self.handle_goodbye(connection_id),
2067 ty => Ok(vec![control_error_frame(
2068 &frame,
2069 "unsupported_control_frame",
2070 format!("unsupported channel-0 frame {ty:?}"),
2071 )?]),
2072 }
2073 }
2074
2075 pub async fn handle_control_frame(
2076 &self,
2077 ctx: &RouteCtx,
2078 frame: Frame,
2079 ) -> Result<Vec<Frame>, RouterError> {
2080 self.handle_control_frame_timed(ctx, frame, None).await
2081 }
2082
2083 #[cfg(any(test, feature = "test-support"))]
2084 pub fn with_test_operator_app_policy(
2085 mut self,
2086 policy: crate::operator_apps::TestOperatorAppPolicy,
2087 ) -> Self {
2088 self.operator_app_policy = policy.into();
2089 self
2090 }
2091
2092 #[cfg(all(windows, any(test, feature = "test-support")))]
2096 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
2097 self.windows_verify = Some(verify);
2098 self
2099 }
2100
2101 #[cfg(all(windows, any(test, feature = "test-support")))]
2103 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
2104 self.windows_flash = Some(flash);
2105 self
2106 }
2107
2108 #[cfg(all(windows, any(test, feature = "test-support")))]
2112 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
2113 self.windows_track = Some(track);
2114 self
2115 }
2116
2117 pub(crate) fn record_captured_peer(
2118 &self,
2119 connection: ConnectionId,
2120 peer: subc_os::local_peer::CapturedPeer,
2121 ) {
2122 self.record_observed_peer(connection, peer.observation().clone());
2123 let identity = self.operator_app_policy.config_identity(peer.clone());
2124 #[cfg(windows)]
2125 let identity = identity.or_else(|| self.config_windows_launch_identity(&peer));
2126 if let Some(identity) = identity {
2127 self.config_desktop_stamps
2130 .lock()
2131 .unwrap_or_else(|p| p.into_inner())
2132 .insert(connection, identity);
2133 }
2134 self.captured_peers
2135 .lock()
2136 .unwrap_or_else(|p| p.into_inner())
2137 .insert(connection, peer.clone());
2138 #[cfg(windows)]
2139 if let Some(stamp) = self.stamp_windows_launch(&peer) {
2140 self.operator_apps
2141 .lock()
2142 .unwrap_or_else(|p| p.into_inner())
2143 .insert(connection, stamp);
2144 }
2145 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
2146 self.operator_apps
2147 .lock()
2148 .unwrap_or_else(|p| p.into_inner())
2149 .insert(connection, stamp);
2150 }
2151 }
2152
2153 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
2154 self.local_connections
2155 .lock()
2156 .unwrap_or_else(|p| p.into_inner())
2157 .insert(connection);
2158 }
2159
2160 fn connection_operator_app(
2163 &self,
2164 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
2165 ) -> Option<subc_protocol::peer::OperatorApp> {
2166 stamp
2167 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
2168 .map(|stamp| stamp.app.clone())
2169 }
2170
2171 fn handle_connection_describe(
2172 &self,
2173 ctx: &RouteCtx,
2174 frame: Frame,
2175 ) -> Result<Vec<Frame>, RouterError> {
2176 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
2177 return Ok(vec![control_error_frame(
2178 &frame,
2179 "invalid_control_body",
2180 "malformed connection.describe body",
2181 )?]);
2182 }
2183 let local = self
2184 .local_connections
2185 .lock()
2186 .unwrap_or_else(|p| p.into_inner())
2187 .contains(&ctx.connection_id);
2188 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
2189 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
2190 Ok(vec![control_response_body_frame(
2191 &frame,
2192 &subc_control::ConnectionDescribeReply::new(local, operator_app),
2193 ops::CONNECTION_DESCRIBE,
2194 )?])
2195 }
2196
2197 pub(crate) fn record_observed_peer(
2198 &self,
2199 connection: ConnectionId,
2200 peer: subc_os::local_peer::ObservedPeer,
2201 ) {
2202 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
2203 let peer = match peer {
2204 OsPeer::Unobserved => ObservedPeer::unobserved(),
2205 OsPeer::Observed(peer) => {
2206 let observed = match peer.user {
2207 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
2208 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
2209 };
2210 observed
2211 .with_exe(
2212 peer.exe
2213 .and_then(|path| path.into_os_string().into_string().ok()),
2214 )
2215 .with_signing_id(peer.signing_id)
2216 }
2217 };
2218 self.observed_peers
2219 .lock()
2220 .unwrap_or_else(|p| p.into_inner())
2221 .insert(connection, peer);
2222 }
2223
2224 pub(crate) fn record_caller_endpoints(
2225 &self,
2226 connection: ConnectionId,
2227 endpoints: Option<(SocketAddr, SocketAddr)>,
2228 ) {
2229 let mut callers = self
2230 .caller_endpoints
2231 .lock()
2232 .unwrap_or_else(|p| p.into_inner());
2233 callers.remove(&connection);
2234 if let Some(endpoints) = endpoints {
2235 callers.insert(connection, endpoints);
2236 }
2237 }
2238
2239 #[cfg(test)]
2240 pub(crate) fn with_caller_lookup_for_test(
2241 mut self,
2242 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2243 ) -> Self {
2244 self.caller_lookup = Some(Arc::new(lookup));
2245 self
2246 }
2247
2248 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2249 let endpoints = self
2250 .caller_endpoints
2251 .lock()
2252 .unwrap_or_else(|p| p.into_inner())
2253 .get(&connection)
2254 .copied();
2255 #[cfg(test)]
2256 let lookup = self.caller_lookup.clone();
2257 tokio::task::spawn_blocking(move || {
2258 #[cfg(test)]
2259 if let Some(lookup) = lookup {
2260 return lookup(endpoints);
2261 }
2262 endpoints
2263 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2264 .unwrap_or_default()
2265 })
2266 .await
2267 .unwrap_or_default()
2268 }
2269
2270 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2271 frame.header.channel == 0
2272 && frame.header.ty == FrameType::Request
2273 && self
2274 .forwarding
2275 .module_endpoint_for_connection(ctx.connection_id)
2276 .is_ok_and(|endpoint| endpoint.is_none())
2277 && parse_client_control_request(&frame.body)
2278 .ok()
2279 .is_some_and(|request| privileged_operation(&request).is_some())
2280 }
2281
2282 pub(crate) async fn handle_control_frame_timed(
2283 &self,
2284 ctx: &RouteCtx,
2285 frame: Frame,
2286 dispatch_started_at: Option<StdInstant>,
2287 ) -> Result<Vec<Frame>, RouterError> {
2288 let first = ctx.egress.first_ingress_frame();
2289 self.handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
2290 .await
2291 }
2292
2293 pub(crate) async fn handle_control_frame_ingress(
2294 &self,
2295 ctx: &RouteCtx,
2296 frame: Frame,
2297 dispatch_started_at: Option<StdInstant>,
2298 first: bool,
2299 ) -> Result<Vec<Frame>, RouterError> {
2300 if ctx.egress.ingress_terminal() {
2301 return Ok(Vec::new());
2302 }
2303 #[cfg(test)]
2304 self.claims.record_dispatch(frame.header.corr);
2305 let claim_binding = self.claims.binding(ctx.connection_id);
2306 let mut prefix_attempt = (frame.header.ty == FrameType::Hello
2307 && claim_binding
2308 .as_ref()
2309 .is_some_and(|binding| binding.prefix_module_id().is_some()))
2310 .then(|| self.prefix_hello_attempt(ctx));
2311 if claim_binding
2312 .as_ref()
2313 .is_some_and(|binding| binding.prefix_module_id().is_some())
2314 {
2315 let registered = self
2316 .forwarding
2317 .module_endpoint_for_connection(ctx.connection_id)
2318 .map_err(RouterError::Forwarding)?
2319 .is_some();
2320 if !registered && frame.header.ty != FrameType::Hello {
2321 return self
2322 .refuse_claim_connection(ctx, &frame, "claim_hello_required")
2323 .await;
2324 }
2325 if registered && frame.header.ty == FrameType::Goodbye {
2326 ctx.egress.terminate_ingress();
2327 let result = self.handle_goodbye(ctx.connection_id);
2328 self.claims.connection_closed(ctx.connection_id);
2329 self.request_refusal_close(
2330 ctx.connection_id,
2331 Some(&ctx.egress),
2332 CloseReason::new("claim_registration_ended", "prefix registration ended"),
2333 );
2334 return result;
2335 }
2336 if registered && frame.header.ty == FrameType::Request {
2337 let allowed =
2338 serde_json::from_slice::<ControlOpProbe>(&frame.body).is_ok_and(|probe| {
2339 matches!(
2340 probe.op.as_str(),
2341 "catalog.update"
2342 | "catalog.update_role_declarations"
2343 | "scope.describe"
2344 | "supervisor.live_roots"
2345 | "activity.record"
2346 )
2347 });
2348 if !allowed {
2349 return Ok(vec![control_error_frame(
2350 &frame,
2351 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2352 "prefix registrations have only brokered module operations",
2353 )?]);
2354 }
2355 }
2356 if registered
2357 && frame.header.ty == FrameType::Push
2358 && !serde_json::from_slice::<ControlOpProbe>(&frame.body)
2359 .is_ok_and(|probe| probe.op == "route.status")
2360 {
2361 return Ok(vec![control_error_frame(
2362 &frame,
2363 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2364 "prefix registrations may push only route.status for routes they serve",
2365 )?]);
2366 }
2367 }
2368 match frame.header.ty {
2369 FrameType::Ping => Ok(vec![pong(&frame)?]),
2370 FrameType::Hello => {
2371 if claim_binding
2372 .as_ref()
2373 .is_some_and(|binding| binding.prefix_module_id().is_none())
2374 {
2375 return self
2376 .refuse_claim_connection(ctx, &frame, "claim_hello_not_allowed")
2377 .await;
2378 }
2379 let channel_spawn = self.supervisor.registration_spawn(ctx.connection_id);
2383 if let Some(spawn) = &channel_spawn {
2384 spawn.wait_launch().await;
2385 if !spawn.launch_ready() {
2388 ctx.egress.terminate_ingress();
2389 return Ok(Vec::new());
2390 }
2391 #[cfg(test)]
2392 self.pause_hello_ingress_for_test().await;
2393 } else if claim_binding.is_none() {
2394 if let Ok(hello) = serde_json::from_slice::<serde_json::Value>(&frame.body) {
2395 if let (Some(module), Some(nonce)) = (
2396 hello
2397 .get("manifest")
2398 .and_then(|value| value.get("module_id"))
2399 .and_then(|value| value.as_str()),
2400 hello.get("launch_nonce").and_then(|value| value.as_str()),
2401 ) {
2402 if !self
2403 .config_runtime
2404 .as_ref()
2405 .is_some_and(|owner| owner.module_is_gated(module))
2406 && !self.supervisor.wait_program_binding(module, nonce).await
2407 {
2408 self.request_refusal_close(
2409 ctx.connection_id,
2410 Some(&ctx.egress),
2411 CloseReason::new(
2412 "program_refused",
2413 "the spawned program binding failed",
2414 ),
2415 );
2416 return Ok(vec![control_error_frame(
2417 &frame,
2418 if self
2419 .config_runtime
2420 .as_ref()
2421 .is_some_and(|owner| owner.is_recovering())
2422 {
2423 "config_gated"
2424 } else {
2425 "program_refused"
2426 },
2427 "the spawned program binding failed",
2428 )?]);
2429 }
2430 }
2431 }
2432 }
2433 let handler = self.clone();
2434 let connection = ctx.connection_id;
2435 let egress = ctx.egress.clone();
2436 let correlation = frame.header.corr;
2437 let dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
2438 let span = tracing::Span::current();
2439 let hello_binding = claim_binding.clone();
2440 let replies = tokio::task::spawn_blocking(move || {
2441 tracing::dispatcher::with_default(&dispatcher, || {
2442 span.in_scope(|| {
2443 handler.handle_hello_checked(
2444 connection,
2445 Some(egress),
2446 frame,
2447 true,
2448 hello_binding,
2449 channel_spawn,
2450 )
2451 })
2452 })
2453 })
2454 .await
2455 .map_err(|error| RouterError::backend(0, correlation, error.to_string()))??;
2456 #[cfg(test)]
2457 if prefix_attempt.is_some() {
2458 self.claims.pause_prefix_commit_for_test(false).await;
2459 }
2460 if let Some(attempt) = &mut prefix_attempt {
2461 let accepted = !replies
2462 .iter()
2463 .any(|reply| reply.header.ty == FrameType::Error)
2464 && self
2465 .registry
2466 .get_module_by_connection(ctx.connection_id)
2467 .map_err(|err| RouterError::backend(0, correlation, err.to_string()))?
2468 .is_some();
2469 if accepted {
2470 attempt.accepted = true;
2471 } else {
2472 ctx.egress.terminate_ingress();
2473 for reply in replies {
2474 ctx.egress.send_flushed(reply).await?;
2475 }
2476 return Ok(Vec::new());
2477 }
2478 }
2479 Ok(replies)
2480 }
2481 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2482 FrameType::Cancel => {
2483 if self
2489 .agent_declarations
2490 .cancel(ctx.connection_id, frame.header.corr)
2491 || self
2492 .forwarding
2493 .operator_confirms()
2494 .cancel(ctx.connection_id, frame.header.corr)
2495 || self.cancel_config_approval(ctx.connection_id, frame.header.corr)
2496 || self
2497 .supervisor
2498 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2499 || self.cancel_activity_subscription(ctx, &frame)?
2500 || self.cancel_catalog_subscription(ctx.connection_id, frame.header.corr)
2501 {
2502 Ok(Vec::new())
2503 } else {
2504 Ok(vec![control_error_frame(
2505 &frame,
2506 "unknown_subscription",
2507 "no subscription has this correlation id",
2508 )?])
2509 }
2510 }
2511 FrameType::Request => {
2512 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2515 let claim_present = op.as_ref().is_some_and(|probe| probe.op == "claim.present");
2516 if let Some(refusal) = self
2517 .recovery_request_refusal(
2518 ctx,
2519 &frame,
2520 op.as_ref().map(|probe| probe.op.as_str()),
2521 )
2522 .await?
2523 {
2524 if claim_present {
2525 self.reclaim_presented_expiry(&frame);
2526 return self
2527 .refuse_claim_connection(ctx, &frame, "config_gated")
2528 .await;
2529 }
2530 return Ok(vec![refusal]);
2531 }
2532 if claim_present {
2533 return self.handle_claim_present(ctx, &frame, first).await;
2534 }
2535 if op.as_ref().is_some_and(|probe| probe.op == "claim.mint") {
2536 return self.handle_claim_mint(ctx, &frame).await;
2537 }
2538 if let Some(refusal) = self.registration_operation_refusal(
2539 ctx,
2540 &frame,
2541 op.as_ref().map(|probe| probe.op.as_str()),
2542 )? {
2543 return Ok(vec![refusal]);
2544 }
2545 if op
2546 .as_ref()
2547 .is_some_and(|probe| probe.op == ops::CATALOG_WATCH)
2548 {
2549 return self.handle_catalog_watch(ctx, frame);
2550 }
2551 if op
2552 .as_ref()
2553 .is_some_and(|probe| probe.op == MODULE_TO_SUBC_OP_CATALOG_UPDATE)
2554 && subc_protocol::provider_declarations::catalog_update_has_provider_declarations(&frame.body)
2555 .unwrap_or(false)
2556 {
2557 return Ok(vec![control_error_frame(
2558 &frame,
2559 error_codes::CATALOG_UPDATE_FROZEN_DECLARATIONS,
2560 "provider_declarations are fixed at HELLO; restart the module to change them",
2561 )?]);
2562 }
2563 if let Some(probe) = op
2564 .as_ref()
2565 .filter(|probe| config_gate::is_module_config_op(&probe.op))
2566 {
2567 return self
2568 .handle_module_config_request(ctx, frame, &probe.op)
2569 .await;
2570 }
2571 if op
2572 .as_ref()
2573 .is_some_and(|probe| config_gate::is_config_op(&probe.op))
2574 {
2575 return self.handle_config_gate_request(ctx, frame).await;
2576 }
2577 if op
2578 .as_ref()
2579 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2580 {
2581 return self.handle_activity_record(ctx, frame).await;
2582 }
2583 if op.as_ref().is_some_and(|probe| {
2584 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2585 }) {
2586 return self.handle_activity_read(ctx, frame).await;
2587 }
2588 if op
2589 .as_ref()
2590 .is_some_and(|probe| probe.op == "operator.confirm")
2591 {
2592 return self.handle_operator_confirm(ctx, frame);
2593 }
2594 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2595 return self
2596 .observe_control_wait(
2597 ctx,
2598 None,
2599 ops::APP_OPEN,
2600 self.handle_app_open(ctx, frame),
2601 )
2602 .await;
2603 }
2604 if op
2605 .as_ref()
2606 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2607 {
2608 return self.handle_connection_describe(ctx, frame);
2609 }
2610 if op
2611 .as_ref()
2612 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2613 {
2614 return self.handle_scope_served_by(ctx.connection_id, frame);
2615 }
2616 if self
2617 .forwarding
2618 .module_endpoint_for_connection(ctx.connection_id)
2619 .map_err(RouterError::Forwarding)?
2620 .is_some()
2621 {
2622 if !is_known_module_request_op(&frame.body) {
2623 return Ok(vec![control_error_frame(
2624 &frame,
2625 "unsupported_control_frame",
2626 "module-originated channel-0 REQUEST is not supported",
2627 )?]);
2628 }
2629 let request = match parse_module_control_request_from_module(&frame.body) {
2630 Ok(request) => request,
2631 Err((err, ControlRequestBodyError::UnknownOp)) => {
2632 return Ok(vec![control_error_frame(
2633 &frame,
2634 "unsupported_control_frame",
2635 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2636 )?])
2637 }
2638 Err((err, ControlRequestBodyError::InvalidBody)) => {
2639 return Ok(vec![control_error_frame(
2640 &frame,
2641 "invalid_control_body",
2642 format!("malformed module control body: {err}"),
2643 )?])
2644 }
2645 };
2646 let op = module_control_request_op(&request);
2647 let corr = frame.header.corr;
2648 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2649 if let ModuleControlRequestFromModule::AgentCommands {
2650 started,
2651 finished,
2652 known,
2653 } = request
2654 {
2655 return self
2656 .agent_declarations
2657 .enqueue(
2658 self,
2659 ctx,
2660 frame,
2661 crate::resource_totals::Declaration {
2662 started,
2663 finished,
2664 known,
2665 },
2666 )
2667 .await;
2668 }
2669 let result =
2670 self.handle_module_control_request(ctx.connection_id, frame, request);
2671 log_slow_control_dispatch(
2672 dispatch_started_at,
2673 op,
2674 ctx.connection_id,
2675 corr,
2676 None,
2677 );
2678 return result;
2679 }
2680
2681 if is_known_module_request_op(&frame.body) {
2682 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2683 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2684 let code = if is_agents {
2685 "not_supervised"
2686 } else if op
2687 .as_ref()
2688 .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2689 {
2690 error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2691 } else {
2692 "not_registered"
2693 };
2694 return Ok(vec![control_error_frame(
2695 &frame,
2696 code,
2697 "module-originated control requests require the module's own registered connection",
2698 )?]);
2699 }
2700
2701 let request = match parse_client_control_request(&frame.body) {
2702 Ok(request) => request,
2703 Err((err, ControlRequestBodyError::UnknownOp)) => {
2704 return Ok(vec![control_error_frame(
2705 &frame,
2706 "unknown_control_op",
2707 format!("unknown client control op: {err}"),
2708 )?])
2709 }
2710 Err((err, ControlRequestBodyError::InvalidBody)) => {
2711 return Ok(vec![control_error_frame(
2712 &frame,
2713 "invalid_control_body",
2714 format!("malformed client control body: {err}"),
2715 )?])
2716 }
2717 };
2718 let op = client_control_request_op(&request);
2719 let corr = frame.header.corr;
2720 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2721 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2722 #[cfg(test)]
2723 if let Some(delay) = self.control_dispatch_delay {
2724 tokio::time::sleep(delay).await;
2725 }
2726 let result = self
2727 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2728 .await;
2729 if let Some(timings) = route_open_timings.as_mut() {
2730 timings.finish();
2731 }
2732 log_slow_control_dispatch(
2733 dispatch_started_at,
2734 op,
2735 ctx.connection_id,
2736 corr,
2737 route_open_timings.as_ref(),
2738 );
2739 result
2740 }
2741 FrameType::Push => {
2742 let Some(endpoint) = self
2743 .forwarding
2744 .module_endpoint_for_connection(ctx.connection_id)
2745 .map_err(RouterError::Forwarding)?
2746 else {
2747 return Ok(vec![control_error_frame(
2748 &frame,
2749 "unsupported_control_frame",
2750 "client-originated channel-0 PUSH is not supported",
2751 )?]);
2752 };
2753 self.handle_status_update(endpoint, frame)
2754 }
2755 FrameType::Response | FrameType::Error
2756 if self
2757 .forwarding
2758 .module_endpoint_for_connection(ctx.connection_id)
2759 .map_err(RouterError::Forwarding)?
2760 .is_some() =>
2761 {
2762 self.handle_module_relay_response(ctx.connection_id, frame)
2763 }
2764 ty => Ok(vec![control_error_frame(
2765 &frame,
2766 "unsupported_control_frame",
2767 format!("unsupported channel-0 frame {ty:?}"),
2768 )?]),
2769 }
2770 }
2771
2772 #[cfg(test)]
2773 fn pause_cleanup_for_test(&self, stage: u8) {
2774 let mut slot = self.cleanup_pause.lock().unwrap_or_else(|p| p.into_inner());
2775 if slot.as_ref().is_some_and(|(at, _)| *at == stage) {
2776 let (_, (reached, release)) = slot.take().unwrap();
2777 drop(slot);
2778 let _ = reached.send(());
2779 let _ = release.recv_timeout(Duration::from_secs(5));
2780 }
2781 }
2782
2783 #[cfg(all(test, unix, feature = "test-support"))]
2786 pub(crate) fn after_nonce_attestation_for_test(&self, hook: impl FnOnce() + Send + 'static) {
2787 *self
2788 .after_nonce_attestation
2789 .lock()
2790 .unwrap_or_else(|p| p.into_inner()) = Some(Box::new(hook));
2791 }
2792
2793 #[cfg(test)]
2794 pub(super) fn pause_after_nonce_attestation_for_test(&self) {
2795 let hook = self
2796 .after_nonce_attestation
2797 .lock()
2798 .unwrap_or_else(|p| p.into_inner())
2799 .take();
2800 if let Some(hook) = hook {
2801 hook();
2802 }
2803 }
2804
2805 #[cfg(test)]
2806 async fn pause_hello_ingress_for_test(&self) {
2807 let pause = self
2808 .hello_ingress_pause
2809 .lock()
2810 .unwrap_or_else(|p| p.into_inner())
2811 .take();
2812 if let Some((reached, release)) = pause {
2813 let _ = reached.send(());
2814 let _ = release.await;
2815 }
2816 }
2817
2818 pub fn cleanup_connection(
2819 &self,
2820 connection_id: ConnectionId,
2821 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2822 self.agent_declarations.close(connection_id);
2836 self.close_catalog_subscriptions(connection_id);
2837 #[cfg(test)]
2838 self.pause_cleanup_for_test(0);
2839 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
2840 spawn.retire_registration_channel();
2841 }
2842 self.supervisor
2843 .forget_registration_connection(connection_id);
2844 #[cfg(test)]
2845 self.pause_cleanup_for_test(1);
2846 self.local_connections
2847 .lock()
2848 .unwrap_or_else(|p| p.into_inner())
2849 .remove(&connection_id);
2850 self.caller_endpoints
2851 .lock()
2852 .unwrap_or_else(|p| p.into_inner())
2853 .remove(&connection_id);
2854 self.build_class_warned
2855 .lock()
2856 .unwrap_or_else(|p| p.into_inner())
2857 .remove(&connection_id);
2858 self.observed_peers
2859 .lock()
2860 .unwrap_or_else(|p| p.into_inner())
2861 .remove(&connection_id);
2862 self.captured_peers
2863 .lock()
2864 .unwrap_or_else(|p| p.into_inner())
2865 .remove(&connection_id);
2866 self.config_desktop_stamps
2867 .lock()
2868 .unwrap_or_else(|p| p.into_inner())
2869 .remove(&connection_id);
2870 self.operator_apps
2871 .lock()
2872 .unwrap_or_else(|p| p.into_inner())
2873 .remove(&connection_id);
2874 self.forwarding
2875 .operator_confirms()
2876 .connection_closed(connection_id);
2877 let cleanup = self.cleanup_connection_with_end_reason(
2878 connection_id,
2879 RegistrationEndReason::ConnectionClosed,
2880 );
2881 self.claims.connection_closed(connection_id);
2884 cleanup
2885 }
2886
2887 fn cleanup_connection_with_end_reason(
2888 &self,
2889 connection_id: ConnectionId,
2890 requested_reason: RegistrationEndReason,
2891 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2892 #[cfg(test)]
2893 if let Some(pause) = self
2894 .activity_request_pause
2895 .lock()
2896 .unwrap()
2897 .as_ref()
2898 .filter(|pause| pause.connection == connection_id)
2899 {
2900 pause
2901 .cleanup_started
2902 .store(true, std::sync::atomic::Ordering::Release);
2903 }
2904 if let Some(runtime) = &self.config_runtime {
2905 runtime.disconnected(connection_id);
2906 }
2907 self.agent_declarations.close(connection_id);
2908 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2909 self.registry
2910 .get_module_by_connection(connection_id)?
2911 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2912 .and_then(|module| module.registration_end_reason().ok().flatten())
2913 .unwrap_or(requested_reason)
2914 } else {
2915 requested_reason
2916 };
2917 let crash_closed = self
2918 .registry
2919 .get_module_by_connection(connection_id)?
2920 .and_then(|registration| {
2921 self.forwarding
2922 .module_endpoint_for_connection(connection_id)
2923 .ok()
2924 .flatten()
2925 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2926 .map(|routes| (registration.manifest.module_id, routes))
2927 });
2928 let crash_closed = crash_closed.map(|(module_id, routes)| {
2929 let terminal = match self.supervisor.get(&module_id) {
2930 None => false,
2931 Some(module) => match module.will_recover_after_connection_loss() {
2932 Ok(will_recover) => !will_recover,
2933 Err(err) => {
2934 warn!(
2935 %module_id,
2936 error = %err,
2937 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2938 );
2939 false
2940 }
2941 },
2942 };
2943 let (reason, drained) = match self
2947 .forwarding
2948 .daemon_drain_result_for_connection(connection_id)
2949 {
2950 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2951 Ok(None) => (RouteCloseReason::Crash, false),
2952 Err(err) => {
2953 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2954 (RouteCloseReason::Crash, false)
2955 }
2956 };
2957 (module_id, routes, reason, drained, terminal)
2958 });
2959 let registrations = self.deregister_connection(connection_id, end_reason);
2960 let ended_authority = self
2963 .forwarding
2964 .target_authority_for_connection(connection_id)
2965 .ok()
2966 .flatten();
2967 let cleanup = if crash_closed.is_some() {
2968 self.forwarding.cleanup_connection_counted(connection_id)
2969 } else {
2970 self.forwarding
2976 .cleanup_connection(connection_id)
2977 .map(|released| crate::forwarding::ConnectionCleanup {
2978 released,
2979 abandoned_relays: 0,
2980 })
2981 };
2982 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2987 let abandoned = cleanup
2988 .as_ref()
2989 .map_or(0, |cleanup| cleanup.abandoned_relays);
2990 send_route_control_pushes(
2991 &self.forwarding,
2992 routes,
2993 ClientControlPush::RouteClosed {
2994 module_id,
2995 channels: Vec::new(),
2996 reason,
2997 drained,
2998 abandoned,
2999 excluded_subscriptions: 0,
3000 terminal: Some(terminal),
3001 },
3002 );
3003 }
3004 if let Ok(cleanup) = cleanup {
3005 self.emit_route_goodbyes(cleanup.released);
3006 }
3007 if let Some(record) = ended_authority {
3008 self.change_spawn_authority(&record);
3009 }
3010 if matches!(®istrations, Ok(r) if !r.is_empty()) {
3014 crate::supervise::notify_registration_release();
3015 self.capability_evaluator.wake_deadline_loop();
3016 self.refresh_capability_requirements();
3017 }
3018 self.supervisor.remove_spawn_subscribers(connection_id);
3019 self.activity_streams
3020 .lock()
3021 .unwrap_or_else(|p| p.into_inner())
3022 .retain(|(id, _), task| {
3023 if *id == connection_id {
3024 task.abort();
3025 false
3026 } else {
3027 true
3028 }
3029 });
3030 self.hello_launch_nonces
3033 .lock()
3034 .unwrap_or_else(|poisoned| poisoned.into_inner())
3035 .forget(connection_id);
3036 self.scopes
3037 .write()
3038 .unwrap_or_else(|poisoned| poisoned.into_inner())
3039 .release_connection(connection_id);
3040 registrations
3041 }
3042
3043 pub(crate) fn handle_route_goodbye(
3044 &self,
3045 connection_id: ConnectionId,
3046 route_channel: u16,
3047 route_epoch: u32,
3048 ) -> Result<bool, RouterError> {
3049 debug!(
3050 connection_id = connection_id.get(),
3051 route_channel, route_epoch, "handling route GOODBYE"
3052 );
3053 let RouteRelease::Removed(released_route) = self
3054 .forwarding
3055 .release_client_route(connection_id, route_channel, route_epoch)
3056 .map_err(RouterError::Forwarding)?
3057 else {
3058 return Ok(false);
3059 };
3060 self.emit_route_goodbyes(vec![released_route]);
3061 Ok(true)
3062 }
3063
3064 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
3065 scope_expiry::ScopeExpiryContext::new(self).emit_route_goodbyes(released_routes);
3066 }
3067
3068 fn send_abandoned_route_bind_goodbye(
3080 &self,
3081 module_sink: &crate::FrameSink,
3082 negotiated_ver: u8,
3083 module_channel: u16,
3084 module_epoch: u32,
3085 ) {
3086 let frame = match Frame::build_with_version(
3087 negotiated_ver,
3088 FrameType::Goodbye,
3089 control_flags(),
3090 module_channel,
3091 module_epoch,
3092 0,
3093 Vec::new(),
3094 ) {
3095 Ok(frame) => frame,
3096 Err(err) => {
3097 warn!(
3098 route_channel = module_channel,
3099 error = %err,
3100 "failed to build GOODBYE for abandoned route.bind"
3101 );
3102 return;
3103 }
3104 };
3105 crate::forwarding::send_module_route_goodbye(
3106 &self.counters,
3107 module_sink,
3108 frame,
3109 None,
3110 "abandoned route.bind",
3111 );
3112 }
3113
3114 #[cfg(test)]
3115 fn handle_hello(
3116 &self,
3117 connection_id: ConnectionId,
3118 egress: Option<crate::router::FrameSink>,
3119 frame: Frame,
3120 ) -> Result<Vec<Frame>, RouterError> {
3121 let binding = self.claims.binding(connection_id);
3122 let channel_spawn = self.supervisor.registration_spawn(connection_id);
3123 self.handle_hello_checked(connection_id, egress, frame, false, binding, channel_spawn)
3124 }
3125
3126 fn handle_hello_checked(
3127 &self,
3128 connection_id: ConnectionId,
3129 sink: Option<crate::FrameSink>,
3130 frame: Frame,
3131 wait_for_gate: bool,
3132 prefix_binding: Option<claims::ClaimBinding>,
3133 channel_spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3134 ) -> Result<Vec<Frame>, RouterError> {
3135 let primary_sink = channel_spawn.as_ref().and_then(|_| sink.clone());
3136 #[cfg(test)]
3137 if prefix_binding.is_some() {
3138 self.claims.pause_prefix_hello_for_test(5);
3139 match self.claims.take_prefix_hello_failure_for_test() {
3140 Some("error") => {
3141 return Err(RouterError::backend(
3142 0,
3143 frame.header.corr,
3144 "injected HELLO failure",
3145 ))
3146 }
3147 Some("panic") => panic!("injected HELLO task panic"),
3148 _ => {}
3149 }
3150 }
3151 if let Some(binding) = &prefix_binding {
3152 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3153 let module = value
3154 .as_ref()
3155 .and_then(|value| value["manifest"]["module_id"].as_str());
3156 if !binding.admit_prefix_hello(module)
3157 || !self.prefix_binding_live(binding, connection_id)
3158 || !module.is_some_and(|module| {
3159 self.supervisor
3160 .prefix_claim_authorized(&binding.registration.module_id, module)
3161 })
3162 || binding.registration.check().is_err()
3163 {
3164 return Ok(vec![control_error_frame(
3165 &frame,
3166 "claim_prefix_refused",
3167 "HELLO must name the prefix claim's exact module id once",
3168 )?]);
3169 }
3170 }
3171 struct RefuseUnregisteredChannel {
3172 spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3173 sink: Option<crate::FrameSink>,
3174 }
3175 impl Drop for RefuseUnregisteredChannel {
3176 fn drop(&mut self) {
3177 if let Some(spawn) = &self.spawn {
3178 if !spawn.registered() {
3179 if let Some(sink) = &self.sink {
3180 sink.terminate_ingress();
3181 }
3182 spawn.retire_registration_channel();
3183 }
3184 }
3185 }
3186 }
3187 let _channel_attempt = RefuseUnregisteredChannel {
3188 spawn: channel_spawn.clone(),
3189 sink: sink.clone(),
3190 };
3191 if let Some(spawn) = &channel_spawn {
3192 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3193 if let Err(code) = spawn.admit_hello(
3194 value
3195 .as_ref()
3196 .and_then(|value| value["manifest"]["module_id"].as_str()),
3197 ) {
3198 return Ok(vec![control_error_frame(
3199 &frame,
3200 code,
3201 "registration channel HELLO refused",
3202 )?]);
3203 }
3204 }
3205 debug!(
3206 connection_id = connection_id.get(),
3207 corr = frame.header.corr,
3208 "handling HELLO"
3209 );
3210 if self
3214 .registry
3215 .get_module_by_connection(connection_id)
3216 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3217 .is_some()
3218 {
3219 return Ok(vec![control_error_frame(
3220 &frame,
3221 "invalid_hello",
3222 "connection is already registered as a module",
3223 )?]);
3224 }
3225 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
3226 Ok(value) => value,
3227 Err(err) => {
3228 return Ok(vec![control_error_frame(
3229 &frame,
3230 "invalid_hello",
3231 format!("malformed HELLO body: {err}"),
3232 )?])
3233 }
3234 };
3235 if let Err(detail) = catalog::validate_hello_declarations(&hello_value) {
3236 return Ok(vec![control_error_frame(
3237 &frame,
3238 error_codes::INVALID_PROVIDER_DECLARATIONS,
3239 detail,
3240 )?]);
3241 }
3242 if let Some(module) = hello_value
3243 .get("manifest")
3244 .and_then(|value| value.get("module_id"))
3245 .and_then(|value| value.as_str())
3246 {
3247 let recovering = self
3248 .config_runtime
3249 .as_ref()
3250 .is_some_and(|owner| owner.check_recovery().unwrap_or(true));
3251 if recovering
3255 && channel_spawn.as_ref().is_none_or(|spawn| {
3256 !self.supervisor.has_live_verified_channel_binding(spawn)
3257 || spawn.check().is_err()
3258 })
3259 && !(channel_spawn.is_none()
3260 && hello_value
3261 .get("launch_nonce")
3262 .and_then(|value| value.as_str())
3263 .is_some_and(|nonce| {
3264 self.supervisor.spawned_consumer_authorized(module, nonce)
3265 && self
3266 .supervisor
3267 .has_live_verified_program_binding(module, nonce)
3268 }))
3269 {
3270 return Ok(vec![control_error_frame(
3271 &frame,
3272 "config_gated",
3273 "recovery admits only previously bound supervised launches",
3274 )?]);
3275 }
3276 if self
3277 .config_runtime
3278 .as_ref()
3279 .is_some_and(|owner| owner.module_is_gated(module))
3280 && (channel_spawn
3281 .as_ref()
3282 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3283 || self.config_runtime.as_ref().is_some_and(|owner| {
3284 !recovering && owner.check_module_hold(module).is_err()
3285 }))
3286 {
3287 return Ok(vec![control_error_frame(
3288 &frame,
3289 "reserved_module",
3290 "a gated identity belongs only to its supervised child",
3291 )?]);
3292 }
3293 }
3294 if channel_spawn.is_none() {
3295 if let (Some(module), Some(nonce)) = (
3296 hello_value
3297 .get("manifest")
3298 .and_then(|value| value.get("module_id"))
3299 .and_then(|value| value.as_str()),
3300 hello_value
3301 .get("launch_nonce")
3302 .and_then(|value| value.as_str()),
3303 ) {
3304 let binding = self.supervisor.has_program_binding_record(module, nonce);
3305 let required = self
3306 .config_runtime
3307 .as_ref()
3308 .is_some_and(|owner| owner.module_is_gated(module));
3309 if let Some(error) = (binding || required)
3310 .then(|| {
3311 self.supervisor
3312 .check_current_program_binding(module, nonce)
3313 .err()
3314 })
3315 .flatten()
3316 {
3317 let code = if error.code == "program_refused" {
3318 "program_refused"
3319 } else {
3320 "config_delivery_deferred"
3321 };
3322 self.request_refusal_close(
3323 connection_id,
3324 sink.as_ref(),
3325 CloseReason::new(code, error.detail.clone()),
3326 );
3327 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3328 }
3329 }
3330 }
3331 if let Err(err) = validate_hello_display_name(&hello_value) {
3332 return Ok(vec![control_error_frame(
3333 &frame,
3334 "invalid_manifest",
3335 err.to_string(),
3336 )?]);
3337 }
3338 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
3339 return Ok(vec![control_error_frame(
3340 &frame,
3341 "invalid_capability_grammar",
3342 err.to_string(),
3343 )?]);
3344 }
3345 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
3346 return Ok(vec![control_error_frame(
3347 &frame,
3348 "invalid_manifest",
3349 err.to_string(),
3350 )?]);
3351 }
3352 if let Err(err) = validate_hello_event_declarations(&hello_value) {
3353 return Ok(vec![control_error_frame(
3354 &frame,
3355 "invalid_event_declaration",
3356 err.to_string(),
3357 )?]);
3358 }
3359 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
3360 return Ok(vec![control_error_frame(
3361 &frame,
3362 "invalid_manifest",
3363 err.to_string(),
3364 )?]);
3365 }
3366 if let Some(provenance) = hello_value
3367 .get("manifest")
3368 .and_then(|manifest| manifest.get("provenance"))
3369 {
3370 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
3371 return Ok(vec![control_error_frame(
3372 &frame,
3373 "invalid_manifest",
3374 format!("malformed manifest provenance: {err}"),
3375 )?]);
3376 }
3377 }
3378 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
3379 let module_id = hello_value["manifest"]["module_id"]
3380 .as_str()
3381 .unwrap_or("<unknown>");
3382 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
3383 }
3384 if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
3385 let module_id = hello_value
3386 .get("manifest")
3387 .and_then(|m| m.get("module_id"))
3388 .and_then(serde_json::Value::as_str)
3389 .unwrap_or("unknown");
3390 warn!(module_id, %reason, "dropping operation_deferral from HELLO");
3391 }
3392 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
3393 let module_id = hello_value["manifest"]["module_id"]
3394 .as_str()
3395 .unwrap_or("<unknown>");
3396 warn!(module_id, %reason, "dropping flow_operations from HELLO");
3397 }
3398 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
3399 let module_id = hello_value["manifest"]["module_id"]
3400 .as_str()
3401 .unwrap_or("<unknown>");
3402 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
3403 }
3404 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
3405 Ok(hello) => hello,
3406 Err(err) => {
3407 return Ok(vec![control_error_frame(
3408 &frame,
3409 "invalid_hello",
3410 format!("malformed HELLO body: {err}"),
3411 )?])
3412 }
3413 };
3414
3415 if hello.protocol_ver != hello.manifest.protocol_ver {
3416 return Ok(vec![control_error_frame(
3417 &frame,
3418 "invalid_manifest",
3419 format!(
3420 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
3421 hello.protocol_ver, hello.manifest.protocol_ver
3422 ),
3423 )?]);
3424 }
3425
3426 if hello.manifest.module_id.trim().is_empty() {
3427 return Ok(vec![control_error_frame(
3428 &frame,
3429 "invalid_manifest",
3430 "manifest module_id must not be empty",
3431 )?]);
3432 }
3433
3434 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
3435 Ok(negotiated_ver) => negotiated_ver,
3436 Err(message) => {
3437 return Ok(vec![control_error_frame(
3438 &frame,
3439 "version_unsupported",
3440 message,
3441 )?])
3442 }
3443 };
3444
3445 let _publication_gate = if let Some(owner) = &self.config_runtime {
3451 let gate = owner
3452 .module_gate(&hello.manifest.module_id)
3453 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3454 Some(if wait_for_gate {
3455 gate.blocking_lock_owned()
3456 } else {
3457 match gate.try_lock_owned() {
3458 Ok(guard) => guard,
3459 Err(_) => {
3460 return Ok(vec![control_error_frame(
3461 &frame,
3462 "config_delivery_deferred",
3463 "module admission is busy",
3464 )?])
3465 }
3466 }
3467 })
3468 } else {
3469 None
3470 };
3471 #[cfg(test)]
3472 if let Some(hold) = &self.hello_before_publication_hold {
3473 hold.wait();
3474 hold.wait();
3475 }
3476 let module = &hello.manifest.module_id;
3477 let required = self
3478 .config_runtime
3479 .as_ref()
3480 .is_some_and(|owner| owner.module_is_gated(module));
3481 let recovering = self
3482 .config_runtime
3483 .as_ref()
3484 .is_some_and(|owner| owner.is_recovering());
3485 if recovering
3493 && channel_spawn.as_ref().is_none_or(|spawn| {
3494 !self.supervisor.has_live_verified_channel_binding(spawn) || spawn.check().is_err()
3495 })
3496 && !(channel_spawn.is_none()
3497 && hello.launch_nonce.as_deref().is_some_and(|nonce| {
3498 self.supervisor.spawned_consumer_authorized(module, nonce)
3499 && self
3500 .supervisor
3501 .has_live_verified_program_binding(module, nonce)
3502 }))
3503 {
3504 return Ok(vec![control_error_frame(
3505 &frame,
3506 "config_gated",
3507 "recovery admits only previously bound supervised launches",
3508 )?]);
3509 }
3510 if required
3511 && (channel_spawn
3512 .as_ref()
3513 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3514 || self
3515 .config_runtime
3516 .as_ref()
3517 .is_some_and(|owner| !recovering && owner.check_module_hold(module).is_err()))
3518 {
3519 return Ok(vec![control_error_frame(
3520 &frame,
3521 "reserved_module",
3522 "gated admission requires the current supervised child's registration channel and verified image binding",
3523 )?]);
3524 }
3525 if let Some(spawn) = &channel_spawn {
3526 if let Err(code) = spawn.check() {
3527 return Ok(vec![control_error_frame(
3528 &frame,
3529 code,
3530 "registration launch is no longer live",
3531 )?]);
3532 }
3533 } else if let Some(nonce) = hello.launch_nonce.as_deref() {
3534 if required || self.supervisor.has_program_binding_record(module, nonce) {
3535 if let Err(error) = self.supervisor.check_current_program_binding(module, nonce) {
3536 let code = if error.code == "program_refused" {
3537 "program_refused"
3538 } else {
3539 "config_delivery_deferred"
3540 };
3541 self.request_refusal_close(
3542 connection_id,
3543 sink.as_ref(),
3544 CloseReason::new(code, error.detail.clone()),
3545 );
3546 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3547 }
3548 }
3549 }
3550
3551 let swap_admission = match &channel_spawn {
3559 Some(spawn) if spawn.is_candidate() => SwapHelloAdmission::Candidate,
3560 Some(_) => SwapHelloAdmission::NotSwapping,
3561 None if prefix_binding.is_some() => SwapHelloAdmission::NotSwapping,
3562 None => self
3563 .supervisor
3564 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref()),
3565 };
3566 if swap_admission == SwapHelloAdmission::ProtocolNone {
3567 warn!(
3568 module_id = %hello.manifest.module_id,
3569 connection_id = connection_id.get(),
3570 "HELLO refused: the module_id belongs to a protocol:none module, which never registers"
3571 );
3572 return Ok(vec![control_error_frame(
3575 &frame,
3576 "swap_token_invalid",
3577 format!(
3578 "module_id '{}' is a protocol:none module; a protocol:none module may not register",
3579 hello.manifest.module_id
3580 ),
3581 )?]);
3582 }
3583 if swap_admission == SwapHelloAdmission::ChannelRequired {
3584 warn!(
3585 module_id = %hello.manifest.module_id,
3586 connection_id = connection_id.get(),
3587 "HELLO refused: the module_id is gated and the HELLO did not arrive over its registration channel"
3588 );
3589 return Ok(vec![control_error_frame(
3592 &frame,
3593 "swap_token_invalid",
3594 format!(
3595 "module_id '{}' is gated; it must register over its supervised registration channel",
3596 hello.manifest.module_id
3597 ),
3598 )?]);
3599 }
3600 if swap_admission == SwapHelloAdmission::Refused {
3601 warn!(
3602 module_id = %hello.manifest.module_id,
3603 connection_id = connection_id.get(),
3604 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
3605 );
3606 return Ok(vec![control_error_frame(
3607 &frame,
3608 "swap_token_invalid",
3609 format!(
3610 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
3611 hello.manifest.module_id
3612 ),
3613 )?]);
3614 }
3615 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
3616
3617 let verification = if swap_candidate && channel_spawn.is_none() {
3622 ReservedHelloVerification {
3623 nonce_verified: true,
3624 prefix_owner: None,
3625 }
3626 } else {
3627 let verification = if let Some(binding) = &prefix_binding {
3628 if !self.supervisor.prefix_claim_authorized(
3629 &binding.registration.module_id,
3630 &hello.manifest.module_id,
3631 ) {
3632 return Ok(vec![control_error_frame(
3633 &frame,
3634 "claim_prefix_refused",
3635 "prefix ownership changed",
3636 )?]);
3637 }
3638 Ok(ReservedHelloVerification {
3639 nonce_verified: true,
3640 prefix_owner: Some(binding.registration.module_id.clone()),
3641 })
3642 } else if channel_spawn.is_some() {
3643 self.supervisor
3644 .channel_hello_verification(&hello.manifest.module_id)
3645 } else {
3646 self.supervisor.reserved_hello_verification(
3647 &hello.manifest.module_id,
3648 hello.launch_nonce.as_deref(),
3649 )
3650 };
3651 match verification {
3652 Ok(verified) => verified,
3653 Err(rejection) => {
3654 let message = match rejection {
3655 ReservedHelloRejection::Exact { module_id } => format!(
3656 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
3657 ),
3658 ReservedHelloRejection::Prefix {
3659 prefix,
3660 owner_module_id,
3661 } => format!(
3662 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
3663 hello.manifest.module_id
3664 ),
3665 };
3666 return Ok(vec![control_error_frame(
3667 &frame,
3668 "reserved_module",
3669 message,
3670 )?]);
3671 }
3672 }
3673 };
3674
3675 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
3676 &hello.manifest.module_id,
3677 hello.manifest.capabilities.as_ref(),
3678 );
3679 if let Some(refusal) = reserved_capability_refusals.first() {
3680 let capability = refusal.capability.clone();
3681 let bound_module = refusal.claimants[0].clone();
3682 log_duplicate_claim_events(reserved_capability_refusals);
3683 return Ok(vec![control_error_frame(
3684 &frame,
3685 "reserved_capability",
3686 format!(
3687 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3688 capability, bound_module, hello.manifest.module_id
3689 ),
3690 )?]);
3691 }
3692
3693 if self
3696 .forwarding
3697 .connection_has_client_routes(connection_id)
3698 .map_err(RouterError::Forwarding)?
3699 {
3700 return Ok(vec![control_error_frame(
3701 &frame,
3702 "invalid_hello",
3703 "connection has open client routes and cannot also register as a module",
3704 )?]);
3705 }
3706
3707 self.hello_launch_nonces
3715 .lock()
3716 .unwrap_or_else(|poisoned| poisoned.into_inner())
3717 .record(
3718 connection_id,
3719 if prefix_binding.is_some() || channel_spawn.is_some() {
3720 None
3721 } else {
3722 hello.launch_nonce.as_deref()
3723 },
3724 );
3725 let control_ops = effective_module_control_ops(hello.control_ops);
3726 let prefix_submodule = verification.prefix_owner.is_some();
3727 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
3730 #[cfg(test)]
3731 struct PublicationFinished(Arc<std::sync::Barrier>);
3732 #[cfg(test)]
3733 impl Drop for PublicationFinished {
3734 fn drop(&mut self) {
3735 self.0.wait();
3736 }
3737 }
3738 #[cfg(test)]
3739 let _publication_finished = self.hello_after_launch_check_hold.as_ref().map(|hold| {
3740 hold.wait();
3741 hold.wait();
3742 PublicationFinished(hold.clone())
3743 });
3744 let publication_spawn = channel_spawn
3745 .as_ref()
3746 .or_else(|| prefix_binding.as_ref().map(|binding| &binding.registration));
3747 #[cfg(test)]
3748 if prefix_binding.is_some() {
3749 self.claims.pause_prefix_hello_for_test(7);
3750 }
3751 let _registration_publication = match publication_spawn {
3752 Some(spawn) => Some(match spawn.begin_publication() {
3753 Ok(guard) => guard,
3754 Err(code) => {
3755 return Ok(vec![control_error_frame(
3756 &frame,
3757 code,
3758 "registration ended before HELLO publication",
3759 )?])
3760 }
3761 }),
3762 None => None,
3763 };
3764 let (_prefix_live, _prefix_ownership) = if let Some(binding) = &prefix_binding {
3765 let live = self.claims.lock_prefix_binding(binding, connection_id);
3766 let ownership = self.supervisor.prefix_claim_publication(
3767 &binding.registration.module_id,
3768 &hello.manifest.module_id,
3769 );
3770 if live.is_none()
3771 || ownership.is_none()
3772 || !self.claim_primary_registered(binding.spawn, &binding.registration)
3773 || binding.prefix_module_id() != Some(hello.manifest.module_id.as_str())
3774 {
3775 return Ok(vec![control_error_frame(
3776 &frame,
3777 "claim_prefix_refused",
3778 "prefix authority ended before publication",
3779 )?]);
3780 }
3781 (live, ownership)
3782 } else {
3783 (None, None)
3784 };
3785 if let Some(spawn) = publication_spawn {
3786 self.forwarding
3787 .bind_spawn(connection_id, spawn)
3788 .map_err(RouterError::Forwarding)?;
3789 }
3790 let hello_authority = match (&channel_spawn, &prefix_binding) {
3800 (Some(spawn), _) => Some(spawn.authority.clone()),
3801 (None, None) => hello.launch_nonce.as_deref().and_then(|nonce| {
3802 self.supervisor
3803 .spawned_consumer_record(&hello.manifest.module_id, nonce)
3804 }),
3805 (None, Some(_)) => None,
3806 }
3807 .map(|record| HelloAuthority {
3808 record,
3809 requested: hello
3810 .manifest
3811 .provider_declarations
3812 .clone()
3813 .unwrap_or_default(),
3814 froze: Default::default(),
3815 });
3816 #[cfg(test)]
3817 if prefix_binding.is_some() {
3818 self.claims.pause_prefix_hello_for_test(8);
3819 }
3820 if swap_candidate {
3821 let module_id = hello.manifest.module_id.clone();
3822 let mut differing_declarations = Default::default();
3823 let reply = self.register_swap_candidate(
3824 connection_id,
3825 sink,
3826 &frame,
3827 hello.manifest,
3828 negotiated_ver,
3829 control_ops,
3830 hello_ack,
3831 hello_authority.as_ref(),
3832 &mut differing_declarations,
3833 #[cfg(windows)]
3834 hello.launch_nonce.as_deref(),
3835 )?;
3836 if let Some(spawn) = &channel_spawn {
3837 if self
3838 .registry
3839 .get_module_by_connection(connection_id)
3840 .ok()
3841 .flatten()
3842 .is_some()
3843 {
3844 if let Some(sink) = primary_sink {
3845 spawn.bind_primary_sink(sink);
3846 }
3847 spawn.hello_accepted();
3848 }
3849 } else if self
3850 .registry
3851 .get_module_by_connection(connection_id)
3852 .ok()
3853 .flatten()
3854 .is_some()
3855 {
3856 self.supervisor
3857 .retire_unused_registration_channel(&module_id, true);
3858 }
3859 if self
3860 .registry
3861 .get_module_by_connection(connection_id)
3862 .ok()
3863 .flatten()
3864 .is_some()
3865 {
3866 self.file_development_widening(
3867 hello_authority.map(|authority| authority.record),
3868 differing_declarations,
3869 );
3870 }
3871 return Ok(reply);
3872 }
3873 let registration = match self
3880 .registry
3881 .register_unpublished_verified_with_control_ops(
3882 hello.manifest,
3883 negotiated_ver,
3884 connection_id,
3885 control_ops,
3886 verification.nonce_verified,
3887 ) {
3888 Ok(registration) => registration,
3889 Err(RegistryError::DuplicateModuleId { module_id }) => {
3890 return Ok(vec![control_error_frame(
3891 &frame,
3892 "duplicate_module_id",
3893 format!(
3894 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
3895 ),
3896 )?])
3897 }
3898 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3899 return Ok(vec![control_error_frame(
3900 &frame,
3901 "invalid_module_id",
3902 err.to_string(),
3903 )?])
3904 }
3905 Err(err) => {
3906 return Ok(vec![control_error_frame(
3907 &frame,
3908 "registry_error",
3909 err.to_string(),
3910 )?])
3911 }
3912 };
3913 let differing_declarations =
3914 match self.admit_hello_authority(connection_id, hello_authority.as_ref()) {
3915 Ok(differing) => differing,
3916 Err(err) => {
3917 if matches!(
3918 self.rollback_unpublished_hello(connection_id),
3919 Ok(r) if !r.is_empty()
3920 ) {
3921 crate::supervise::notify_registration_release();
3922 }
3923 return Err(RouterError::Forwarding(err));
3924 }
3925 };
3926 #[cfg(test)]
3927 self.hold_between_registry_and_forwarding();
3928
3929 let mut reply = Vec::new();
3930 let publication = self
3933 .registry
3934 .publish_registration_after(connection_id, || {
3935 if let Some(sink) = sink {
3936 let concurrency = manifest_concurrency(®istration.manifest);
3952 self.forwarding.register_module_connection_acked(
3953 connection_id,
3954 registration.manifest.module_id.clone(),
3955 negotiated_ver,
3956 concurrency,
3957 sink,
3958 crate::forwarding::ModuleHelloAck {
3959 frame: hello_ack,
3960 verified_prefix_owner: verification.prefix_owner,
3961 },
3962 )?;
3963 } else {
3964 reply.push(hello_ack);
3967 }
3968 #[cfg(test)]
3969 self.hold_between_ack_and_publication();
3970 Ok(())
3971 })
3972 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3973 let published = match publication {
3974 Ok(published) => published,
3975 Err(err) => {
3976 if matches!(
3983 self.rollback_unpublished_hello(connection_id),
3984 Ok(r) if !r.is_empty()
3985 ) {
3986 crate::supervise::notify_registration_release();
3987 }
3988 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
3989 return Ok(vec![control_error_frame(
3990 &frame,
3991 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
3992 "invalid_hello"
3993 } else {
3994 forwarding_error_code(&err)
3995 },
3996 err.to_string(),
3997 )?]);
3998 }
3999 };
4000 if !published {
4001 debug!(
4002 module_id = %registration.manifest.module_id,
4003 connection_id = connection_id.get(),
4004 "registration was removed before it could be published"
4005 );
4006 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
4007 return Ok(reply);
4008 }
4009
4010 drop(_prefix_ownership);
4011 drop(_prefix_live);
4012
4013 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
4022 info!(
4023 module_id = %registration.manifest.module_id,
4024 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
4025 );
4026 }
4027 if registration.manifest.registration_channel == Some(false)
4028 && self
4029 .supervisor
4030 .module_is_gated(®istration.manifest.module_id)
4031 && self
4032 .config_runtime
4033 .as_ref()
4034 .and_then(|owner| owner.registration_declaration(®istration.manifest.module_id))
4035 .as_deref()
4036 == Some("declared")
4037 {
4038 warn!(module_id = %registration.manifest.module_id,
4039 "HELLO registration_channel is false despite the approved channel declaration; registration remains admitted");
4040 }
4041
4042 self.apply_registration_capabilities(®istration);
4043
4044 self.supervisor
4045 .record_activity_registration(®istration.manifest.module_id);
4046
4047 info!(
4048 module_id = %registration.manifest.module_id,
4049 module_version = %registration.manifest.module_version,
4050 negotiated_ver,
4051 routable_provider = manifest_provides_routable_role(®istration.manifest),
4052 connection_id = connection_id.get(),
4053 "module registered"
4054 );
4055
4056 if let Some(spawn) = &channel_spawn {
4057 if let Some(sink) = primary_sink {
4058 spawn.bind_primary_sink(sink);
4059 }
4060 spawn.hello_accepted();
4061 } else if !prefix_submodule {
4062 self.supervisor
4063 .retire_unused_registration_channel(®istration.manifest.module_id, false);
4064 }
4065 self.file_development_widening(
4066 hello_authority.map(|authority| authority.record),
4067 differing_declarations,
4068 );
4069 #[cfg(windows)]
4070 if let Some(nonce) = hello.launch_nonce.as_deref() {
4071 self.supervisor
4072 .program_hello_accepted(®istration.manifest.module_id, nonce);
4073 }
4074
4075 #[cfg(test)]
4076 if let Some(hold) = &self.hello_inside_publication_hold {
4077 hold.wait();
4078 hold.wait();
4079 }
4080
4081 Ok(reply)
4082 }
4083
4084 #[allow(clippy::too_many_arguments)]
4101 fn register_swap_candidate(
4102 &self,
4103 connection_id: ConnectionId,
4104 sink: Option<crate::FrameSink>,
4105 frame: &Frame,
4106 manifest: ModuleManifest,
4107 negotiated_ver: u8,
4108 control_ops: Vec<String>,
4109 hello_ack: Frame,
4110 hello_authority: Option<&HelloAuthority>,
4111 differing_declarations: &mut ProviderDeclarations,
4112 #[cfg(windows)] launch_nonce: Option<&str>,
4113 ) -> Result<Vec<Frame>, RouterError> {
4114 let module_id = manifest.module_id.clone();
4115 self.supervisor
4118 .with_swap_candidate_admission(&module_id, || {
4119 let reply = self.publish_swap_candidate(
4120 connection_id,
4121 sink,
4122 frame,
4123 manifest,
4124 negotiated_ver,
4125 control_ops,
4126 hello_ack,
4127 hello_authority,
4128 differing_declarations,
4129 #[cfg(windows)]
4130 launch_nonce,
4131 );
4132 let admitted = self
4133 .registry
4134 .get_candidate(&module_id)
4135 .ok()
4136 .flatten()
4137 .is_some_and(|candidate| candidate.connection_id == connection_id);
4138 (reply, admitted)
4139 })
4140 .unwrap_or_else(|| {
4141 Ok(vec![control_error_frame(
4142 frame,
4143 "swap_token_invalid",
4144 "swap ended before candidate HELLO publication",
4145 )?])
4146 })
4147 }
4148
4149 #[allow(clippy::too_many_arguments)]
4150 fn publish_swap_candidate(
4151 &self,
4152 connection_id: ConnectionId,
4153 sink: Option<crate::FrameSink>,
4154 frame: &Frame,
4155 manifest: ModuleManifest,
4156 negotiated_ver: u8,
4157 control_ops: Vec<String>,
4158 hello_ack: Frame,
4159 hello_authority: Option<&HelloAuthority>,
4160 differing_declarations: &mut ProviderDeclarations,
4161 #[cfg(windows)] launch_nonce: Option<&str>,
4162 ) -> Result<Vec<Frame>, RouterError> {
4163 let module_id = manifest.module_id.clone();
4164 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
4165 manifest,
4166 negotiated_ver,
4167 connection_id,
4168 control_ops,
4169 ) {
4170 Ok(registration) => registration,
4171 Err(RegistryError::DuplicateModuleId { module_id }) => {
4172 return Ok(vec![control_error_frame(
4173 frame,
4174 "duplicate_module_id",
4175 format!(
4176 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
4177 ),
4178 )?])
4179 }
4180 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
4181 return Ok(vec![control_error_frame(
4182 frame,
4183 "invalid_module_id",
4184 err.to_string(),
4185 )?])
4186 }
4187 Err(err) => {
4188 return Ok(vec![control_error_frame(
4189 frame,
4190 "registry_error",
4191 err.to_string(),
4192 )?])
4193 }
4194 };
4195 *differing_declarations = match self.admit_hello_authority(connection_id, hello_authority) {
4196 Ok(differing) => differing,
4197 Err(err) => {
4198 if matches!(
4199 self.rollback_unpublished_hello(connection_id),
4200 Ok(r) if !r.is_empty()
4201 ) {
4202 crate::supervise::notify_registration_release();
4203 }
4204 return Err(RouterError::Forwarding(err));
4205 }
4206 };
4207 #[cfg(test)]
4208 self.hold_between_registry_and_forwarding();
4209 let mut reply = Vec::new();
4210 let publication = self
4211 .registry
4212 .publish_registration_after(connection_id, || {
4213 if let Some(sink) = sink {
4214 let concurrency = manifest_concurrency(®istration.manifest);
4218 self.forwarding.register_candidate_module_connection_acked(
4219 connection_id,
4220 module_id.clone(),
4221 negotiated_ver,
4222 concurrency,
4223 sink,
4224 hello_ack,
4225 )?;
4226 } else {
4227 reply.push(hello_ack);
4228 }
4229 #[cfg(test)]
4230 self.hold_between_ack_and_publication();
4231 Ok(())
4232 })
4233 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4234 let published = match publication {
4235 Ok(published) => published,
4236 Err(err) => {
4237 if matches!(
4238 self.rollback_unpublished_hello(connection_id),
4239 Ok(r) if !r.is_empty()
4240 ) {
4241 crate::supervise::notify_registration_release();
4242 }
4243 self.refuse_hello_authority(connection_id, hello_authority);
4244 return Ok(vec![control_error_frame(
4245 frame,
4246 forwarding_error_code(&err),
4247 err.to_string(),
4248 )?]);
4249 }
4250 };
4251 if !published {
4252 debug!(
4253 module_id = %module_id,
4254 connection_id = connection_id.get(),
4255 "swap candidate registration was removed before it could be published"
4256 );
4257 self.refuse_hello_authority(connection_id, hello_authority);
4258 return Ok(reply);
4259 }
4260 info!(
4261 module_id = %module_id,
4262 module_version = %registration.manifest.module_version,
4263 negotiated_ver,
4264 ready = registration.ready,
4265 connection_id = connection_id.get(),
4266 "swap candidate registered; not routable until cutover"
4267 );
4268 #[cfg(windows)]
4269 if let Some(nonce) = launch_nonce {
4270 self.supervisor.program_hello_accepted(&module_id, nonce);
4271 }
4272 Ok(reply)
4273 }
4274
4275 fn admit_hello_authority(
4281 &self,
4282 connection: ConnectionId,
4283 authority: Option<&HelloAuthority>,
4284 ) -> Result<ProviderDeclarations, ForwardingError> {
4285 let Some(authority) = authority else {
4286 return Ok(ProviderDeclarations::new());
4287 };
4288 let froze = authority
4289 .record
4290 .freeze_declarations(authority.requested.clone());
4291 authority
4292 .froze
4293 .store(froze.is_some(), std::sync::atomic::Ordering::Relaxed);
4294 self.forwarding
4295 .bind_target_authority(connection, authority.record.clone())
4296 .inspect_err(|_| self.refuse_hello_authority(connection, Some(authority)))?;
4297 Ok(froze.unwrap_or_default())
4298 }
4299
4300 fn refuse_hello_authority(&self, connection: ConnectionId, authority: Option<&HelloAuthority>) {
4305 let Some(authority) = authority else {
4306 return;
4307 };
4308 if let Err(err) = self.forwarding.unbind_target_authority(connection) {
4309 warn!(error = %err, "failed to unbind a refused HELLO from its spawn");
4310 }
4311 if authority
4312 .froze
4313 .swap(false, std::sync::atomic::Ordering::Relaxed)
4314 {
4315 authority.record.unfreeze_refused_declarations();
4316 }
4317 }
4318
4319 fn build_hello_ack(
4320 &self,
4321 frame: &Frame,
4322 negotiated_ver: u8,
4323 module_id: &str,
4324 ) -> Result<Frame, RouterError> {
4325 let ack = ModuleHelloAckBody {
4326 negotiated_ver,
4327 subc_ops: module_subc_ops(),
4328 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
4329 storage: self
4330 .storage_config
4331 .as_ref()
4332 .map(|cfg| cfg.descriptor_for(module_id)),
4333 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4334 };
4335 let body = serde_json::to_vec(&ack).map_err(|err| {
4336 RouterError::backend(
4337 0,
4338 frame.header.corr,
4339 format!("failed to encode HELLO_ACK: {err}"),
4340 )
4341 })?;
4342
4343 Frame::build_with_version(
4344 negotiated_ver,
4345 FrameType::HelloAck,
4346 control_flags(),
4347 0,
4348 0,
4349 frame.header.corr,
4350 body,
4351 )
4352 .map_err(RouterError::FrameBuild)
4353 }
4354
4355 async fn handle_client_control_request(
4356 &self,
4357 ctx: &RouteCtx,
4358 frame: Frame,
4359 request: ClientControlRequest,
4360 route_open_timings: &mut Option<RouteOpenTimings>,
4361 ) -> Result<Vec<Frame>, RouterError> {
4362 if let Some(owner) = &self.config_runtime {
4363 let module = match &request {
4364 ClientControlRequest::SupervisorRestart { module_id, .. }
4365 | ClientControlRequest::SupervisorReload { module_id }
4366 | ClientControlRequest::SupervisorSwap { module_id, .. }
4367 | ClientControlRequest::SupervisorSetEnabled {
4368 module_id,
4369 enabled: true,
4370 } => Some(module_id.as_str()),
4371 _ => None,
4372 };
4373 let rescan = matches!(
4374 &request,
4375 ClientControlRequest::SupervisorRescan { preview: false }
4376 | ClientControlRequest::SupervisorRescanChecked { preview: false, .. }
4377 );
4378 let held = if module.is_some() || rescan {
4379 let module = module.map(str::to_owned);
4380 owner
4381 .on_blocking(move |owner| owner.checked_admission_hold(module.as_deref()))
4382 .await
4383 } else {
4384 Ok(())
4385 };
4386 if let Err(error) = held {
4387 if error.code != "config_delivery_deferred" {
4388 let code = if error.code == "store_tampered" {
4389 "store_tampered"
4390 } else {
4391 "config_gated"
4392 };
4393 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
4394 }
4395 }
4396 }
4397 if let Some(operation) = privileged_operation(&request) {
4398 let caller = self.resolve_caller(ctx.connection_id).await;
4399 let op = client_control_request_op(&request);
4400 let text = prompt_text(operation, &caller);
4401 match self.privileged_operator_prompt {
4402 PrivilegedOperatorPrompt::Log => {
4403 info!(target: "subc_daemon::privileged_prompt", operation = op,
4404 target = ?operation, caller = %caller.description(), summary = %text,
4405 "privileged request would prompt the operator");
4406 }
4407 PrivilegedOperatorPrompt::Enforce => {
4408 let key =
4409 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
4410 let stamp = self
4411 .operator_apps
4412 .lock()
4413 .unwrap_or_else(|p| p.into_inner())
4414 .get(&ctx.connection_id)
4415 .cloned();
4416 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
4417 &self.operator_app_policy,
4418 &self.launch_table,
4419 stamp.as_ref(),
4420 )
4421 .then(|| self.shortcut_context(stamp, None, None));
4422 let outcome = self
4423 .forwarding
4424 .operator_confirms()
4425 .approve_privileged_with_shortcut(
4426 ctx,
4427 &frame,
4428 op,
4429 text,
4430 caller.description(),
4431 key,
4432 shortcut,
4433 )
4434 .await;
4435 if outcome.code != "confirmed" {
4436 return Ok(vec![outcome.frame(&frame)]);
4437 }
4438 }
4439 }
4440 }
4441 match request {
4442 ClientControlRequest::DaemonShutdownIntent { intent } => {
4443 if intent.as_str() != Some("restart") {
4444 return Ok(vec![control_error_frame(
4445 &frame,
4446 "invalid_params",
4447 "intent must be 'restart'",
4448 )?]);
4449 }
4450 if let Some(owner) = &self.config_runtime {
4455 match owner
4456 .on_blocking(|owner| owner.assign_daemon_restart())
4457 .await
4458 {
4459 Ok(Some(generation)) => {
4460 info!(
4461 generation,
4462 "daemon restart will warm up approved generation"
4463 )
4464 }
4465 Ok(None) => {}
4466 Err(error) => warn!(
4467 %error,
4468 "daemon restart intent not recorded; the next start boots the last active generation"
4469 ),
4470 }
4471 }
4472 let response = ClientControlResponse::DaemonShutdownIntent {
4473 intent: "restart".to_owned(),
4474 expires_at_ms: self.supervisor.announce_daemon_restart(),
4475 };
4476 Ok(vec![control_response_body_frame(
4477 &frame,
4478 &response,
4479 "ClientControlResponse::DaemonShutdownIntent",
4480 )?])
4481 }
4482 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
4483 ClientControlRequest::CatalogList { module_id } => {
4484 self.handle_catalog_list(frame, module_id)
4485 }
4486 ClientControlRequest::RouteOpen {
4487 target,
4488 identity,
4489 consumer_identity,
4490 consumer_capabilities,
4491 role_versions,
4492 busy_refusals,
4493 admission_facts,
4494 scope,
4495 ..
4496 } => {
4497 let extensions = match serde_json::from_slice::<subc_control::RouteOpenRequestExt>(
4498 &frame.body,
4499 ) {
4500 Ok(extensions) => extensions,
4501 Err(err) => {
4502 return Ok(vec![control_error_frame(
4503 &frame,
4504 "invalid_control_body",
4505 format!("malformed route.open extensions: {err}"),
4506 )?]);
4507 }
4508 };
4509 self.handle_route_open(
4510 ctx,
4511 frame,
4512 RouteOpenRequest {
4513 target,
4514 identity,
4515 consumer_identity,
4516 consumer_capabilities,
4517 role_versions,
4518 busy_refusals,
4519 admission_facts,
4520 scope,
4521 accepts_close_reasons: extensions.accepts_close_reasons,
4522 },
4523 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
4524 )
4525 .await
4526 }
4527 ClientControlRequest::RoutePoll {
4528 route_channel,
4529 route_epoch,
4530 kind,
4531 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
4532 ClientControlRequest::SupervisorList {} => {
4533 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
4534 self.agent_declarations.refuse(
4535 self,
4536 ctx,
4537 control_error_frame(
4538 &frame,
4539 "busy",
4540 "supervisor list work is at capacity; retry",
4541 )?,
4542 );
4543 return Ok(Vec::new());
4544 };
4545 let handler = self.clone();
4547 let sink = ctx.egress.clone();
4548 tokio::spawn(
4549 async move {
4550 let _permit = permit;
4551 let result = handler.handle_supervisor_list(frame.clone()).await;
4552 let replies = match result {
4553 Ok(replies) => replies,
4554 Err(error) => {
4555 control_error_frame(&frame, "backend_error", error.to_string())
4556 .into_iter()
4557 .collect()
4558 }
4559 };
4560 for reply in replies {
4561 if !matches!(
4562 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
4563 .await,
4564 Ok(Ok(()))
4565 ) {
4566 break;
4567 }
4568 }
4569 }
4570 .with_current_subscriber(),
4571 );
4572 Ok(Vec::new())
4573 }
4574 ClientControlRequest::SupervisorSpawnSnapshot {} => {
4575 self.handle_supervisor_spawn_snapshot(frame)
4576 }
4577 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
4578 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
4579 }
4580 ClientControlRequest::SupervisorRestart {
4581 module_id,
4582 drain_timeout_ms,
4583 } => {
4584 self.log_supervisor_request_received(
4585 ctx,
4586 frame.header.corr,
4587 ops::SUPERVISOR_RESTART,
4588 Some(&module_id),
4589 None,
4590 )?;
4591 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4592 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
4593 .await
4594 }
4595 ClientControlRequest::SupervisorSwap {
4596 module_id,
4597 ready_timeout_ms,
4598 } => {
4599 self.log_supervisor_request_received(
4600 ctx,
4601 frame.header.corr,
4602 ops::SUPERVISOR_SWAP,
4603 Some(&module_id),
4604 None,
4605 )?;
4606 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4607 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
4608 .await
4609 }
4610 ClientControlRequest::SupervisorReload { module_id } => {
4611 self.log_supervisor_request_received(
4612 ctx,
4613 frame.header.corr,
4614 ops::SUPERVISOR_RELOAD,
4615 Some(&module_id),
4616 None,
4617 )?;
4618 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4619 self.handle_supervisor_reload(frame, module_id, requester)
4620 .await
4621 }
4622 ClientControlRequest::SupervisorRescan { preview } => {
4623 if !preview {
4624 self.log_supervisor_request_received(
4625 ctx,
4626 frame.header.corr,
4627 ops::SUPERVISOR_RESCAN,
4628 None,
4629 None,
4630 )?;
4631 }
4632 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4633 let operation = self.handle_supervisor_rescan(
4634 frame,
4635 if preview {
4636 RescanMode::Preview
4637 } else {
4638 RescanMode::Execute
4639 },
4640 requester,
4641 );
4642 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
4643 .await
4644 }
4645 ClientControlRequest::SupervisorRescanChecked {
4646 preview,
4647 candidate,
4648 expect,
4649 } => {
4650 self.observe_control_wait(
4651 ctx,
4652 None,
4653 ops::SUPERVISOR_RESCAN,
4654 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
4655 )
4656 .await
4657 }
4658 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
4659 self.log_supervisor_request_received(
4660 ctx,
4661 frame.header.corr,
4662 ops::SUPERVISOR_RELEASE_RESERVED,
4663 Some(&module_id),
4664 None,
4665 )?;
4666 self.handle_supervisor_release_reserved(frame, module_id)
4667 .await
4668 }
4669 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
4670 self.log_supervisor_request_received(
4671 ctx,
4672 frame.header.corr,
4673 ops::SUPERVISOR_SET_ENABLED,
4674 Some(&module_id),
4675 Some(enabled),
4676 )?;
4677 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4678 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
4679 .await
4680 }
4681 ClientControlRequest::SupervisorHealthProbe { module_id } => {
4682 self.handle_supervisor_health_probe(ctx, frame, module_id)
4683 .await
4684 }
4685 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
4686 ClientControlRequest::SupervisorRoutes { module_id } => {
4687 self.handle_supervisor_routes(frame, module_id)
4688 }
4689 ClientControlRequest::SupervisorProvenance { module_id } => {
4690 self.handle_supervisor_provenance(ctx, frame, module_id)
4691 .await
4692 }
4693 ClientControlRequest::SupervisorStderrTail {
4694 module_id,
4695 max_lines,
4696 max_bytes,
4697 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
4698 ClientControlRequest::SupervisorTerminals { module_id } => {
4699 self.handle_supervisor_terminals(ctx, frame, module_id)
4700 .await
4701 }
4702 }
4703 }
4704
4705 fn log_supervisor_request_received(
4706 &self,
4707 ctx: &RouteCtx,
4708 corr: u64,
4709 op: &'static str,
4710 module_id: Option<&str>,
4711 enabled: Option<bool>,
4712 ) -> Result<(), RouterError> {
4713 let caller = self.supervisor_requester(ctx, corr)?;
4714 let caller = principal_label(&caller);
4715 info!(
4718 target: "control",
4719 op,
4720 module_id,
4721 enabled,
4722 connection_id = ctx.connection_id.get(),
4723 caller = %caller,
4724 "supervisor request received"
4725 );
4726 Ok(())
4727 }
4728
4729 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
4730 Ok(self
4731 .registry
4732 .get_module_by_connection(ctx.connection_id)
4733 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4734 .map(|registration| Principal::Reserved {
4735 module_id: registration.manifest.module_id,
4736 })
4737 .unwrap_or(Principal::Direct))
4738 }
4739
4740 fn handle_module_control_request(
4741 &self,
4742 connection_id: ConnectionId,
4743 frame: Frame,
4744 request: ModuleControlRequestFromModule,
4745 ) -> Result<Vec<Frame>, RouterError> {
4746 match request {
4747 ModuleControlRequestFromModule::RouteAuthorize {
4748 principal,
4749 scope,
4750 target,
4751 expected,
4752 } => self.handle_route_authorize(
4753 connection_id,
4754 frame,
4755 principal,
4756 scope,
4757 target,
4758 expected,
4759 ),
4760 ModuleControlRequestFromModule::CatalogUpdate {
4761 provides,
4762 capabilities,
4763 ready,
4764 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
4765 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
4766 self.handle_catalog_update_role_declarations(
4767 connection_id,
4768 frame,
4769 role_declarations,
4770 )
4771 }
4772 ModuleControlRequestFromModule::LiveRoots {} => {
4773 let registered = self
4774 .registry
4775 .get_module_by_connection(connection_id)
4776 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4777 let Some(registration) = registered else {
4778 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
4779 };
4780 let response = self
4781 .forwarding
4782 .live_roots(®istration.manifest.module_id)
4783 .map_err(RouterError::Forwarding)?;
4784 Ok(vec![control_response_body_frame(
4785 &frame,
4786 &response,
4787 "ModuleControlResponseToModule::LiveRoots",
4788 )?])
4789 }
4790 ModuleControlRequestFromModule::AgentCommands {
4791 started,
4792 finished,
4793 known,
4794 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
4795 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
4796 self.handle_scope_sync(connection_id, frame, generation, scopes)
4797 }
4798 ModuleControlRequestFromModule::ScopeApply {
4799 generation,
4800 upsert,
4801 end,
4802 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
4803 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
4804 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
4805 }
4806 _ => Ok(vec![control_error_frame(
4807 &frame,
4808 "unsupported_control_frame",
4809 "unsupported module control operation",
4810 )?]),
4811 }
4812 }
4813
4814 fn handle_scope_served_by(
4815 &self,
4816 connection_id: ConnectionId,
4817 frame: Frame,
4818 ) -> Result<Vec<Frame>, RouterError> {
4819 let registered = self
4820 .registry
4821 .get_module_by_connection(connection_id)
4822 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4823 let Some(registered) = registered else {
4824 return Ok(vec![control_error_frame(
4825 &frame,
4826 "not_registered",
4827 "scope.served_by requires an active module registration owned by this connection",
4828 )?]);
4829 };
4830 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
4831 {
4832 Ok(request) => request,
4833 Err(err) => {
4834 return Ok(vec![control_error_frame(
4835 &frame,
4836 "invalid_control_body",
4837 format!("malformed scope.served_by body: {err}"),
4838 )?])
4839 }
4840 };
4841 let answer = self
4842 .scopes
4843 .read()
4844 .unwrap_or_else(|poisoned| poisoned.into_inner())
4845 .served_by(
4846 ®istered.manifest.module_id,
4847 &request.scope_ref,
4848 request.scope_epoch,
4849 &request.module_id,
4850 );
4851 let reply = subc_control::ScopeServedByReply::new(
4852 answer,
4853 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4854 );
4855 Ok(vec![control_response_body_frame(
4856 &frame,
4857 &reply,
4858 "ScopeServedByReply",
4859 )?])
4860 }
4861
4862 fn handle_operator_confirm(
4863 &self,
4864 ctx: &RouteCtx,
4865 frame: Frame,
4866 ) -> Result<Vec<Frame>, RouterError> {
4867 use crate::operator_confirm::{audit, Outcome};
4868 let registration = self
4869 .registry
4870 .get_module_by_connection(ctx.connection_id)
4871 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4872 let Some(registration) = registration else {
4873 let outcome = Outcome::refusal("not_registered");
4874 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4875 return Ok(vec![outcome.frame(&frame)]);
4876 };
4877 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
4878 Ok(request) => request,
4879 Err(_) => {
4880 let outcome = Outcome::refusal("invalid_control_body");
4881 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4882 return Ok(vec![outcome.frame(&frame)]);
4883 }
4884 };
4885 let declared = matches!(
4886 request.kind,
4887 Some(subc_protocol::session::OperatorConfirmKind::Operation)
4888 ) && request.operation.as_ref().is_some_and(|operation| {
4889 registration
4890 .manifest
4891 .confirm_operations
4892 .as_ref()
4893 .is_some_and(|operations| operations.contains(operation))
4894 });
4895 let module_id = registration.manifest.module_id;
4896 let nonce = self
4900 .hello_launch_nonces
4901 .lock()
4902 .unwrap_or_else(|p| p.into_inner())
4903 .nonce(ctx.connection_id)
4904 .map(str::to_owned);
4905 let nonce_proven = self
4908 .confirm_channel_proven(ctx.connection_id, &module_id)
4909 .unwrap_or_else(|| {
4910 nonce.as_deref().is_some_and(|nonce| {
4911 self.supervisor
4912 .spawned_consumer_authorized(&module_id, nonce)
4913 })
4914 });
4915 let confirms = self.forwarding.operator_confirms();
4916 self.forwarding
4917 .with_operator_route(
4918 ctx.connection_id,
4919 request.route_channel,
4920 request.route_epoch,
4921 |binding| {
4922 let shortcut = binding
4923 .filter(|binding| {
4924 nonce_proven
4925 && crate::operator_confirm::valid_summary(&request.summary)
4926 && declared
4927 && crate::operator_confirm::ShortcutContext::route_eligible(
4928 &self.operator_app_policy,
4929 &self.launch_table,
4930 binding,
4931 )
4932 })
4933 .map(|binding| {
4934 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
4935 &self.operator_app_policy,
4936 binding,
4937 );
4938 let stamp = if phone.is_some() {
4939 None
4940 } else {
4941 binding.operator_app.clone()
4942 };
4943 self.shortcut_context(
4944 stamp,
4945 phone,
4946 Some((request.route_channel, request.route_epoch)),
4947 )
4948 });
4949 confirms.admit_with_shortcut(
4950 ctx,
4951 frame,
4952 module_id,
4953 nonce_proven,
4954 request,
4955 binding,
4956 shortcut,
4957 )
4958 },
4959 )
4960 .map_err(RouterError::Forwarding)
4961 }
4962
4963 fn shortcut_context(
4964 &self,
4965 stamp: Option<crate::operator_apps::OperatorAppStamp>,
4966 phone_fingerprint: Option<String>,
4967 route: Option<(u16, u32)>,
4968 ) -> crate::operator_confirm::ShortcutContext {
4969 crate::operator_confirm::ShortcutContext {
4970 ledger: self.activity.clone(),
4971 forwarding: self.forwarding.clone(),
4972 launches: self.launch_table.clone(),
4973 stamp,
4974 phone_fingerprint,
4975 route,
4976 connections: self.operator_apps.clone(),
4977 #[cfg(test)]
4978 hooks: Default::default(),
4979 }
4980 }
4981
4982 #[cfg(any(test, feature = "test-support"))]
4984 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
4985 self.operator_app_policy.phone_enabled = enabled;
4986 self
4987 }
4988
4989 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
4992 let mut scopes = ScopeRecords::from(scopes);
4993 for (index, record) in scopes.records.iter_mut().enumerate() {
4994 if let Some(refusal) = Self::canonicalize_workdir(record) {
4995 scopes.refusals.insert(index, refusal);
4996 }
4997 }
4998 scopes
4999 }
5000
5001 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
5002 let workdir = record.workdir.as_mut()?;
5003 let path = std::path::Path::new(workdir);
5004 if !path.is_absolute() {
5005 return Some((
5006 error_codes::INVALID_WORKDIR,
5007 "workdir must be absolute".to_string(),
5008 ));
5009 }
5010 match ProjectRootId::from_path_allowing_missing(path) {
5011 Ok(root) => match root.as_path().to_str() {
5012 Some(canonical) => {
5013 *workdir = canonical.to_string();
5014 None
5015 }
5016 None => Some((
5017 error_codes::INVALID_WORKDIR,
5018 "canonical workdir is not UTF-8".to_string(),
5019 )),
5020 },
5021 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
5022 }
5023 }
5024
5025 fn handle_scope_sync(
5029 &self,
5030 connection_id: ConnectionId,
5031 frame: Frame,
5032 generation: u64,
5033 scopes: Vec<ScopeRecord>,
5034 ) -> Result<Vec<Frame>, RouterError> {
5035 self.handle_scope_change(connection_id, frame, generation, scopes, None)
5036 }
5037
5038 fn handle_scope_change(
5039 &self,
5040 connection_id: ConnectionId,
5041 frame: Frame,
5042 generation: u64,
5043 scopes: Vec<ScopeRecord>,
5044 end: Option<Vec<ScopeEnd>>,
5045 ) -> Result<Vec<Frame>, RouterError> {
5046 let is_apply = end.is_some();
5047 let Some(registration) = self
5048 .registry
5049 .get_module_by_connection(connection_id)
5050 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5051 else {
5052 return Ok(vec![control_error_frame(
5053 &frame,
5054 "not_registered",
5055 "scope.sync and scope.apply require an active module registration owned by this connection",
5056 )?]);
5057 };
5058 let owner = registration.manifest.module_id;
5059 let scopes = Self::prepare_scope_records(scopes);
5060 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
5061 let is_current_launch = |connection: ConnectionId| {
5062 self.connection_is_current_launch(&owner, connection, current_nonce.as_deref())
5063 };
5064 let mut table = self
5069 .scopes
5070 .write()
5071 .unwrap_or_else(|poisoned| poisoned.into_inner());
5072 let outcome = match end {
5073 Some(end) => table.apply(
5074 &owner,
5075 connection_id,
5076 is_current_launch,
5077 generation,
5078 scopes,
5079 end,
5080 ),
5081 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
5082 };
5083 let drained = match &outcome {
5084 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
5085 Err(_) => Vec::new(),
5086 };
5087 let has_deadlines = outcome.is_ok() && table.has_deadlines();
5088 drop(table);
5089 if has_deadlines {
5090 self.scope_deadline_added.notify_one();
5091 }
5092 match outcome {
5093 Ok(applied) => {
5094 let counts = ScopeOutcomeCounts::of(&applied.results);
5095 info!(
5096 owner = %owner,
5097 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5098 generation,
5099 records = applied.results.len(),
5100 created = counts.created,
5101 replaced = counts.replaced,
5102 updated = counts.updated,
5103 unchanged = counts.unchanged,
5104 refused = counts.refused,
5105 ended = applied.ended.len(),
5106 tag_changes = applied.tag_changes.len(),
5107 routes_closed = drained.len(),
5108 "scope change accepted"
5109 );
5110 for refused in applied
5116 .results
5117 .iter()
5118 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
5119 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
5120 {
5121 warn!(
5122 owner = %owner,
5123 generation,
5124 scope_ref = %refused.scope_ref,
5125 scope_epoch = refused.scope_epoch,
5126 code = refused.code.as_deref().unwrap_or(""),
5127 "scope record refused"
5128 );
5129 }
5130 self.close_scope_drained_routes(drained);
5131 let response = if is_apply {
5132 ModuleControlResponseToModule::ScopeApply {
5133 generation,
5134 results: applied.results,
5135 end_results: applied.end_results,
5136 ended: applied.ended,
5137 }
5138 } else {
5139 ModuleControlResponseToModule::ScopeSync {
5140 generation,
5141 results: applied.results,
5142 ended: applied.ended,
5143 }
5144 };
5145 Ok(vec![control_response_body_frame(
5146 &frame,
5147 &response,
5148 "scope change response",
5149 )?])
5150 }
5151 Err(refusal) => {
5152 info!(
5153 owner = %owner,
5154 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5155 generation,
5156 code = refusal.code,
5157 "scope change refused"
5158 );
5159 Ok(vec![control_error_frame(
5160 &frame,
5161 refusal.code,
5162 refusal.message,
5163 )?])
5164 }
5165 }
5166 }
5167
5168 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
5170 scope_expiry::ScopeExpiryContext::new(self).close_scope_drained_routes(drained);
5171 }
5172
5173 fn handle_route_authorize(
5179 &self,
5180 connection_id: ConnectionId,
5181 frame: Frame,
5182 principal: Principal,
5183 scope: subc_protocol::scope::RouteAuthorizeScope,
5184 target: String,
5185 expected: subc_protocol::scope::RouteAuthorizeExpected,
5186 ) -> Result<Vec<Frame>, RouterError> {
5187 use crate::scopes::admission::{admit_route, AdmissionMode};
5188 use subc_protocol::scope::{
5189 RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
5190 };
5191 let registry = self
5192 .registry
5193 .route_admission_snapshot()
5194 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5195 let owner_configured = match &scope.owner {
5196 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5197 _ => false,
5198 };
5199 let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
5200 let answer = {
5201 let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
5202 let permitted = registry
5203 .iter()
5204 .find(|r| r.connection_id == connection_id)
5205 .is_some_and(|r| {
5206 scopes.is_federated_owner(&Principal::Reserved {
5207 module_id: r.manifest.module_id.clone(),
5208 }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
5209 caps.provides
5210 .iter()
5211 .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
5212 })
5213 });
5214 if !permitted {
5215 drop(scopes);
5216 return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
5217 "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
5218 }
5219 admit_route(
5220 &scopes,
5221 ®istry,
5222 &principal,
5223 &target,
5224 Some(&selector),
5225 owner_configured,
5226 AdmissionMode::Authorize(&expected),
5227 )
5228 };
5229 let response = match answer {
5230 Ok(admission) => {
5231 let mut response = RouteAuthorizeResponse::new(
5232 RouteAuthorizeOutcome::Authorized,
5233 "route admission authorized",
5234 );
5235 response.content_version = admission.content_version;
5236 response
5237 }
5238 Err(refusal) => refusal.response(),
5239 };
5240 Ok(vec![control_response_body_frame(
5241 &frame,
5242 &response,
5243 "RouteAuthorizeResponse",
5244 )?])
5245 }
5246
5247 fn handle_scope_describe(
5250 &self,
5251 connection_id: ConnectionId,
5252 frame: Frame,
5253 owner: Principal,
5254 scope_ref: String,
5255 ) -> Result<Vec<Frame>, RouterError> {
5256 let registered = self
5257 .registry
5258 .get_module_by_connection(connection_id)
5259 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5260 let Some(registered) = registered else {
5261 return Ok(vec![control_error_frame(
5262 &frame,
5263 "not_registered",
5264 "scope.describe requires an active module registration owned by this connection",
5265 )?]);
5266 };
5267 let scopes = self
5268 .scopes
5269 .read()
5270 .unwrap_or_else(|poisoned| poisoned.into_inner());
5271 if scopes.is_federated_owner(&owner)
5272 && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
5273 {
5274 return Ok(vec![control_error_frame(
5275 &frame,
5276 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
5277 "scope.describe of the federated owner requires scope-federated/v1",
5278 )?]);
5279 }
5280 let mut description = scopes.describe(&owner, &scope_ref);
5281 drop(scopes);
5282 if owner
5283 != (Principal::Reserved {
5284 module_id: registered.manifest.module_id.clone(),
5285 })
5286 {
5287 if let Some(facts) = description
5288 .stamp
5289 .as_mut()
5290 .and_then(|stamp| stamp.attributes.federated.as_mut())
5291 {
5292 facts.stamp_required = false;
5293 }
5294 }
5295 if !registered
5298 .manifest
5299 .capabilities
5300 .as_ref()
5301 .is_some_and(|caps| {
5302 caps.provides
5303 .iter()
5304 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5305 })
5306 {
5307 if let Some(stamp) = description.stamp.as_mut() {
5308 stamp.attributes.on_behalf_of_agent = None;
5309 }
5310 }
5311 if !registered
5312 .manifest
5313 .capabilities
5314 .as_ref()
5315 .is_some_and(|caps| {
5316 caps.provides
5317 .iter()
5318 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5319 })
5320 {
5321 if let Some(stamp) = description.stamp.as_mut() {
5322 stamp.attributes.flow_version = None;
5323 stamp.attributes.flow_code_hash = None;
5324 stamp.attributes.flow_install_id = None;
5325 }
5326 }
5327 let owner_configured = match &owner {
5328 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5333 _ => false,
5334 };
5335 let response = ModuleControlResponseToModule::ScopeDescribe {
5336 status: description.status,
5337 scope_epoch: description.scope_epoch,
5338 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
5339 owner_synced: description.owner_synced,
5340 owner_configured,
5341 scope: description.stamp,
5342 };
5343 Ok(vec![control_response_body_frame(
5344 &frame,
5345 &response,
5346 "ModuleControlResponseToModule::ScopeDescribe",
5347 )?])
5348 }
5349
5350 fn handle_catalog_update_role_declarations(
5351 &self,
5352 connection_id: ConnectionId,
5353 frame: Frame,
5354 role_declarations: serde_json::Value,
5355 ) -> Result<Vec<Frame>, RouterError> {
5356 if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
5357 return Ok(vec![control_error_frame(
5358 &frame,
5359 "invalid_role_declarations",
5360 err.to_string(),
5361 )?]);
5362 }
5363 let declarations = serde_json::from_value(role_declarations)
5364 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5365 if self
5366 .registry
5367 .replace_role_declarations_for_connection(connection_id, declarations)
5368 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5369 .is_none()
5370 {
5371 return Ok(vec![control_error_frame(
5372 &frame,
5373 "not_registered",
5374 "catalog.update_role_declarations requires an active module registration owned by this connection",
5375 )?]);
5376 }
5377 control_response_body_frame(
5378 &frame,
5379 &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
5380 "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
5381 )
5382 .map(|frame| vec![frame])
5383 }
5384
5385 fn handle_catalog_update(
5386 &self,
5387 connection_id: ConnectionId,
5388 frame: Frame,
5389 provides: Vec<ProviderRole>,
5390 capabilities: Option<CapabilityDeclarations>,
5391 ready: Option<bool>,
5392 ) -> Result<Vec<Frame>, RouterError> {
5393 self.refresh_capability_requirements();
5394 let Some(registration) = self
5395 .registry
5396 .get_module_by_connection(connection_id)
5397 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5398 else {
5399 return Ok(vec![control_error_frame(
5400 &frame,
5401 "not_registered",
5402 "catalog.update requires an active module registration owned by this connection",
5403 )?]);
5404 };
5405
5406 if let Some(message) =
5407 catalog_update_frozen_field_message(®istration.manifest, &provides)
5408 {
5409 return Ok(vec![control_error_frame(
5410 &frame,
5411 "catalog_update_frozen_field",
5412 message,
5413 )?]);
5414 }
5415
5416 let mut candidate = registration.manifest.clone();
5417 candidate.provides = provides.clone();
5418 candidate.capabilities = capabilities
5419 .clone()
5420 .or_else(|| registration.manifest.capabilities.clone());
5421 if let Some(rich_ui) = &candidate.rich_ui {
5422 if let Err(err) = rich_ui.validate() {
5423 return Ok(vec![control_error_frame(
5424 &frame,
5425 "invalid_manifest",
5426 err.to_string(),
5427 )?]);
5428 }
5429 }
5430 if let Err(err) = candidate.validate_capability_grammar() {
5431 return Ok(vec![control_error_frame(
5432 &frame,
5433 "invalid_capability_grammar",
5434 err.to_string(),
5435 )?]);
5436 }
5437
5438 let mut conflicts = self
5441 .capability_evaluator
5442 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
5443 if let Some(conflict) = conflicts.first() {
5444 let message = format!(
5445 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
5446 conflict.capability, conflict.claimants[0], candidate.module_id
5447 );
5448 for conflict in &mut conflicts {
5449 conflict.source = DuplicateClaimSource::CatalogUpdate;
5450 }
5451 log_duplicate_claim_events(conflicts);
5452 return Ok(vec![control_error_frame(
5453 &frame,
5454 "reserved_capability",
5455 message,
5456 )?]);
5457 }
5458
5459 let updated = match self.registry.replace_catalog_for_connection(
5460 connection_id,
5461 provides,
5462 capabilities,
5463 ready,
5464 ) {
5465 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => {
5466 return Ok(vec![control_error_frame(
5467 &frame,
5468 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5469 "scope-federated/v1 is fixed at HELLO; restart the module to change it",
5470 )?])
5471 }
5472 Err(crate::registry::RegistryError::CallerAttestationCapabilityChanged) => {
5473 return Ok(vec![control_error_frame(
5474 &frame,
5475 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5476 "caller-attestation/v1 is fixed at HELLO; restart the module to change it",
5477 )?])
5478 }
5479 result => {
5480 result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5481 }
5482 };
5483 if updated.is_none() {
5484 return Ok(vec![control_error_frame(
5485 &frame,
5486 "not_registered",
5487 "catalog.update requires an active module registration owned by this connection",
5488 )?]);
5489 }
5490 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
5491 log_duplicate_claim_events(
5492 self.capability_evaluator
5493 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
5494 );
5495 }
5496 if capability_census_trigger(
5497 registration.manifest.capabilities.as_ref(),
5498 updated
5499 .as_ref()
5500 .and_then(|entry| entry.manifest.capabilities.as_ref()),
5501 ) {
5502 self.enforce_capability_denies();
5503 }
5504 self.refresh_capability_requirements();
5505
5506 let response = ModuleControlResponseToModule::CatalogUpdate {};
5507 control_response_body_frame(
5508 &frame,
5509 &response,
5510 "ModuleControlResponseToModule::CatalogUpdate",
5511 )
5512 .map(|frame| vec![frame])
5513 }
5514
5515 pub(crate) fn with_activity_ledger_config(
5516 mut self,
5517 run_dir: &Path,
5518 incarnation: String,
5519 config: crate::activity::ActivityConfig,
5520 ) -> Self {
5521 self.activity = Some(crate::activity::ActivityLedger::open_config(
5522 run_dir.join("activity.jsonl"),
5523 incarnation,
5524 config,
5525 ));
5526 self.supervisor.set_activity(self.activity.clone());
5527 self
5528 }
5529
5530 pub(crate) async fn configure_activity(
5531 &self,
5532 config: crate::activity::ActivityConfig,
5533 ) -> Result<(), &'static str> {
5534 match &self.activity {
5535 Some(ledger) => ledger.configure(config).await,
5536 None => Ok(()),
5537 }
5538 }
5539
5540 fn cancel_activity_subscription(
5541 &self,
5542 ctx: &RouteCtx,
5543 frame: &Frame,
5544 ) -> Result<bool, RouterError> {
5545 let task = self
5546 .activity_streams
5547 .lock()
5548 .unwrap_or_else(|p| p.into_inner())
5549 .remove(&(ctx.connection_id, frame.header.corr));
5550 let Some(task) = task else {
5551 return Ok(false);
5552 };
5553 task.abort();
5554 let end = Frame::build_with_version(
5555 response_version(frame),
5556 FrameType::StreamEnd,
5557 control_flags(),
5558 0,
5559 0,
5560 frame.header.corr,
5561 Vec::new(),
5562 )
5563 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
5564 let sink = ctx.egress.clone();
5565 tokio::spawn(async move {
5566 let _ = sink.send(end).await;
5567 });
5568 Ok(true)
5569 }
5570
5571 async fn handle_activity_read(
5572 &self,
5573 ctx: &RouteCtx,
5574 frame: Frame,
5575 ) -> Result<Vec<Frame>, RouterError> {
5576 #[cfg(test)]
5577 {
5578 let pause = self
5579 .activity_request_pause
5580 .lock()
5581 .unwrap()
5582 .clone()
5583 .filter(|pause| pause.connection == ctx.connection_id);
5584 if let Some(pause) = pause {
5585 let _cancelled = ActivityRequestCancellation(pause.clone());
5586 pause.started.notify_one();
5587 std::future::pending::<()>().await;
5588 }
5589 }
5590 use subc_control::{
5591 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
5592 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
5593 };
5594 #[derive(Deserialize)]
5595 struct Identity {
5596 #[serde(default)]
5597 consumer_identity: Option<ConsumerIdentity>,
5598 }
5599 let mut bound_spawn = None;
5603 let legacy = match self.activity_reader(ctx.connection_id) {
5604 channel_operations::ActivityReader::Module { module_id, spawn } => {
5605 bound_spawn = Some(spawn);
5606 Some(Ok(Principal::Reserved { module_id }))
5607 }
5608 channel_operations::ActivityReader::Refused => {
5610 return Ok(vec![control_error_frame(
5611 &frame,
5612 ACTIVITY_READ_FORBIDDEN,
5613 "helper, relay and retired connections cannot read activity",
5614 )?]);
5615 }
5616 channel_operations::ActivityReader::Legacy => None,
5617 };
5618 let principal = match legacy {
5622 Some(principal) => principal,
5623 None => match self
5624 .forwarding
5625 .module_id_for_connection(ctx.connection_id)
5626 .map_err(RouterError::Forwarding)?
5627 {
5628 Some(module_id) => Ok(Principal::Reserved { module_id }),
5629 None => {
5630 let identity: Identity = match serde_json::from_slice(&frame.body) {
5631 Ok(identity) => identity,
5632 Err(_) => {
5633 return Ok(vec![control_error_frame(
5634 &frame,
5635 ACTIVITY_MALFORMED,
5636 "malformed activity read",
5637 )?])
5638 }
5639 };
5640 self.route_open_principal(
5641 ctx.connection_id,
5642 &frame,
5643 identity.consumer_identity,
5644 )?
5645 }
5646 },
5647 };
5648 let admitted = matches!(principal, Ok(Principal::Direct))
5649 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
5650 if !admitted {
5651 return Ok(vec![control_error_frame(
5652 &frame,
5653 ACTIVITY_READ_FORBIDDEN,
5654 "activity reads require an operator or trusted mirror principal",
5655 )?]);
5656 }
5657 let Some(ledger) = &self.activity else {
5658 return Ok(vec![control_error_frame(
5659 &frame,
5660 ACTIVITY_UNAVAILABLE,
5661 "activity ledger unavailable",
5662 )?]);
5663 };
5664 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
5665 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
5666 {
5667 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
5668 Ok(request) => ledger.query(request).await,
5669 Err(_) => Err(ACTIVITY_MALFORMED),
5670 };
5671 return match result {
5672 Ok(reply) => Ok(vec![control_response_body_frame(
5673 &frame,
5674 &reply,
5675 ops::ACTIVITY_QUERY,
5676 )?]),
5677 Err(code) => Ok(vec![control_error_frame(
5678 &frame,
5679 code,
5680 "activity query refused",
5681 )?]),
5682 };
5683 }
5684 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
5685 Ok(request) => request,
5686 Err(_) => {
5687 return Ok(vec![control_error_frame(
5688 &frame,
5689 ACTIVITY_MALFORMED,
5690 "malformed activity subscription",
5691 )?])
5692 }
5693 };
5694 let mut subscription = match ledger.subscribe(request.since).await {
5695 Ok(subscription) => subscription,
5696 Err(code) => {
5697 return Ok(vec![control_error_frame(
5698 &frame,
5699 code,
5700 "activity subscription refused",
5701 )?])
5702 }
5703 };
5704 let key = (ctx.connection_id, frame.header.corr);
5705 let sink = ctx.egress.clone();
5706 let mut streams = self
5707 .activity_streams
5708 .lock()
5709 .unwrap_or_else(|p| p.into_inner());
5710 streams.retain(|_, task| !task.is_finished());
5711 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
5712 return Ok(vec![control_error_frame(
5713 &frame,
5714 "duplicate_subscription",
5715 "activity subscription correlation id already in use",
5716 )?]);
5717 }
5718 let stream = async move {
5719 loop {
5720 let entry = if let Some(entry) = subscription.replay.pop_front() {
5721 Some(entry)
5722 } else {
5723 subscription.receiver.recv().await
5724 };
5725 let Some(entry) = entry else {
5726 break;
5727 };
5728 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
5729 return;
5730 };
5731 let Ok(next) = Frame::build_with_version(
5732 response_version(&frame),
5733 FrameType::StreamData,
5734 control_flags(),
5735 0,
5736 0,
5737 frame.header.corr,
5738 body,
5739 ) else {
5740 return;
5741 };
5742 if sink.send(next).await.is_err() {
5743 return;
5744 }
5745 }
5746 if let Some(code) = subscription.terminal_code() {
5749 if let Ok(end) =
5750 control_error_frame(&frame, code, "activity subscriber fell behind")
5751 {
5752 let _ = sink.send(end).await;
5753 }
5754 }
5755 };
5756 let task = tokio::spawn(async move {
5760 match bound_spawn {
5761 Some(spawn) => {
5762 tokio::select! {
5763 () = stream => {}
5764 () = spawn.wait_closed() => {}
5765 }
5766 }
5767 None => stream.await,
5768 }
5769 });
5770 streams.insert(key, task.abort_handle());
5771 Ok(Vec::new())
5772 }
5773
5774 async fn handle_activity_record(
5775 &self,
5776 ctx: &RouteCtx,
5777 frame: Frame,
5778 ) -> Result<Vec<Frame>, RouterError> {
5779 use subc_control::{
5780 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
5781 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
5782 };
5783 let result = async {
5784 let module = self
5785 .forwarding
5786 .module_id_for_connection(ctx.connection_id)
5787 .ok()
5788 .flatten()
5789 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
5790 let request: ActivityRecordRequest =
5791 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
5792 crate::activity::validate(&module, &request)?;
5793 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
5794 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
5795 let at_ms = std::time::SystemTime::now()
5796 .duration_since(std::time::UNIX_EPOCH)
5797 .unwrap_or_default()
5798 .as_millis() as u64;
5799 ledger.record(module, request, actor, at_ms).await
5800 }
5801 .await;
5802 match result {
5803 Ok(entry) => Ok(vec![control_response_body_frame(
5804 &frame,
5805 &ActivityRecordResponse { seq: entry.seq },
5806 "activity.record",
5807 )?]),
5808 Err(code) => Ok(vec![control_error_frame(
5809 &frame,
5810 code,
5811 "activity recording refused",
5812 )?]),
5813 }
5814 }
5815
5816 fn activity_actor(
5817 &self,
5818 connection: ConnectionId,
5819 module: &str,
5820 on: Option<&subc_control::ActivityOn>,
5821 ) -> Result<subc_control::ActivityActor, &'static str> {
5822 use crate::forwarding::{DataRoute, DataRouteState};
5823 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
5824 let principal = |id: &str| Principal::Reserved {
5825 module_id: id.to_owned(),
5826 };
5827 let mut actor = ActivityActor {
5828 principal: principal(module),
5829 scope_owner: None,
5830 agent_id: None,
5831 run_id: None,
5832 flow_id: None,
5833 project_root: None,
5834 workdir: None,
5835 project_id: None,
5836 };
5837 let scope = match on {
5838 None => return Ok(actor),
5839 Some(ActivityOn::Scope(scope)) => {
5840 let selector = subc_protocol::scope::ScopeSelector::new(
5841 principal(module),
5842 scope.scope_ref.clone(),
5843 Some(scope.epoch),
5844 );
5845 self.scopes
5846 .read()
5847 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5848 .attribute(&principal(module), module, &selector, true)
5849 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5850 }
5851 Some(ActivityOn::Route(route)) => {
5852 let DataRoute::Module(DataRouteState::Bound(binding)) = self
5853 .forwarding
5854 .lookup_data_route(connection, route.channel, route.epoch)
5855 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5856 else {
5857 return Err(ACTIVITY_ACTOR_UNKNOWN);
5858 };
5859 if binding.module_id != module {
5860 return Err(ACTIVITY_ACTOR_UNKNOWN);
5861 }
5862 actor.principal = binding.principal.clone();
5863 actor.project_root = binding
5864 .project_root
5865 .as_ref()
5866 .map(|root| root.as_path().to_path_buf());
5867 let Some(bound) = &binding.scope else {
5868 return Ok(actor);
5869 };
5870 let owner = principal(&bound.owner);
5871 let selector = subc_protocol::scope::ScopeSelector::new(
5872 owner.clone(),
5873 bound.scope_ref.clone(),
5874 Some(bound.tag.scope_epoch),
5875 );
5876 let admitted = self
5877 .scopes
5878 .read()
5879 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5880 .attribute(&owner, module, &selector, true)
5881 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
5882 if admitted.tag != bound.tag {
5883 return Err(ACTIVITY_ACTOR_UNKNOWN);
5884 }
5885 admitted
5886 }
5887 };
5888 actor.scope_owner = Some(scope.stamp.owner);
5889 actor.agent_id = scope.stamp.attributes.agent_id;
5890 actor.run_id = scope.stamp.attributes.run_id;
5891 actor.flow_id = scope.stamp.attributes.flow_id;
5892 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
5893 actor.project_id = scope.stamp.project_id;
5894 Ok(actor)
5895 }
5896
5897 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5898 self.refresh_capability_requirements();
5899 let mut counters = self.counters.snapshot();
5905 let losses = self.resource_tracker.loss_diagnostics();
5906 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
5907 if let Some(obj) = counters.as_object_mut() {
5908 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
5909 }
5910 }
5911 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
5912 obj.insert(
5913 "activity_unparseable_lines".into(),
5914 ledger.read_errors().into(),
5915 );
5916 obj.insert(
5917 "activity_lifecycle_append_failures".into(),
5918 ledger.lifecycle_failures().into(),
5919 );
5920 obj.insert(
5921 "activity_started_digest_skips".into(),
5922 ledger.lifecycle_digest_skips().into(),
5923 );
5924 }
5925 if let (Ok((connections_with_routes, max)), Some(obj)) = (
5926 self.forwarding.client_route_concentration(),
5927 counters.as_object_mut(),
5928 ) {
5929 obj.insert(
5930 "client_connections_with_routes".into(),
5931 connections_with_routes.into(),
5932 );
5933 obj.insert("max_routes_on_one_connection".into(), max.into());
5934 }
5935 if let (Some(open_breakers), Some(obj)) = (
5941 self.route_bind_breakers.open_snapshot(),
5942 counters.as_object_mut(),
5943 ) {
5944 obj.insert("route_bind_breakers_open".into(), open_breakers);
5945 }
5946 if let (Some(reason), Some(obj)) = (
5951 self.local_endpoint_unavailable.clone(),
5952 counters.as_object_mut(),
5953 ) {
5954 obj.insert("local_endpoint_unavailable".into(), reason.into());
5955 }
5956 let response = ClientControlResponse::ServerDescribe {
5957 protocol_ver: PROTOCOL_VERSION,
5958 subc_ops: subc_ops(),
5959 capabilities: self.describe_capabilities(),
5960 connected_clients: self.connected_clients.count(),
5961 counters: Some(counters),
5962 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
5963 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
5964 capability_requirements: self.capability_requirement_statuses(),
5965 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
5966 config_path: self
5967 .rescan
5968 .as_ref()
5969 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
5970 };
5971 Ok(vec![control_response_body_frame(
5972 &frame,
5973 &response,
5974 "ClientControlResponse::ServerDescribe",
5975 )?])
5976 }
5977
5978 fn handle_catalog_list(
5979 &self,
5980 frame: Frame,
5981 module_id: Option<String>,
5982 ) -> Result<Vec<Frame>, RouterError> {
5983 let (generation, modules) = self
5984 .registry
5985 .list_modules_with_nonce_verification()
5986 .map_err(|err| {
5987 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
5988 })?;
5989 let entries = modules
5990 .into_iter()
5991 .filter(|(registration, _)| {
5992 module_id
5993 .as_deref()
5994 .map(|wanted| registration.manifest.module_id == wanted)
5995 .unwrap_or(true)
5996 })
5997 .map(|(registration, nonce_verified)| {
5998 let entry = self
5999 .catalog_discovery_entry(®istration, module_id.is_some())
6000 .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
6001 let not_ready = self.not_ready_reason(®istration);
6002 let mut entry = entry;
6003 entry.roles = registration.manifest.provides;
6004 entry.control_ops = registration.control_ops;
6005 Ok(entry
6006 .with_ready(not_ready.is_none())
6007 .with_display_name(registration.manifest.display_name)
6008 .with_not_ready(not_ready)
6009 .with_module_version(Some(registration.manifest.module_version))
6010 .with_provenance(registration.manifest.provenance)
6011 .with_capabilities(registration.manifest.capabilities)
6012 .with_self_signals(registration.manifest.self_signals)
6013 .with_events(registration.manifest.events)
6014 .with_operation_delivery(registration.manifest.operation_delivery)
6015 .with_operation_payload(registration.manifest.operation_payload)
6016 .with_flow_operations(registration.manifest.flow_operations)
6017 .with_operation_deferral(registration.manifest.operation_deferral)
6018 .with_confirm_operations(registration.manifest.confirm_operations)
6019 .with_rich_ui(registration.manifest.rich_ui)
6020 .with_role_declarations(registration.manifest.role_declarations)
6021 .with_nonce_verified(nonce_verified))
6022 })
6023 .collect::<Result<Vec<_>, RouterError>>()?;
6024 let response = ClientControlResponse::CatalogList {
6025 generation,
6026 modules: entries,
6027 subc_ops: subc_ops(),
6028 };
6029 Ok(vec![control_response_body_frame(
6030 &frame,
6031 &response,
6032 "ClientControlResponse::CatalogList",
6033 )?])
6034 }
6035
6036 fn route_open_caller_from_frame(&self, ctx: &RouteCtx, frame: &Frame) -> RouteOpenCaller {
6040 let Ok(ClientControlRequest::RouteOpen {
6041 identity,
6042 consumer_identity,
6043 scope,
6044 ..
6045 }) = parse_client_control_request(&frame.body)
6046 else {
6047 return RouteOpenCaller::unknown();
6048 };
6049 let principal = self
6050 .attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref())
6051 .map(|attested| attested.principal);
6052 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
6053 }
6054
6055 fn route_open_principal(
6056 &self,
6057 connection_id: ConnectionId,
6058 frame: &Frame,
6059 consumer_identity: Option<ConsumerIdentity>,
6060 ) -> Result<Result<Principal, Frame>, RouterError> {
6061 let attested = self
6062 .attest_route_open_consumer(connection_id, consumer_identity.as_ref())
6063 .map(|attested| attested.principal);
6064 self.route_open_principal_attested(frame, consumer_identity, attested)
6065 }
6066
6067 fn route_open_principal_attested(
6071 &self,
6072 frame: &Frame,
6073 consumer_identity: Option<ConsumerIdentity>,
6074 attested: Option<Principal>,
6075 ) -> Result<Result<Principal, Frame>, RouterError> {
6076 if let Some(principal) = attested {
6077 return Ok(Ok(principal));
6078 }
6079 Ok(Err(control_error_frame(
6080 frame,
6081 "bad_consumer_identity",
6082 consumer_identity.map_or_else(
6083 || "connection claim is no longer authorized for routes".to_string(),
6084 |identity| format!(
6085 "consumer_identity for module_id '{}' conflicts with its connection claim or did not match a supervised launch nonce",
6086 identity.module_id
6087 ),
6088 ),
6089 )?))
6090 }
6091
6092 #[allow(clippy::too_many_arguments)]
6099 fn route_open_refusal_frame(
6100 &self,
6101 ctx: &RouteCtx,
6102 caller: &RouteOpenCaller,
6103 frame: &Frame,
6104 module_id: &str,
6105 reason: &'static str,
6106 code: &'static str,
6107 message: impl Into<String>,
6108 ) -> Result<Frame, RouterError> {
6109 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
6110 let detail = if matches!(
6111 reason,
6112 "registration_not_active"
6113 | "supervisor_not_live"
6114 | "no_forwarding_connection"
6115 | "reloading"
6116 ) {
6117 let expected_back = (reason == "reloading").then_some(true);
6120 self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
6121 } else {
6122 None
6123 };
6124 control_error_body_frame(
6125 frame,
6126 ErrorBody {
6127 code: code.to_string(),
6128 message: message.into(),
6129 detail,
6130 },
6131 )
6132 }
6133
6134 fn route_open_recovery_detail(
6135 &self,
6136 module_id: &str,
6137 corr: u64,
6138 detail: Option<serde_json::Value>,
6139 expected_back: Option<bool>,
6140 ) -> Result<Option<serde_json::Value>, RouterError> {
6141 let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
6142 return Ok(detail);
6143 };
6144 let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
6145 detail["state"] = serde_json::json!(status.state.to_string());
6146 if let Some(expected_back) = expected_back {
6147 detail["expected_back"] = serde_json::json!(expected_back);
6148 } else if let Some(module) = self.supervisor.get(module_id) {
6149 match module.will_recover_after_connection_loss() {
6150 Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
6151 Err(err) => warn!(
6152 %module_id,
6153 error = %err,
6154 "failed to read route.open recovery verdict; omitting expected_back"
6155 ),
6156 }
6157 }
6158 Ok(Some(detail))
6159 }
6160
6161 #[allow(clippy::too_many_arguments)]
6170 fn route_bind_rejection(
6171 &self,
6172 ctx: &RouteCtx,
6173 caller: &RouteOpenCaller,
6174 frame: &Frame,
6175 target_module_id: &str,
6176 body: ErrorBody,
6177 daemon_authored: bool,
6178 ) -> Result<Vec<Frame>, RouterError> {
6179 if daemon_authored {
6185 let scope_code = match body.code.as_str() {
6186 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
6187 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
6188 _ => None,
6189 };
6190 if let Some(code) = scope_code {
6191 self.observe_route_open_refusal(
6192 ctx,
6193 caller,
6194 target_module_id,
6195 "scope_changed_before_commit",
6196 code,
6197 );
6198 return Ok(vec![control_error_body_frame(frame, body)?]);
6199 }
6200 }
6201 self.counters
6202 .increment_route_open_refused("module_rejected");
6203 info!(
6204 target: "control",
6205 code = "module_rejected",
6206 module_code = ?body.code,
6207 module_id = ?target_module_id,
6208 connection_id = ctx.connection_id.get(),
6209 principal = %caller.principal,
6210 scope = %caller.scope,
6211 harness = %caller.harness,
6212 "route.open refused"
6213 );
6214 let refusal = if daemon_authored {
6215 control_error_body_frame(frame, body)?
6216 } else {
6217 control_relayed_error_body_frame(frame, body)?
6218 };
6219 Ok(vec![refusal])
6220 }
6221
6222 #[allow(clippy::too_many_arguments)]
6240 fn route_open_breaker_refusal_frame(
6241 &self,
6242 ctx: &RouteCtx,
6243 caller: &RouteOpenCaller,
6244 frame: &Frame,
6245 module_id: &str,
6246 consecutive_timeouts: u32,
6247 retry_in: Duration,
6248 probe_in_flight: bool,
6249 ) -> Result<Frame, RouterError> {
6250 self.counters
6251 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
6252 debug!(
6253 target: "control",
6254 code = "module_timeout",
6255 module_id = ?module_id,
6256 connection_id = ctx.connection_id.get(),
6257 principal = %caller.principal,
6258 scope = %caller.scope,
6259 harness = %caller.harness,
6260 consecutive_timeouts,
6261 retry_in_ms = retry_in.as_millis() as u64,
6262 probe_in_flight,
6263 "route.open refused by open bind-relay breaker"
6264 );
6265 let detail = if probe_in_flight {
6272 "one test route is already being tried; retry once it settles".to_string()
6273 } else {
6274 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
6275 };
6276 control_error_frame(
6277 frame,
6278 "module_timeout",
6279 format!(
6280 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
6281 timed out in a row); {detail}; its established routes are unaffected"
6282 ),
6283 )
6284 }
6285
6286 fn observe_route_open_refusal(
6299 &self,
6300 ctx: &RouteCtx,
6301 caller: &RouteOpenCaller,
6302 module_id: &str,
6303 reason: &'static str,
6304 code: &'static str,
6305 ) {
6306 self.counters.increment_route_open_refused(code);
6307 info!(
6308 target: "control",
6309 code,
6310 reason,
6311 module_id = ?module_id,
6312 connection_id = ctx.connection_id.get(),
6313 principal = %caller.principal,
6314 scope = %caller.scope,
6315 harness = %caller.harness,
6316 "route.open refused"
6317 );
6318 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
6319 self.route_outages.record_not_serving(module_id, reason);
6320 }
6321 }
6322
6323 fn observe_route_open_accept(
6368 &self,
6369 ctx: &RouteCtx,
6370 module_id: &str,
6371 principal: &str,
6372 scope: &str,
6373 ) {
6374 self.route_outages.record_accepted(module_id);
6375 self.counters.increment_route_open_accepted(principal);
6376 info!(
6377 target: "control",
6378 principal,
6379 module_id,
6380 connection_id = ctx.connection_id.get(),
6381 scope = %scope,
6382 "route.open accepted"
6383 );
6384 }
6385
6386 fn supervised_absent_route_open_refusal_frame(
6387 &self,
6388 ctx: &RouteCtx,
6389 caller: &RouteOpenCaller,
6390 frame: &Frame,
6391 module_id: &str,
6392 code: &'static str,
6393 status: &crate::supervise::ModuleStatus,
6394 ) -> Result<Frame, RouterError> {
6395 self.counters.increment_route_open_refused(code);
6396 info!(
6397 target: "control",
6398 code,
6399 reason = "supervised_not_registered",
6400 module_id = ?module_id,
6401 connection_id = ctx.connection_id.get(),
6402 principal = %caller.principal,
6403 scope = %caller.scope,
6404 harness = %caller.harness,
6405 state = %status.state,
6406 enabled = status.enabled,
6407 live = status.live,
6408 "route.open refused"
6409 );
6410 self.route_outages
6414 .record_not_serving(module_id, "supervised_not_registered");
6415 let message = if let Some(pid) = status.waiting_for_previous_pid {
6416 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
6417 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
6418 })
6419 } else {
6420 format!(
6421 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
6422 status.state, status.enabled, status.live
6423 )
6424 };
6425 let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
6429 control_error_body_frame(
6430 frame,
6431 ErrorBody {
6432 code: code.to_string(),
6433 message,
6434 detail: self.route_open_recovery_detail(
6435 module_id,
6436 frame.header.corr,
6437 None,
6438 expected_back,
6439 )?,
6440 },
6441 )
6442 }
6443
6444 async fn handle_route_open(
6445 &self,
6446 ctx: &RouteCtx,
6447 frame: Frame,
6448 request: RouteOpenRequest,
6449 timings: &mut RouteOpenTimings,
6450 ) -> Result<Vec<Frame>, RouterError> {
6451 let RouteOpenRequest {
6452 target,
6453 mut identity,
6454 consumer_identity,
6455 consumer_capabilities,
6456 role_versions,
6457 busy_refusals,
6458 admission_facts,
6459 scope,
6460 accepts_close_reasons,
6461 } = request;
6462 let target_module_id = target_module_id(&target).to_string();
6463 timings.module_id.clone_from(&target_module_id);
6464 if self
6465 .registry
6466 .get_module_by_connection(ctx.connection_id)
6467 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6468 .is_some()
6469 {
6470 return Ok(vec![control_error_frame(
6471 &frame,
6472 "invalid_request",
6473 "module connections cannot open client routes",
6474 )?]);
6475 }
6476 debug!(
6477 connection_id = ctx.connection_id.get(),
6478 corr = frame.header.corr,
6479 module_id = %target_module_id,
6480 "handling route.open"
6481 );
6482
6483 let attested =
6489 self.attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref());
6490 let mut caller = RouteOpenCaller::new(
6491 attested.as_ref().map(|attested| &attested.principal),
6492 scope.as_ref(),
6493 &identity,
6494 );
6495
6496 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
6502 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
6503 self.observe_route_open_refusal(
6504 ctx,
6505 &caller,
6506 &target_module_id,
6507 "invalid_role_versions",
6508 error_codes::INVALID_REQUEST,
6509 );
6510 return Ok(vec![control_error_body_frame(
6511 &frame,
6512 ErrorBody {
6513 code: error_codes::INVALID_REQUEST.to_string(),
6514 message: error.to_string(),
6515 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
6516 },
6517 )?]);
6518 }
6519
6520 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
6521 if status.waiting_for_previous_pid.is_some() {
6522 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6523 ctx,
6524 &caller,
6525 &frame,
6526 &target_module_id,
6527 "module_warming",
6528 &status,
6529 )?]);
6530 }
6531 }
6532
6533 let admission_registry = self
6552 .registry
6553 .route_admission_snapshot()
6554 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6555 let Some(registration) =
6556 crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
6557 else {
6558 if let Some((status, warming)) =
6559 self.supervisor_status(&target_module_id, frame.header.corr)?
6560 {
6561 if status.protocol == ModuleProtocol::None {
6568 return Ok(vec![self.route_open_refusal_frame(
6569 ctx,
6570 &caller,
6571 &frame,
6572 &target_module_id,
6573 "protocol_none",
6574 error_codes::MODULE_NO_PROTOCOL,
6575 format!(
6576 "module_id '{target_module_id}' is declared protocol: none; \
6577 it speaks no subc wire and serves no routes"
6578 ),
6579 )?]);
6580 }
6581 let code = if warming {
6582 "module_warming"
6583 } else {
6584 "target_unavailable"
6585 };
6586 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6587 ctx,
6588 &caller,
6589 &frame,
6590 &target_module_id,
6591 code,
6592 &status,
6593 )?]);
6594 }
6595 if let Some(removed_ago_ms) =
6596 self.supervisor.removal_tombstone_age_ms(&target_module_id)
6597 {
6598 return Ok(vec![self.route_open_refusal_frame(
6599 ctx,
6600 &caller,
6601 &frame,
6602 &target_module_id,
6603 "removed",
6604 error_codes::MODULE_REMOVED,
6605 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
6606 )?]);
6607 }
6608 return Ok(vec![self.route_open_refusal_frame(
6609 ctx,
6610 &caller,
6611 &frame,
6612 &target_module_id,
6613 "not_registered",
6614 error_codes::UNKNOWN_MODULE,
6615 format!("module_id '{target_module_id}' is not registered"),
6616 )?]);
6617 };
6618
6619 if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
6623 .is_err()
6624 {
6625 self.counters
6626 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
6627 info!(
6628 target: "control",
6629 code = error_codes::MODULE_WARMING,
6630 module_id = ?target_module_id,
6631 connection_id = ctx.connection_id.get(),
6632 principal = %caller.principal,
6633 scope = %caller.scope,
6634 harness = %caller.harness,
6635 reason = "declared_not_ready",
6636 "route.open refused"
6637 );
6638 self.route_outages
6641 .record_not_serving(&target_module_id, "declared_not_ready");
6642 return Ok(vec![control_error_body_frame(
6645 &frame,
6646 ErrorBody {
6647 code: error_codes::MODULE_WARMING.to_string(),
6648 message: format!(
6649 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
6650 ),
6651 detail: self.route_open_recovery_detail(
6652 &target_module_id,
6653 frame.header.corr,
6654 Some(serde_json::json!({ "reason": "declared_not_ready" })),
6655 Some(true),
6656 )?,
6657 },
6658 )?]);
6659 }
6660
6661 if let Some(capability) = self.first_unprovided_required_capability(registration) {
6680 self.counters
6681 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
6682 info!(
6683 target: "control",
6684 code = error_codes::MODULE_WARMING,
6685 module_id = ?target_module_id,
6686 connection_id = ctx.connection_id.get(),
6687 principal = %caller.principal,
6688 scope = %caller.scope,
6689 harness = %caller.harness,
6690 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6691 capability = %capability,
6692 "route.open refused"
6693 );
6694 return Ok(vec![control_error_body_frame(
6695 &frame,
6696 ErrorBody {
6697 code: error_codes::MODULE_WARMING.to_string(),
6698 message: format!(
6699 "module_id '{target_module_id}' requires capability '{capability}', \
6700 which no registered module provides; retry"
6701 ),
6702 detail: Some(serde_json::json!({
6703 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6704 "capability": capability,
6705 })),
6706 },
6707 )?]);
6708 }
6709
6710 if !target_has_required_role(&target, ®istration.manifest.provides) {
6711 return Ok(vec![self.route_open_refusal_frame(
6712 ctx,
6713 &caller,
6714 &frame,
6715 &target_module_id,
6716 "role_not_provided",
6717 "target_unavailable",
6718 format!("module_id '{target_module_id}' does not provide the requested target"),
6719 )?]);
6720 }
6721
6722 if registration.state != ChannelState::Active {
6723 return Ok(vec![self.route_open_refusal_frame(
6724 ctx,
6725 &caller,
6726 &frame,
6727 &target_module_id,
6728 "registration_not_active",
6729 "target_unavailable",
6730 format!("module_id '{target_module_id}' is not active"),
6731 )?]);
6732 }
6733
6734 if self
6735 .forwarding
6736 .module_is_draining(&target_module_id)
6737 .map_err(RouterError::Forwarding)?
6738 {
6739 return Ok(vec![self.route_open_refusal_frame(
6740 ctx,
6741 &caller,
6742 &frame,
6743 &target_module_id,
6744 "reloading",
6745 "module_reloading",
6746 format!("module_id '{target_module_id}' is reloading"),
6747 )?]);
6748 }
6749
6750 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
6751 process_liveness.process_live(&target_module_id) == Some(false)
6752 }) {
6753 if process_liveness.process_replacing(&target_module_id) {
6760 return Ok(vec![self.route_open_refusal_frame(
6761 ctx,
6762 &caller,
6763 &frame,
6764 &target_module_id,
6765 "reloading",
6766 "module_reloading",
6767 format!("module_id '{target_module_id}' is reloading"),
6768 )?]);
6769 }
6770 return Ok(vec![self.route_open_refusal_frame(
6771 ctx,
6772 &caller,
6773 &frame,
6774 &target_module_id,
6775 "supervisor_not_live",
6776 "target_unavailable",
6777 format!("module_id '{target_module_id}' is not live"),
6778 )?]);
6779 }
6780
6781 if !self
6782 .forwarding
6783 .has_live_module_connection(&target_module_id)
6784 .map_err(RouterError::Forwarding)?
6785 {
6786 return Ok(vec![self.route_open_refusal_frame(
6787 ctx,
6788 &caller,
6789 &frame,
6790 &target_module_id,
6791 "no_forwarding_connection",
6792 "target_unavailable",
6793 format!("module_id '{target_module_id}' has no live forwarding connection"),
6794 )?]);
6795 }
6796
6797 if let Some(error) =
6798 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
6799 {
6800 self.observe_route_open_refusal(
6801 ctx,
6802 &caller,
6803 &target_module_id,
6804 "op_not_allowed",
6805 "op_not_allowed",
6806 );
6807 return Ok(vec![error]);
6808 }
6809
6810 let principal = match (&attested, consumer_identity.is_some()) {
6811 (None, false) => {
6814 self.observe_route_open_refusal(
6815 ctx,
6816 &caller,
6817 &target_module_id,
6818 "registration_channel_closed",
6819 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6820 );
6821 return Ok(vec![control_error_frame(
6822 &frame,
6823 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6824 "the claim binding's launch has ended",
6825 )?]);
6826 }
6827 _ => match self.route_open_principal_attested(
6828 &frame,
6829 consumer_identity,
6830 attested.as_ref().map(|attested| attested.principal.clone()),
6831 )? {
6832 Ok(principal) => principal,
6833 Err(error) => {
6834 self.observe_route_open_refusal(
6835 ctx,
6836 &caller,
6837 &target_module_id,
6838 "bad_consumer_identity",
6839 "bad_consumer_identity",
6840 );
6841 return Ok(vec![error]);
6842 }
6843 },
6844 };
6845 let attested = attested.expect("an attested principal was resolved above");
6846 timings.principal = principal_label(&principal);
6847
6848 let Ok(mut caller_capture) = attested.capture() else {
6851 self.observe_route_open_refusal(
6852 ctx,
6853 &caller,
6854 &target_module_id,
6855 "registration_channel_closed",
6856 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6857 );
6858 return Ok(vec![control_error_frame(
6859 &frame,
6860 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6861 "the opener's launch has ended",
6862 )?]);
6863 };
6864 let caller_module_id = match &attested.spawn {
6868 Some((record, _)) => Some(record.module_id().to_owned()),
6869 None => match &principal {
6870 Principal::Reserved { module_id } => Some(module_id.clone()),
6871 _ => None,
6872 },
6873 };
6874
6875 let build_class = identity.build_class.take().filter(|build_class| {
6878 if caller_module_id.as_deref() == Some("callosum") {
6879 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
6880 } else {
6881 if self
6882 .build_class_warned
6883 .lock()
6884 .unwrap_or_else(|p| p.into_inner())
6885 .insert(ctx.connection_id)
6886 {
6887 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
6888 "ignoring build_class from a non-Callosum opener");
6889 }
6890 false
6891 }
6892 });
6893
6894 if let Some(opening_registration) = self.caller_registration(&attested, &admission_registry)
6898 {
6899 let opening_module_id = opening_registration.manifest.module_id.as_str();
6900 if let Some(capability) =
6901 denied_capability(&opening_registration.manifest, ®istration.manifest)
6902 {
6903 warn!(
6904 opening_module_id,
6905 target_module_id,
6906 capability,
6907 "refusing route.open because an attested capability deny edge matches"
6908 );
6909 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6910 &frame,
6911 &target_module_id,
6912 "capability_deny_edge",
6913 "capability_forbidden",
6914 format!(
6915 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
6916 ),
6917 )?]);
6918 }
6919 }
6920
6921 if admission_facts.is_some() {
6922 let carrier_matches = caller_module_id.is_some()
6923 && self.admission_facts_carrier_module_id.as_deref() == caller_module_id.as_deref();
6924 if !carrier_matches {
6925 return Ok(vec![self.route_open_refusal_frame(
6926 ctx,
6927 &caller,
6928 &frame,
6929 &target_module_id,
6930 "admission_facts_carrier_not_permitted",
6931 "admission_facts_not_permitted",
6932 "admission facts may only be carried by the configured reserved module",
6933 )?]);
6934 }
6935
6936 let target_allowed = self
6937 .admission_facts_targets
6938 .as_ref()
6939 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
6940 if !target_allowed {
6941 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6942 &frame,
6943 &target_module_id,
6944 "admission_facts_target_not_listed",
6945 "admission_facts_target_not_allowed",
6946 format!(
6947 "admission facts are not permitted for target module_id '{target_module_id}'"
6948 ),
6949 )?]);
6950 }
6951
6952 }
6956
6957 let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
6963 None => (None, None, false),
6964 Some(selector) => {
6965 let owner_configured = match &selector.owner {
6966 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
6972 _ => false,
6973 };
6974 let mut scopes = self
6975 .scopes
6976 .write()
6977 .unwrap_or_else(|poisoned| poisoned.into_inner());
6978 let admitted = crate::scopes::admission::admit_route(
6979 &scopes,
6980 &admission_registry,
6981 &principal,
6982 &target_module_id,
6983 Some(selector),
6984 owner_configured,
6985 crate::scopes::admission::AdmissionMode::Open,
6986 )
6987 .map(|route| {
6988 let admission = route.scope.expect("scoped admission");
6989 let bound = scopes.reserve_bind(
6990 &admission,
6991 &principal,
6992 ctx.connection_id,
6993 selector.hold_presence,
6994 );
6995 (admission, bound)
6996 });
6997 match admitted {
6998 Ok((admission, bound)) => {
6999 (Some(bound), Some(admission.stamp), admission.stamp_required)
7000 }
7001 Err(refusal) => {
7002 return Ok(vec![self.route_open_refusal_frame(
7003 ctx,
7004 &caller,
7005 &frame,
7006 &target_module_id,
7007 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7008 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7009 refusal.message,
7010 )?]);
7011 }
7012 }
7013 }
7014 };
7015 if let Some(stamp) = &scope_stamp {
7018 caller.scope = route_open_scope_label(stamp);
7019 }
7020 if scope_stamp.as_ref().is_some_and(scope_acts_for_another) {
7023 caller_capture = caller_capture.map(crate::spawn_authority::CallerCapture::relayed);
7024 }
7025
7026 timings.enter(RouteOpenPhase::ProjectRoot);
7034 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
7035 Ok(project) => project,
7036 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
7037 };
7038 timings.enter(RouteOpenPhase::Admission);
7039 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
7040 Ok(root) => root,
7041 Err(err) => {
7042 return Ok(vec![control_error_frame(
7043 &frame,
7044 "invalid_project_root",
7045 err.to_string(),
7046 )?])
7047 }
7048 };
7049 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
7055
7056 let _concurrency_guard = match self
7067 .route_bind_concurrency
7068 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
7069 {
7070 Ok(guard) => guard,
7071 Err(in_flight) => {
7072 return Ok(vec![self.route_open_target_capacity_refusal(
7073 ctx,
7074 &caller,
7075 &frame,
7076 &target_module_id,
7077 in_flight,
7078 )?]);
7079 }
7080 };
7081
7082 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
7085 RouteBindAdmission::Admitted { guard, probe } => {
7086 if probe {
7087 info!(
7088 module_id = %target_module_id,
7089 connection_id = ctx.connection_id.get(),
7090 "route.bind breaker half-open: admitting one probe"
7091 );
7092 }
7093 guard
7094 }
7095 RouteBindAdmission::Refused {
7096 consecutive_timeouts,
7097 retry_in,
7098 probe_in_flight,
7099 } => {
7100 return Ok(vec![self.route_open_breaker_refusal_frame(
7101 ctx,
7102 &caller,
7103 &frame,
7104 &target_module_id,
7105 consecutive_timeouts,
7106 retry_in,
7107 probe_in_flight,
7108 )?]);
7109 }
7110 };
7111
7112 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
7117 let relay_deadline = Instant::now() + route_bind_relay_timeout;
7118 let operator_app = self
7119 .operator_apps
7120 .lock()
7121 .unwrap_or_else(|p| p.into_inner())
7122 .get(&ctx.connection_id)
7123 .cloned();
7124 timings.enter(RouteOpenPhase::EgressReservation);
7125 let pending = match self
7126 .forwarding
7127 .begin_route_bind_relay_for(
7128 ctx.connection_id,
7129 ctx.egress.clone(),
7130 response_version(&frame),
7131 frame.header.corr,
7132 &target_module_id,
7133 principal.clone(),
7134 bound_scope,
7135 project_root,
7136 identity.harness.clone(),
7137 build_class,
7138 operator_app.clone(),
7139 crate::forwarding::RouteAuthority {
7140 incarnation: self.supervisor.daemon_incarnation(),
7141 caller: caller_capture
7142 .as_ref()
7143 .map(|caller| caller.authority.pair()),
7144 },
7145 relay_deadline,
7146 busy_refusals == Some(true),
7147 accepts_close_reasons,
7148 )
7149 .await
7150 {
7151 Ok(pending) => pending,
7152 Err(err) => {
7153 return Ok(vec![self.route_open_refusal_frame(
7154 ctx,
7155 &caller,
7156 &frame,
7157 &target_module_id,
7158 "relay_reservation_failed",
7159 forwarding_error_code(&err),
7160 err.to_string(),
7161 )?])
7162 }
7163 };
7164 timings.enter(RouteOpenPhase::Admission);
7165 let crate::forwarding::PendingRouteBindRelay {
7166 endpoint,
7167 module_sink,
7168 negotiated_ver,
7169 client_channel,
7170 client_epoch,
7171 module_channel,
7172 module_epoch,
7173 corr: relay_corr,
7174 receiver,
7175 } = pending;
7176 let mut reservation =
7177 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
7178
7179 let relay_registration = self
7184 .registry
7185 .get_module_by_connection(endpoint.connection_id)
7186 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7187 let relay_capabilities = relay_registration
7188 .as_ref()
7189 .and_then(|registration| registration.manifest.capabilities.as_ref());
7190 if let Some(selector) = &scope {
7191 if let Err(refusal) = self
7194 .scopes
7195 .read()
7196 .unwrap_or_else(|poisoned| poisoned.into_inner())
7197 .check_bind_proof(&principal, selector)
7198 {
7199 reservation.release_and_disarm();
7200 return Ok(vec![self.route_open_refusal_frame(
7201 ctx,
7202 &caller,
7203 &frame,
7204 &target_module_id,
7205 refusal.code,
7206 refusal.code,
7207 refusal.message,
7208 )?]);
7209 }
7210 }
7211 if let Some(stamp) = scope_stamp.as_ref() {
7212 if let Err(refusal) = crate::scopes::check_target_required_stamp(
7213 stamp_required,
7214 &target_module_id,
7215 relay_capabilities,
7216 )
7217 .and_then(|()| {
7218 crate::scopes::check_target_flow_support(
7219 stamp,
7220 &target_module_id,
7221 relay_capabilities,
7222 )
7223 })
7224 .and_then(|()| {
7225 crate::scopes::check_target_agent_run_support(
7226 stamp,
7227 &target_module_id,
7228 relay_capabilities,
7229 )
7230 }) {
7231 reservation.release_and_disarm();
7232 return Ok(vec![self.route_open_refusal_frame(
7233 ctx,
7234 &caller,
7235 &frame,
7236 &target_module_id,
7237 refusal.code,
7238 refusal.code,
7239 refusal.message,
7240 )?]);
7241 }
7242 }
7243
7244 if scope_stamp
7247 .as_ref()
7248 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
7249 && !crate::scopes::target_supports_federated(relay_capabilities)
7250 {
7251 scope_stamp = None;
7253 }
7254 if !relay_capabilities.is_some_and(|caps| {
7255 caps.provides
7256 .iter()
7257 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
7258 }) {
7259 if let Some(stamp) = scope_stamp.as_mut() {
7260 stamp.workdir = None;
7261 stamp.project_id = None;
7262 }
7263 }
7264 let worker_run = scope_stamp
7269 .as_ref()
7270 .is_some_and(crate::scopes::is_worker_run)
7271 && relay_capabilities.is_some_and(|caps| {
7272 caps.provides
7273 .iter()
7274 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
7275 });
7276 if !worker_run
7277 && !relay_capabilities.is_some_and(|caps| {
7278 caps.provides
7279 .iter()
7280 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
7281 })
7282 {
7283 if let Some(stamp) = scope_stamp.as_mut() {
7284 stamp.attributes.on_behalf_of_agent = None;
7285 }
7286 }
7287 if !relay_capabilities.is_some_and(|caps| {
7288 caps.provides
7289 .iter()
7290 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
7291 }) {
7292 if let Some(stamp) = scope_stamp.as_mut() {
7293 stamp.attributes.flow_version = None;
7294 stamp.attributes.flow_code_hash = None;
7295 stamp.attributes.flow_install_id = None;
7296 }
7297 }
7298 let call_mode = if worker_run
7299 || relay_capabilities
7300 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
7301 {
7302 Some(derive_call_mode(scope_stamp.as_ref()))
7303 } else {
7304 if let Some(stamp) = scope_stamp.as_mut() {
7305 stamp.expires_at_ms = None;
7306 }
7307 None
7308 };
7309
7310 debug!(
7311 connection_id = ctx.connection_id.get(),
7312 client_channel,
7313 client_epoch,
7314 module_channel,
7315 module_epoch,
7316 "reserved route handle pair"
7317 );
7318 let route_principal_label = principal_label(&principal);
7321 let relay = ModuleControlRequest::RouteBind {
7322 route_channel: module_channel,
7323 epoch: module_epoch,
7324 target,
7325 identity,
7326 principal: Some(principal),
7327 consumer_capabilities,
7328 role_versions,
7329 admission_facts,
7330 scope: scope_stamp,
7331 call_mode,
7332 };
7333 let peer = RouteBindPeer::new()
7334 .with_observed_peer(
7335 relay_capabilities
7336 .is_some_and(|caps| {
7337 caps.provides
7338 .iter()
7339 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
7340 })
7341 .then(|| {
7342 self.observed_peers
7343 .lock()
7344 .unwrap_or_else(|p| p.into_inner())
7345 .get(&ctx.connection_id)
7346 .cloned()
7347 .unwrap_or_else(ObservedPeer::unobserved)
7348 }),
7349 )
7350 .with_operator_app(
7351 self.connection_operator_app(operator_app.as_ref())
7352 .filter(|_| {
7353 relay_capabilities.is_some_and(|caps| {
7354 caps.provides
7355 .iter()
7356 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
7357 })
7358 }),
7359 )
7360 .with_caller(caller_for_target(
7361 caller_capture.as_ref(),
7362 relay_capabilities,
7363 &target_module_id,
7364 ));
7365 let relay_body = crate::bind_compat::encode_bind(
7366 &relay,
7367 &peer,
7368 &legacy_bind_project,
7369 relay_capabilities,
7370 )
7371 .map_err(|err| {
7372 RouterError::backend(
7373 0,
7374 frame.header.corr,
7375 format!("failed to encode route.bind request: {err}"),
7376 )
7377 })?;
7378 let relay_frame = Frame::build_with_version(
7379 negotiated_ver,
7380 FrameType::Request,
7381 control_flags(),
7382 0,
7383 0,
7384 relay_corr,
7385 relay_body,
7386 )
7387 .map_err(RouterError::FrameBuild)?;
7388
7389 timings.enter(RouteOpenPhase::BindSend);
7390 let sent = module_sink.send(relay_frame).await;
7391 timings.enter(RouteOpenPhase::Admission);
7392 if let Err(err) = sent {
7393 reservation.release_and_disarm();
7394 return Ok(vec![self.route_open_refusal_frame(
7395 ctx,
7396 &caller,
7397 &frame,
7398 &target_module_id,
7399 "relay_send_failed",
7400 "target_unavailable",
7401 err.to_string(),
7402 )?]);
7403 }
7404
7405 #[cfg(test)]
7406 if let Some(hold) = &self.route_bind_relay_sent_hold {
7407 hold.wait().await;
7411 hold.wait().await;
7412 }
7413
7414 if self
7419 .forwarding
7420 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
7421 .map_err(RouterError::Forwarding)?
7422 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
7423 {
7424 self.send_abandoned_route_bind_goodbye(
7425 &module_sink,
7426 negotiated_ver,
7427 module_channel,
7428 module_epoch,
7429 );
7430 }
7431
7432 timings.enter(RouteOpenPhase::BindAckWait);
7433 let outcome = timeout_at(relay_deadline, receiver).await;
7434 timings.enter(RouteOpenPhase::Admission);
7435 match outcome {
7436 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
7437 reservation.disarm();
7438 if breaker.record_accepted() {
7439 info!(
7440 module_id = %target_module_id,
7441 "route.bind breaker closed: the probe was accepted"
7442 );
7443 }
7444 self.observe_route_open_accept(
7445 ctx,
7446 &target_module_id,
7447 &route_principal_label,
7448 &caller.scope,
7449 );
7450 Ok(Vec::new())
7451 }
7452 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
7453 reservation.release_and_disarm();
7454 breaker.record_inconclusive();
7458 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
7459 }
7460 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
7461 reservation.release_and_disarm();
7462 breaker.record_inconclusive();
7463 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
7464 }
7465 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
7466 reservation.release_and_disarm();
7467 breaker.record_inconclusive();
7468 tracing::warn!(
7475 module_id = %target_module_id,
7476 "route.bind relay abandoned: {message}"
7477 );
7478 Ok(vec![self.route_open_refusal_frame(
7479 ctx,
7480 &caller,
7481 &frame,
7482 &target_module_id,
7483 "relay_abandoned",
7484 "target_unavailable",
7485 message,
7486 )?])
7487 }
7488 Ok(Err(_)) => {
7489 reservation.release_and_disarm();
7490 breaker.record_inconclusive();
7491 Ok(vec![self.route_open_refusal_frame(
7492 ctx,
7493 &caller,
7494 &frame,
7495 &target_module_id,
7496 "relay_waiter_canceled",
7497 "target_unavailable",
7498 "route.bind relay waiter was canceled before the module responded",
7499 )?])
7500 }
7501 Err(_) => {
7502 reservation.release_and_disarm();
7503 if let Some(opened) = breaker.record_timeout(
7507 self.route_bind_breaker_threshold,
7508 self.route_bind_breaker_cooldown,
7509 ) {
7510 warn!(
7511 module_id = %target_module_id,
7512 consecutive_timeouts = opened.consecutive_timeouts,
7513 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
7514 reopened_after_probe = opened.reopened_after_probe,
7515 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
7516 );
7517 }
7518 tracing::warn!(
7526 module_id = %target_module_id,
7527 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
7528 "route.bind relay timed out: module did not ack within budget"
7529 );
7530 Ok(vec![self.route_open_refusal_frame(
7531 ctx,
7532 &caller,
7533 &frame,
7534 &target_module_id,
7535 "relay_timed_out",
7536 "module_timeout",
7537 format!(
7538 "module_id '{target_module_id}' did not answer route.bind within {:?}",
7539 route_bind_relay_timeout
7540 ),
7541 )?])
7542 }
7543 }
7544 }
7545
7546 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7547 let response = ClientControlResponse::SupervisorSpawnSnapshot {
7548 snapshot: self.supervisor.spawn_snapshot(),
7549 };
7550 Ok(vec![control_response_body_frame(
7551 &frame,
7552 &response,
7553 "ClientControlResponse::SupervisorSpawnSnapshot",
7554 )?])
7555 }
7556
7557 fn handle_supervisor_spawn_subscribe(
7558 &self,
7559 ctx: &RouteCtx,
7560 frame: Frame,
7561 since: Option<SpawnCursor>,
7562 ) -> Result<Vec<Frame>, RouterError> {
7563 match self.supervisor.subscribe_spawns(
7564 ctx.connection_id,
7565 frame.header.corr,
7566 response_version(&frame),
7567 since,
7568 ctx.egress.clone(),
7569 ) {
7570 Ok(()) => Ok(Vec::new()),
7571 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
7572 Ok(vec![control_error_body_frame(
7573 &frame,
7574 ErrorBody {
7575 code: "spawn_cursor_incarnation_mismatch".to_string(),
7576 message: "spawn cursor belongs to a different daemon incarnation"
7577 .to_string(),
7578 detail: Some(serde_json::json!({
7579 "current_daemon_incarnation": current
7580 })),
7581 },
7582 )?])
7583 }
7584 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
7585 &frame,
7586 ErrorBody {
7587 code: "spawn_cursor_too_old".to_string(),
7588 message: "spawn cursor predates the retained event ring".to_string(),
7589 detail: Some(serde_json::json!({
7590 "oldest_retained_cursor": oldest
7591 })),
7592 },
7593 )?]),
7594 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
7595 0,
7596 frame.header.corr,
7597 format!("failed to open supervisor spawn subscription: {error}"),
7598 )),
7599 }
7600 }
7601
7602 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7603 let handler = self.clone();
7606 let runtime = tokio::runtime::Handle::current();
7607 let subscriber = tracing::dispatcher::get_default(Clone::clone);
7610 let corr = frame.header.corr;
7611 tokio::task::spawn_blocking(move || {
7612 runtime.block_on(
7613 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
7614 .with_subscriber(subscriber),
7615 )
7616 })
7617 .await
7618 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
7619 }
7620
7621 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7622 use crate::config_gate::runtime::ApprovedDaemonView;
7623 let publication = self
7624 .config_runtime
7625 .as_ref()
7626 .and_then(|owner| owner.polled_snapshot().ok());
7627 let approved = publication
7628 .as_ref()
7629 .map(|view| view.approved_daemon.clone())
7630 .unwrap_or_else(|| {
7631 if self.config_runtime.is_some() {
7632 ApprovedDaemonView::Unavailable
7633 } else {
7634 ApprovedDaemonView::NotStamped
7635 }
7636 });
7637 let generation = self
7638 .registry
7639 .generation()
7640 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7641 let supervised = self.supervisor.list();
7642 let mut statuses = BTreeMap::new();
7643 let mut not_running = BTreeSet::new();
7644 for module in &supervised {
7645 let status = module.status_for_control("list").map_err(|err| {
7646 RouterError::backend(
7647 0,
7648 frame.header.corr,
7649 format!("failed to read supervisor status: {err}"),
7650 )
7651 })?;
7652 if !status.enabled || !status.process_alive {
7653 not_running.insert(module.module_id().to_owned());
7654 }
7655 statuses.insert(module.module_id().to_owned(), status);
7656 }
7657 let mut process_resources = BTreeMap::new();
7660 for module in &supervised {
7661 process_resources.insert(
7662 module.module_id().to_owned(),
7663 crate::subtree_resources::read_process(module.clone()).await,
7664 );
7665 }
7666 let mut image_readings = BTreeMap::new();
7669 for module in &supervised {
7670 image_readings.insert(
7671 module.module_id().to_owned(),
7672 module.running_image_agreement().await,
7673 );
7674 }
7675 let readings = self.resource_tracker.list(supervised.clone());
7676 let mut modules = Vec::new();
7677 for module in supervised {
7678 let status = statuses
7679 .remove(module.module_id())
7680 .expect("each supervised module has a status snapshot");
7681 let (configured, _) = module.configuration().map_err(|err| {
7682 RouterError::backend(
7683 0,
7684 frame.header.corr,
7685 format!("failed to read module configuration: {err}"),
7686 )
7687 })?;
7688 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
7689 RouterError::backend(
7690 0,
7691 frame.header.corr,
7692 format!("failed to read retry time: {err}"),
7693 )
7694 })?;
7695 let image = image_readings
7696 .remove(module.module_id())
7697 .expect("each supervised module has an image snapshot");
7698 let resources = process_resources.remove(module.module_id());
7699 let mut verdict =
7700 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
7701 verdict.spec = Some(launch_spec_agreement(
7702 &configured,
7703 status.process_alive,
7704 status.launch_spec_digest.as_ref(),
7705 ));
7706 let pending_reload = Some(verdict);
7707 let pending_entry = match &approved {
7708 ApprovedDaemonView::Approved { config, .. } => config
7709 .modules
7710 .iter()
7711 .find(|m| m.module_id == module.module_id()),
7712 _ => None,
7713 };
7714 let pending = self
7715 .pending_module_fields(
7716 &module,
7717 pending_entry,
7718 false,
7719 Self::registration_channel_in_force(
7720 publication
7721 .as_ref()
7722 .and_then(|view| view.registration_declaration(module.module_id()))
7723 .as_deref(),
7724 pending_entry.map(|entry| entry.registration_channel),
7725 ),
7726 )
7727 .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
7728 let script = configured.script.as_ref().map(|entry| {
7729 let approved = self
7730 .config_runtime
7731 .as_ref()
7732 .and_then(|owner| owner.approved_script(&status.module_id, &entry.path));
7733 crate::supervise::script::status(entry, approved.as_ref())
7734 });
7735 let configured_enabled = match &approved {
7736 ApprovedDaemonView::Approved {
7737 config: approved, ..
7738 } => approved
7739 .modules
7740 .iter()
7741 .find(|module| module.module_id == status.module_id)
7742 .map(|module| module.enabled),
7743 ApprovedDaemonView::NotStamped => self
7744 .capability_evaluator
7745 .configured_enabled(&status.module_id),
7746 ApprovedDaemonView::Unavailable => None,
7747 };
7748 let admission_refused = status
7749 .admission_refused
7750 .clone()
7751 .filter(|_| status.enabled && !status.process_alive)
7752 .filter(|_| {
7753 publication
7754 .as_ref()
7755 .and_then(|view| view.registration_declaration(&status.module_id))
7756 .as_deref()
7757 == Some("undeclared")
7758 });
7759 let mut entry = SupervisorEntry::new(
7762 status.module_id,
7763 status.state.to_string(),
7764 status.enabled,
7765 status.live,
7766 status.health.status,
7767 )
7768 .with_configured_enabled(configured_enabled)
7769 .with_script(script)
7770 .with_admission_refused(admission_refused)
7771 .with_next_retry_at_ms(next_retry_at_ms)
7772 .with_launch_nonce_env(Some(!cfg!(unix)))
7773 .with_launch_nonce_source(status.launch_nonce_source)
7774 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
7775 .with_protocol(status.protocol)
7776 .with_pending_reload(pending_reload)
7777 .with_last_probe_ms(status.health.last_probe_ms)
7778 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
7779 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
7780 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
7781 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
7782 .with_restart_count(Some(status.restart_count))
7783 .with_max_restarts(Some(status.max_restarts))
7784 .with_lifetime_restarts(Some(status.lifetime_restarts))
7785 .with_spawn_generation(Some(status.spawn_generation))
7786 .with_restart_window_secs(Some(status.restart_window.as_secs()))
7787 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
7788 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
7789 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
7790 .with_resources(resources);
7791 entry.pending = pending;
7792 modules.push(entry);
7793 }
7794 for module in &mut modules {
7795 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
7796 module.tree = tree.clone();
7797 module.cpu_last_hour = hour.clone();
7798 }
7799 module.tree_unavailable = readings
7800 .reasons
7801 .get(&module.module_id)
7802 .map(|reason| (*reason).to_owned());
7803 if module.tree.is_none() {
7804 module.tree_unavailable = Some(
7805 if not_running.contains(&module.module_id) {
7806 "not_running"
7807 } else if let Some(reason) = readings.reasons.get(&module.module_id) {
7808 reason
7809 } else {
7810 "not_yet_measured"
7811 }
7812 .to_owned(),
7813 );
7814 }
7815 }
7816 let response = ClientControlResponse::SupervisorList {
7817 generation,
7818 modules,
7819 daemon: readings.daemon,
7820 agent_commands: readings.agent_commands,
7821 };
7822 Ok(vec![control_response_body_frame(
7823 &frame,
7824 &response,
7825 "ClientControlResponse::SupervisorList",
7826 )?])
7827 }
7828
7829 fn handle_supervisor_stderr_tail(
7830 &self,
7831 frame: Frame,
7832 module_id: String,
7833 max_lines: Option<u32>,
7834 max_bytes: Option<u32>,
7835 ) -> Result<Vec<Frame>, RouterError> {
7836 let Some(module) = self.supervisor.get(&module_id) else {
7837 return Ok(vec![control_error_frame(
7838 &frame,
7839 "unknown_module",
7840 format!("module_id '{module_id}' is not supervised"),
7841 )?]);
7842 };
7843
7844 let snapshot = module.stderr_tail(
7845 max_lines.map(|value| value as usize),
7846 max_bytes.map(|value| value as usize),
7847 );
7848
7849 let response = ClientControlResponse::SupervisorStderrTail {
7850 module_id,
7851 tail: StderrTail {
7852 capture: match snapshot.capture {
7853 CaptureState::Captured => StderrCaptureState::Captured,
7854 CaptureState::Incomplete { reason } => {
7855 StderrCaptureState::Incomplete { reason }
7856 }
7857 CaptureState::NotCaptured { reason } => {
7858 StderrCaptureState::NotCaptured { reason }
7859 }
7860 },
7861 entries: snapshot
7862 .entries
7863 .into_iter()
7864 .map(|entry| match entry {
7865 TailEntry::Line {
7866 text,
7867 truncated,
7868 at_ms,
7869 } => StderrTailEntry::Line {
7870 text,
7871 truncated,
7872 at_ms,
7873 },
7874 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
7875 })
7876 .collect(),
7877 dropped_lines: snapshot.dropped_lines,
7878 },
7879 };
7880 Ok(vec![control_response_body_frame(
7881 &frame,
7882 &response,
7883 "ClientControlResponse::SupervisorStderrTail",
7884 )?])
7885 }
7886
7887 async fn observe_control_wait<F: std::future::Future>(
7888 &self,
7889 ctx: &RouteCtx,
7890 target: Option<&str>,
7891 kind: &'static str,
7892 future: F,
7893 ) -> F::Output {
7894 self.counters
7895 .observe_reader_wait(
7896 || {
7897 let module = self
7898 .forwarding
7899 .module_id_for_connection(ctx.connection_id)
7900 .ok()
7901 .flatten();
7902 let principal = if module.is_some() { "module" } else { "direct" };
7903 (
7904 self.resolve_reader_wait_target(kind, target)
7905 .or(module)
7906 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
7907 principal.into(),
7908 )
7909 },
7910 kind,
7911 future,
7912 )
7913 .await
7914 }
7915
7916 pub(crate) fn resolve_reader_wait_target(
7919 &self,
7920 op: &str,
7921 selector: Option<&str>,
7922 ) -> Option<String> {
7923 let selector = selector?;
7924 match op {
7925 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
7926 .supervisor
7927 .get(selector)
7928 .map(|module| module.module_id().to_owned()),
7929 ops::SUPERVISOR_HEALTH_PROBE => self
7930 .registry
7931 .get_module(selector)
7932 .ok()
7933 .flatten()
7934 .map(|registration| registration.manifest.module_id),
7935 _ => None,
7936 }
7937 }
7938
7939 async fn handle_supervisor_terminals(
7940 &self,
7941 ctx: &RouteCtx,
7942 frame: Frame,
7943 module_id: String,
7944 ) -> Result<Vec<Frame>, RouterError> {
7945 let Some(module) = self.supervisor.get(&module_id) else {
7946 return Ok(vec![control_error_frame(
7947 &frame,
7948 "unknown_module",
7949 format!("module_id '{module_id}' is not supervised"),
7950 )?]);
7951 };
7952
7953 let terminals = self
7956 .observe_control_wait(
7957 ctx,
7958 Some(module.module_id()),
7959 ops::SUPERVISOR_TERMINALS,
7960 module.read_durable_terminal_history(),
7961 )
7962 .await
7963 .map_err(|error| {
7964 RouterError::backend(
7965 0,
7966 frame.header.corr,
7967 format!("failed to read terminal history: {error}"),
7968 )
7969 })?;
7970 let response = ClientControlResponse::SupervisorTerminals {
7971 module_id,
7972 terminals,
7973 };
7974 Ok(vec![control_response_body_frame(
7975 &frame,
7976 &response,
7977 "ClientControlResponse::SupervisorTerminals",
7978 )?])
7979 }
7980
7981 fn handle_supervisor_routes(
7982 &self,
7983 frame: Frame,
7984 module_id: Option<String>,
7985 ) -> Result<Vec<Frame>, RouterError> {
7986 let modules = self
7987 .forwarding
7988 .route_census(module_id.as_deref())
7989 .map_err(RouterError::Forwarding)?
7990 .into_iter()
7991 .map(|(module_id, routes)| SupervisorRouteModule {
7992 module_id,
7993 routes: routes
7994 .into_iter()
7995 .map(|route| SupervisorRoute {
7996 consumer: match route.principal {
7997 Principal::Reserved { module_id } => {
7998 SupervisorRouteConsumer::Reserved { module_id }
7999 }
8000 Principal::Direct | Principal::Unverified => {
8001 SupervisorRouteConsumer::Direct {
8002 connection_id: route.goodbye_target.connection_id.get(),
8003 }
8004 }
8005 },
8006 age_ms: Instant::now()
8007 .saturating_duration_since(route.bound_at)
8008 .as_millis()
8009 .try_into()
8010 .unwrap_or(u64::MAX),
8011 draining: route.draining,
8012 drain_reason: route.drain_reason,
8013 })
8014 .collect(),
8015 })
8016 .collect();
8017 let response = ClientControlResponse::SupervisorRoutes { modules };
8018 Ok(vec![control_response_body_frame(
8019 &frame,
8020 &response,
8021 "ClientControlResponse::SupervisorRoutes",
8022 )?])
8023 }
8024
8025 fn running_registration_method(
8026 &self,
8027 module: &str,
8028 ) -> Option<subc_control::RegistrationMethod> {
8029 let registration = self.registry.get_module(module).ok()??;
8030 let connection = registration.connection_id;
8031 let channel = self.supervisor.registration_spawn(connection).is_some();
8032 let nonce = if channel {
8033 None
8034 } else {
8035 self.hello_launch_nonces
8036 .lock()
8037 .ok()?
8038 .nonce(connection)
8039 .map(str::to_owned)
8040 };
8041 if !channel && nonce.is_none() {
8042 return None;
8043 }
8044 self.supervisor
8045 .registration_snapshot_is_current_live(
8046 &self.registry,
8047 module,
8048 connection,
8049 nonce.as_deref(),
8050 )
8051 .then_some(if channel {
8052 subc_control::RegistrationMethod::Channel
8053 } else {
8054 subc_control::RegistrationMethod::Nonce
8055 })
8056 }
8057
8058 async fn handle_supervisor_provenance(
8059 &self,
8060 ctx: &RouteCtx,
8061 frame: Frame,
8062 module_id: Option<String>,
8063 ) -> Result<Vec<Frame>, RouterError> {
8064 let targeted = module_id.is_some();
8065 let selected = if let Some(module_id) = module_id {
8066 let Some(module) = self.supervisor.get(&module_id) else {
8067 return Ok(vec![control_error_frame(
8068 &frame,
8069 "unknown_module",
8070 format!("module_id '{module_id}' is not supervised"),
8071 )?]);
8072 };
8073 vec![module]
8074 } else {
8075 self.supervisor.list()
8076 };
8077
8078 let target = targeted.then(|| selected[0].module_id());
8079 let declarations = if let Some(owner) = &self.config_runtime {
8080 owner
8081 .on_blocking(|owner| owner.registration_declarations())
8082 .await
8083 .ok()
8084 } else {
8085 None
8086 };
8087 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
8088 let mut modules = Vec::with_capacity(selected.len());
8089 let mut extensions = Vec::with_capacity(selected.len());
8090 for module in &selected {
8091 let (status, observed_image) = module
8092 .status_and_running_image_agreement()
8093 .await
8094 .map_err(|err| {
8095 RouterError::backend(
8096 0,
8097 frame.header.corr,
8098 format!("failed to read supervisor status: {err}"),
8099 )
8100 })?;
8101 let module_declared = self
8102 .registry
8103 .get_module(&status.module_id)
8104 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
8105 .and_then(|registration| registration.manifest.provenance)
8106 .map(|build| ModuleDeclaredProvenance::Reported { build })
8107 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
8108 #[cfg(test)]
8109 let running_image = match &self.provenance_probe_override {
8110 Some(result) => result.clone(),
8111 None => observed_image,
8112 };
8113 #[cfg(not(test))]
8114 let running_image = observed_image;
8115 let registration = self.running_registration_method(&status.module_id);
8116 let mut extension = subc_control::SupervisorModuleProvenanceExt::new()
8117 .with_registration(registration);
8118 extension.registration_declaration = declarations
8119 .as_ref()
8120 .and_then(|entries| entries.get(&status.module_id))
8121 .cloned();
8122 if status.process_alive && status.registration_declaration.is_some() {
8123 extension.registration_declaration = status.registration_declaration.clone();
8124 }
8125 extensions.push(extension);
8126 modules.push(SupervisorModuleProvenance {
8127 module_id: status.module_id,
8128 module_declared,
8129 daemon_observed: SupervisorObservedProcess {
8130 launch_nonce_source: status.launch_nonce_source,
8131 pid: status.pid,
8132 spawned_at_ms: status.spawned_at_ms,
8133 spawned_from: status.spawned_from,
8134 running_image,
8135 },
8136 });
8137 }
8138 let daemon = SupervisorDaemonProvenance {
8139 daemon_build: self.daemon_provenance.build.clone(),
8140 daemon_observed: DaemonObservedProcess {
8141 pid: self.daemon_provenance.pid,
8142 started_at_ms: self
8143 .daemon_provenance
8144 .start_clock
8145 .map(|clock| clock.started_at_ms())
8146 .or(self.daemon_provenance.started_at_ms),
8147 running_image: self
8148 .daemon_provenance
8149 .probe
8150 .observe(
8151 self.daemon_provenance.pid,
8152 self.daemon_provenance.executable_path.as_deref(),
8153 self.daemon_provenance.executable_identity.clone(),
8154 self.daemon_provenance.process_start_time,
8155 )
8156 .await,
8157 },
8158 };
8159 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
8160 let mut response = serde_json::to_value(response)
8161 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
8162 for (entry, extension) in response["modules"]
8163 .as_array_mut()
8164 .expect("provenance modules are an array")
8165 .iter_mut()
8166 .zip(extensions)
8167 {
8168 extension.write_to(entry).map_err(|error| {
8169 RouterError::backend(0, frame.header.corr, error.to_string())
8170 })?;
8171 }
8172 Ok(vec![control_response_body_frame(
8173 &frame,
8174 &response,
8175 "ClientControlResponse::SupervisorProvenance",
8176 )?])
8177 })
8178 .await
8179 }
8180
8181 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
8182 self.refresh_capability_requirements();
8183 let publication = self
8184 .config_runtime
8185 .as_ref()
8186 .and_then(|owner| owner.polled_snapshot().ok());
8187 let generation = self
8188 .registry
8189 .generation()
8190 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
8191 let modules = self
8192 .supervisor
8193 .list()
8194 .into_iter()
8195 .map(|module| {
8196 let status = module.status_for_control("health").map_err(|err| {
8197 RouterError::backend(
8198 0,
8199 frame.header.corr,
8200 format!("failed to read supervisor health: {err}"),
8201 )
8202 })?;
8203 let module_id = status.module_id;
8204 let capability_detail = self
8205 .capability_evaluator
8206 .required_problem_detail(&module_id);
8207 let declaration_warning = self.registry.get_module(&module_id).ok().flatten()
8208 .filter(|registration| registration.manifest.registration_channel == Some(false))
8209 .filter(|_| publication.as_ref().and_then(|view| view.registration_declaration(&module_id))
8210 .as_deref().map(|next| if status.process_alive { status.registration_declaration.as_deref().unwrap_or(next) } else { next }) == Some("declared"))
8211 .map(|_| format!("{module_id}: HELLO registration_channel is false despite the approved channel declaration"));
8212 let capability_detail = append_capability_problem_detail(capability_detail, declaration_warning);
8213 Ok(SupervisorHealthEntry {
8214 module_id,
8215 status: status.health.status,
8216 detail: append_capability_problem_detail(
8217 status.health.detail,
8218 capability_detail,
8219 ),
8220 metrics: status.health.metrics,
8221 consecutive_failures: status.health.consecutive_failures,
8222 late_answer_count: status.health.late_answer_count,
8223 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
8224 last_action: status.health.last_action,
8225 last_action_ms: status.health.last_action_ms,
8226 last_probe_ms: status.health.last_probe_ms,
8227 })
8228 })
8229 .collect::<Result<Vec<_>, RouterError>>()?;
8230 let response = ClientControlResponse::SupervisorHealth {
8231 generation,
8232 modules,
8233 };
8234 Ok(vec![control_response_body_frame(
8235 &frame,
8236 &response,
8237 "ClientControlResponse::SupervisorHealth",
8238 )?])
8239 }
8240
8241 async fn handle_supervisor_restart(
8242 &self,
8243 frame: Frame,
8244 module_id: String,
8245 drain_timeout_ms: Option<u64>,
8246 requester: Principal,
8247 ) -> Result<Vec<Frame>, RouterError> {
8248 let operation_lock = self.supervisor.operation_lock();
8249 let _operation_guard = operation_lock.lock().await;
8250 let Some(module) = self.supervisor.get(&module_id) else {
8251 return Ok(vec![control_error_frame(
8252 &frame,
8253 "unknown_module",
8254 format!("module_id '{module_id}' is not supervised"),
8255 )?]);
8256 };
8257
8258 self.route_outages.mark_operator_action(&module_id);
8259 if let Err(err) = module
8260 .restart_requested(drain_timeout_ms, Some(requester))
8261 .await
8262 {
8263 self.route_outages
8264 .operator_action_ended_unrefused(&module_id);
8265 let (code, message) = match err {
8266 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8267 crate::supervise::SuperviseError::Disabled { .. } => {
8268 ("module_disabled", err.to_string())
8269 }
8270 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8271 ("swap_in_progress", err.to_string())
8272 }
8273 _ => (
8274 "target_unavailable",
8275 format!("failed to restart module_id '{module_id}': {err}"),
8276 ),
8277 };
8278 return Ok(vec![control_error_frame(&frame, code, message)?]);
8279 }
8280
8281 let response = ClientControlResponse::SupervisorAck {
8282 module_id,
8283 applied: true,
8284 };
8285 Ok(vec![control_response_body_frame(
8286 &frame,
8287 &response,
8288 "ClientControlResponse::SupervisorAck",
8289 )?])
8290 }
8291
8292 async fn handle_supervisor_swap(
8296 &self,
8297 frame: Frame,
8298 module_id: String,
8299 ready_timeout_ms: Option<u64>,
8300 requester: Principal,
8301 ) -> Result<Vec<Frame>, RouterError> {
8302 let module = {
8309 let operation_lock = self.supervisor.operation_lock();
8310 let _operation_guard = operation_lock.lock().await;
8311 self.supervisor.get(&module_id)
8312 };
8313 let Some(module) = module else {
8314 return Ok(vec![control_error_frame(
8315 &frame,
8316 "unknown_module",
8317 format!("module_id '{module_id}' is not supervised"),
8318 )?]);
8319 };
8320
8321 self.route_outages.mark_operator_action(&module_id);
8322 if let Err(err) = module
8323 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
8324 .await
8325 {
8326 self.route_outages
8327 .operator_action_ended_unrefused(&module_id);
8328 use crate::supervise::SuperviseError;
8329 let message = err.to_string();
8330 let error = match err {
8331 SuperviseError::ConfigHeld { .. } => ErrorBody::new("config_gated", message),
8332 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
8333 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
8334 code: "swap_refused".to_string(),
8335 message,
8336 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
8337 },
8338 SuperviseError::SwapFailed {
8339 arm,
8340 candidate_exit,
8341 ..
8342 } => ErrorBody {
8343 code: "swap_failed".to_string(),
8344 message,
8345 detail: Some(serde_json::json!({
8346 "arm": arm.as_str(),
8347 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
8348 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
8349 })),
8350 },
8351 _ => ErrorBody::new(
8352 "target_unavailable",
8353 format!("failed to swap module_id '{module_id}': {message}"),
8354 ),
8355 };
8356 return Ok(vec![control_error_body_frame(&frame, error)?]);
8357 }
8358 self.route_outages
8362 .operator_action_ended_unrefused(&module_id);
8363
8364 let response = ClientControlResponse::SupervisorAck {
8365 module_id,
8366 applied: true,
8367 };
8368 Ok(vec![control_response_body_frame(
8369 &frame,
8370 &response,
8371 "ClientControlResponse::SupervisorAck",
8372 )?])
8373 }
8374
8375 async fn handle_supervisor_reload(
8376 &self,
8377 frame: Frame,
8378 module_id: String,
8379 requester: Principal,
8380 ) -> Result<Vec<Frame>, RouterError> {
8381 let operation_lock = self.supervisor.operation_lock();
8382 let _operation_guard = operation_lock.lock().await;
8383 let Some(module) = self.supervisor.get(&module_id) else {
8384 return Ok(vec![control_error_frame(
8385 &frame,
8386 "unknown_module",
8387 format!("module_id '{module_id}' is not supervised"),
8388 )?]);
8389 };
8390
8391 self.route_outages.mark_operator_action(&module_id);
8392 if let Err(err) = module.reload_requested(Some(requester)).await {
8393 self.route_outages
8394 .operator_action_ended_unrefused(&module_id);
8395 let (code, message) = match err {
8396 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8397 crate::supervise::SuperviseError::Disabled { .. } => {
8398 ("module_disabled", err.to_string())
8399 }
8400 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8401 ("swap_in_progress", err.to_string())
8402 }
8403 _ => (
8404 "reload_failed",
8405 format!("failed to reload module_id '{module_id}': {err}"),
8406 ),
8407 };
8408 return Ok(vec![control_error_frame(&frame, code, message)?]);
8409 }
8410
8411 let response = ClientControlResponse::SupervisorAck {
8412 module_id,
8413 applied: true,
8414 };
8415 Ok(vec![control_response_body_frame(
8416 &frame,
8417 &response,
8418 "ClientControlResponse::SupervisorAck",
8419 )?])
8420 }
8421
8422 async fn handle_supervisor_rescan(
8426 &self,
8427 frame: Frame,
8428 mode: RescanMode,
8429 requester: Principal,
8430 ) -> Result<Vec<Frame>, RouterError> {
8431 let preview = mode.previews();
8432 let mut approved_preview = None;
8433 if matches!(mode, RescanMode::Preview) {
8434 if let Some(owner) = &self.config_runtime {
8435 match owner
8436 .on_blocking(|owner| {
8437 owner.observe_file_proposal()?;
8438 owner.approved_daemon_configuration()
8439 })
8440 .await
8441 {
8442 Ok(config) => approved_preview = config,
8443 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8444 }
8445 }
8446 }
8447 let Some(context) = self.rescan.clone() else {
8448 return Ok(vec![control_error_frame(
8449 &frame,
8450 "rescan_unavailable",
8451 "the daemon was not started with a reloadable config path".to_string(),
8452 )?]);
8453 };
8454
8455 let operation_lock = self.supervisor.operation_lock();
8456 let _operation_guard = operation_lock.lock().await;
8457 let loaded = match self.load_rescan_config(&mode, &context) {
8461 Ok(config) => config,
8462 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
8463 };
8464 let Some((config, activity_config)) = loaded else {
8472 return Ok(vec![control_error_frame(
8473 &frame,
8474 "invalid_daemon_config",
8475 format!(
8476 "daemon config not found at {}; refusing to rescan (an absent config would \
8477 retire every supervised module)",
8478 context.config_path.display()
8479 ),
8480 )?]);
8481 };
8482 let (
8483 configured_port,
8484 storage_config,
8485 admission_facts_carrier_module_id,
8486 admission_facts_targets,
8487 scope_authority_owners,
8488 federated_scope_owner,
8489 privileged_operator_prompt,
8490 modules,
8491 reserved_capabilities,
8492 ) = (
8493 config.port,
8494 config.storage,
8495 config.admission_facts_carrier_module_id,
8496 config.admission_facts_targets,
8497 config.scope_authority_owners,
8498 config.federated_scope_owner,
8499 config.privileged_operator_prompt,
8500 config.modules,
8501 config.reserved_capabilities,
8502 );
8503
8504 let mut restart_required = Vec::new();
8513 for section in RestartRequiredSection::ALL {
8514 let changed = match section {
8515 RestartRequiredSection::Port => configured_port != context.configured_port,
8516 RestartRequiredSection::Storage => storage_config != context.storage_config,
8517 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
8518 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
8519 }
8520 RestartRequiredSection::AdmissionFactsTargets => {
8521 admission_facts_targets != context.admission_facts_targets
8522 }
8523 RestartRequiredSection::ScopeAuthorityOwners => {
8524 scope_authority_owners != context.scope_authority_owners
8525 }
8526 RestartRequiredSection::FederatedScopeOwner => {
8527 federated_scope_owner != context.federated_scope_owner
8528 }
8529 RestartRequiredSection::PrivilegedOperatorPrompt => {
8530 privileged_operator_prompt != context.privileged_operator_prompt
8531 }
8532 };
8533 if changed {
8534 restart_required.push(section.label().to_string());
8535 }
8536 }
8537 if !restart_required.is_empty() {
8538 warn!(
8539 config_path = %context.config_path.display(),
8540 sections = %restart_required.join(", "),
8541 "daemon config changed outside the modules section; restart the daemon to apply those changes"
8542 );
8543 }
8544
8545 for configured in &modules {
8546 if let Err(err) = validate_spec(&configured.module_spec()) {
8547 return Ok(vec![control_error_frame(
8548 &frame,
8549 "invalid_daemon_config",
8550 format!("supervisor rescan rejected daemon config: {err}"),
8551 )?]);
8552 }
8553 }
8554
8555 let configured_capabilities = modules
8556 .iter()
8557 .map(|module| (module.module_id.clone(), module.enabled))
8558 .collect::<Vec<_>>();
8559 let preview_capability_warnings = if preview {
8560 let (_, registrations) = self.runtime_capability_snapshot()?;
8561 let current_modules = self
8562 .supervisor
8563 .list()
8564 .into_iter()
8565 .map(|module| module.module_id().to_string())
8566 .collect::<BTreeSet<_>>();
8567 let resulting_modules = configured_capabilities.clone();
8568 let removed = current_modules
8569 .into_iter()
8570 .filter(|module_id| {
8571 !resulting_modules
8572 .iter()
8573 .any(|(configured_id, _)| configured_id == module_id)
8574 })
8575 .collect::<Vec<_>>();
8576 let disabled = match &mode {
8579 RescanMode::Candidate(_) => resulting_modules
8580 .iter()
8581 .filter(|(_, enabled)| !enabled)
8582 .map(|(module_id, _)| module_id.clone())
8583 .collect::<Vec<_>>(),
8584 _ => Vec::new(),
8585 };
8586 self.capability_evaluator.preview_warnings(
8587 resulting_modules,
8588 &removed,
8589 &disabled,
8590 ®istrations,
8591 )
8592 } else {
8593 Vec::new()
8594 };
8595 let reconciled = match (&mode, approved_preview.as_ref()) {
8596 (RescanMode::Preview, Some(approved)) => Ok((
8597 config_gate::preview_approved_daemon(approved, &modules),
8598 Vec::new(),
8599 )),
8600 (RescanMode::Candidate(candidate), _) => self
8601 .preview_candidate(&context, modules, candidate)
8602 .map(|result| (result, Vec::new())),
8603 _ => {
8604 self.reconcile_supervised_modules(
8605 &context.supervisor,
8606 modules,
8607 preview,
8608 requester.clone(),
8609 )
8610 .await
8611 }
8612 };
8613 let (result, changed) = match reconciled {
8614 Ok(result) => result,
8615 Err(message) => {
8616 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
8617 }
8618 };
8619 if !preview {
8620 if let Some(ledger) = &self.activity {
8621 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
8622 ledger.config_changed(&result.added, &result.removed, &changed, requester);
8623 }
8624 if let Err(code) = self.configure_activity(activity_config).await {
8625 warn!(
8626 code,
8627 "activity retention could not be applied during rescan"
8628 );
8629 }
8630 self.capability_evaluator
8631 .configure(configured_capabilities, reserved_capabilities);
8632 self.capability_evaluator.wake_deadline_loop();
8633 self.refresh_capability_requirements();
8634 }
8635 let mut result = result;
8636 result.restart_required = restart_required;
8637 result.capability_warnings = preview_capability_warnings;
8638 let response = ClientControlResponse::SupervisorRescan { result };
8639 Ok(vec![control_response_body_frame(
8640 &frame,
8641 &response,
8642 "ClientControlResponse::SupervisorRescan",
8643 )?])
8644 }
8645
8646 async fn handle_supervisor_release_reserved(
8647 &self,
8648 frame: Frame,
8649 module_id: String,
8650 ) -> Result<Vec<Frame>, RouterError> {
8651 let Some(context) = self.rescan.clone() else {
8652 return Ok(vec![control_error_frame(
8653 &frame,
8654 "release_unavailable",
8655 "reserved-id release requires a daemon started with a reloadable config path",
8656 )?]);
8657 };
8658 let operation_lock = self.supervisor.operation_lock();
8659 let _operation_guard = operation_lock.lock().await;
8660 let approved = if let Some(owner) = &self.config_runtime {
8661 match owner
8664 .on_blocking(|owner| {
8665 owner.observe_file_proposal()?;
8666 owner.approved_daemon_configuration()
8667 })
8668 .await
8669 {
8670 Ok(config) => config,
8671 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8672 }
8673 } else {
8674 None
8675 };
8676 let loaded = if let Some(approved) = approved {
8677 Ok(Some(approved))
8678 } else {
8679 crate::daemon_config::load(&context.config_path)
8680 };
8681 let loaded = match loaded {
8682 Ok(Some(config)) => config,
8683 Ok(None) => {
8684 return Ok(vec![control_error_frame(
8685 &frame,
8686 "invalid_daemon_config",
8687 format!(
8688 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
8689 context.config_path.display()
8690 ),
8691 )?])
8692 }
8693 Err(err) => {
8694 return Ok(vec![control_error_frame(
8695 &frame,
8696 "invalid_daemon_config",
8697 format!("unable to verify reserved-id release against daemon config: {err}"),
8698 )?])
8699 }
8700 };
8701 if loaded
8702 .modules
8703 .iter()
8704 .any(|configured| configured.module_id == module_id)
8705 {
8706 return Ok(vec![control_error_frame(
8707 &frame,
8708 "reserved_module_configured",
8709 format!(
8710 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
8711 ),
8712 )?]);
8713 }
8714 if !self.supervisor.release_retained_reserved_gate(&module_id) {
8715 return Ok(vec![control_error_frame(
8716 &frame,
8717 "reserved_gate_not_retained",
8718 format!(
8719 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
8720 ),
8721 )?]);
8722 }
8723
8724 let response = ClientControlResponse::SupervisorAck {
8725 module_id,
8726 applied: true,
8727 };
8728 Ok(vec![control_response_body_frame(
8729 &frame,
8730 &response,
8731 "ClientControlResponse::SupervisorAck",
8732 )?])
8733 }
8734
8735 fn plan_rescan(
8739 &self,
8740 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8741 ) -> Result<RescanPlan, String> {
8742 let mut current = BTreeMap::new();
8743 for module in self.supervisor.list() {
8744 let (spec, health) = module.configuration().map_err(|err| {
8745 format!(
8746 "failed to read configuration for module_id '{}': {err}",
8747 module.module_id()
8748 )
8749 })?;
8750 let enabled = module
8751 .status()
8752 .map_err(|err| {
8753 format!(
8754 "failed to read status for module_id '{}': {err}",
8755 module.module_id()
8756 )
8757 })?
8758 .enabled;
8759 current.insert(
8760 module.module_id().to_string(),
8761 (module, spec, health, enabled),
8762 );
8763 }
8764 let configured = configured_modules
8765 .into_iter()
8766 .map(|module| (module.module_id.clone(), module))
8767 .collect::<BTreeMap<_, _>>();
8768
8769 let added = configured
8770 .keys()
8771 .filter(|module_id| !current.contains_key(*module_id))
8772 .cloned()
8773 .collect::<Vec<_>>();
8774 let removed = current
8775 .keys()
8776 .filter(|module_id| !configured.contains_key(*module_id))
8777 .cloned()
8778 .collect::<Vec<_>>();
8779 let mut changed_pending_reload = Vec::new();
8780 let mut changed_pending = Vec::new();
8781 let mut configuration_changes = BTreeSet::new();
8782 let mut enabled_changes = BTreeSet::new();
8783 let mut unchanged = 0_u32;
8784 let publication = self
8785 .config_runtime
8786 .as_ref()
8787 .and_then(|owner| owner.polled_snapshot().ok());
8788
8789 for (module_id, configured_module) in &configured {
8790 let Some((module, current_spec, current_health, current_enabled)) =
8791 current.get(module_id)
8792 else {
8793 continue;
8794 };
8795 let launch_changed = *current_spec != configured_module.module_spec();
8799 let configuration_changed =
8800 launch_changed || *current_health != configured_module.health;
8801 let pending = self.pending_module_fields(
8802 module,
8803 Some(configured_module),
8804 configuration_changed,
8805 Self::registration_channel_in_force(
8806 publication
8807 .as_ref()
8808 .and_then(|view| view.registration_declaration(module_id))
8809 .as_deref(),
8810 publication
8811 .as_ref()
8812 .and_then(|view| match &view.approved_daemon {
8813 crate::config_gate::runtime::ApprovedDaemonView::Approved {
8814 config,
8815 ..
8816 } => config
8817 .modules
8818 .iter()
8819 .find(|entry| entry.module_id == *module_id)
8820 .map(|entry| entry.registration_channel),
8821 _ => None,
8822 }),
8823 ),
8824 )?;
8825 let has_pending = !pending.is_empty();
8826 if has_pending {
8827 changed_pending.push(subc_control::SupervisorPendingModule {
8828 module_id: module_id.clone(),
8829 fields: pending
8830 .into_iter()
8831 .map(|mut field| {
8832 field.value = None;
8833 field
8834 })
8835 .collect(),
8836 });
8837 }
8838 let enabled_changed = *current_enabled != configured_module.enabled
8841 || self
8842 .capability_evaluator
8843 .configured_enabled(module_id)
8844 .is_some_and(|enabled| enabled != configured_module.enabled);
8845 if configuration_changed {
8846 configuration_changes.insert(module_id.clone());
8847 }
8848 if launch_changed {
8849 changed_pending_reload.push(module_id.clone());
8850 }
8851 if enabled_changed {
8852 enabled_changes.insert(module_id.clone());
8853 }
8854 if !configuration_changed && !enabled_changed && !has_pending {
8855 unchanged = unchanged.saturating_add(1);
8856 }
8857 }
8858 Ok(RescanPlan {
8859 current,
8860 configured,
8861 added,
8862 removed,
8863 changed_pending_reload,
8864 changed_pending,
8865 configuration_changes,
8866 enabled_changes,
8867 unchanged,
8868 })
8869 }
8870
8871 async fn reconcile_supervised_modules(
8880 &self,
8881 supervisor: &Supervisor,
8882 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8883 preview: bool,
8884 requester: Principal,
8885 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
8886 let plan = self.plan_rescan(configured_modules)?;
8887
8888 if preview {
8894 return Ok((plan.preview_result(), Vec::new()));
8895 }
8896 let changed = plan
8897 .configuration_changes
8898 .union(&plan.enabled_changes)
8899 .cloned()
8900 .collect::<Vec<_>>();
8901 let RescanPlan {
8902 current,
8903 configured,
8904 added,
8905 removed,
8906 changed_pending_reload,
8907 changed_pending,
8908 configuration_changes,
8909 enabled_changes,
8910 unchanged,
8911 } = plan;
8912 if !(added.is_empty()
8917 && removed.is_empty()
8918 && configuration_changes.is_empty()
8919 && changed_pending.is_empty()
8920 && enabled_changes.is_empty())
8921 {
8922 if let Some(context) = &self.rescan {
8923 context
8924 .generation
8925 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8926 }
8927 }
8928
8929 for module_id in &removed {
8930 let module = ¤t
8931 .get(module_id)
8932 .expect("removed module came from current supervisor state")
8933 .0;
8934 module
8935 .retire_requested(Some(requester.clone()))
8936 .await
8937 .map_err(|err| {
8938 format!("failed to retire module_id '{module_id}' during rescan: {err}")
8939 })?;
8940 self.supervisor.record_rescan_removal(module_id);
8974 self.supervisor.retire(module_id);
8975 self.route_outages.forget(module_id);
8976 }
8977
8978 for module_id in configured.keys() {
8979 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
8980 continue;
8981 };
8982 let configured_module = configured
8983 .get(module_id)
8984 .expect("configured module id came from configured map");
8985 if configuration_changes.contains(module_id) {
8986 module
8987 .update_configuration(
8988 configured_module.module_spec(),
8989 configured_module.health.clone(),
8990 configured_module.drain_timeout_ms,
8991 )
8992 .await
8993 .map_err(|err| {
8994 format!(
8995 "failed to update module_id '{module_id}' configuration during rescan: {err}"
8996 )
8997 })?;
8998 }
8999 if *current_enabled != configured_module.enabled {
9002 self.route_outages.mark_operator_action(module_id);
9005 module
9006 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
9007 .await
9008 .map_err(|err| {
9009 self.route_outages.operator_action_ended_unrefused(module_id);
9010 format!(
9011 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
9012 configured_module.enabled
9013 )
9014 })?;
9015 }
9016 }
9017
9018 for module_id in &added {
9019 let configured_module = configured
9020 .get(module_id)
9021 .expect("added module id came from configured map");
9022 supervisor
9023 .supervise_configured_requested_async(
9024 configured_module.module_spec(),
9025 configured_module.enabled,
9026 configured_module.health.clone(),
9027 configured_module.drain_timeout_ms,
9028 configured_module.restart,
9029 configured_module.register_timeout,
9030 Some(requester.clone()),
9031 )
9032 .await
9033 .map_err(|err| {
9034 format!("failed to add module_id '{module_id}' during rescan: {err}")
9035 })?;
9036 }
9037
9038 if let Some(context) = &self.rescan {
9039 let mut entries = context
9040 .module_entries
9041 .lock()
9042 .unwrap_or_else(|e| e.into_inner());
9043 entries.record(&configured, &added, &removed);
9044 }
9045
9046 Ok((
9047 SupervisorRescanResult {
9048 added,
9049 removed,
9050 changed_pending_reload,
9051 changed_pending,
9052 enabled_changes: enabled_changes.iter().cloned().collect(),
9053 unchanged,
9054 preview: false,
9055 restart_required: Vec::new(),
9058 capability_warnings: Vec::new(),
9059 module_actions: Vec::new(),
9060 earlier_pending: Vec::new(),
9061 expect: None,
9062 },
9063 changed,
9064 ))
9065 }
9066
9067 async fn handle_supervisor_set_enabled(
9068 &self,
9069 frame: Frame,
9070 module_id: String,
9071 enabled: bool,
9072 requester: Principal,
9073 ) -> Result<Vec<Frame>, RouterError> {
9074 let operation_lock = self.supervisor.operation_lock();
9075 let _operation_guard = operation_lock.lock().await;
9076 let Some(module) = self.supervisor.get(&module_id) else {
9077 return Ok(vec![control_error_frame(
9078 &frame,
9079 "unknown_module",
9080 format!("module_id '{module_id}' is not supervised"),
9081 )?]);
9082 };
9083
9084 self.route_outages.mark_operator_action(&module_id);
9087 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
9088 Ok(applied) => applied,
9089 Err(err) => {
9090 self.route_outages
9091 .operator_action_ended_unrefused(&module_id);
9092 return Ok(vec![control_error_frame(
9093 &frame,
9094 if err.is_config_gate_error() {
9095 "config_gated"
9096 } else {
9097 "target_unavailable"
9098 },
9099 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
9100 )?]);
9101 }
9102 };
9103 if !applied {
9104 self.route_outages
9107 .operator_action_ended_unrefused(&module_id);
9108 }
9109
9110 self.capability_evaluator.wake_deadline_loop();
9111 self.refresh_capability_requirements();
9112 let response = ClientControlResponse::SupervisorAck { module_id, applied };
9113 Ok(vec![control_response_body_frame(
9114 &frame,
9115 &response,
9116 "ClientControlResponse::SupervisorAck",
9117 )?])
9118 }
9119
9120 async fn handle_supervisor_health_probe(
9121 &self,
9122 ctx: &RouteCtx,
9123 frame: Frame,
9124 module_id: String,
9125 ) -> Result<Vec<Frame>, RouterError> {
9126 self.refresh_capability_requirements();
9127 let Some(registration) = self
9128 .registry
9129 .get_module(&module_id)
9130 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
9131 else {
9132 return Ok(vec![control_error_frame(
9133 &frame,
9134 "unknown_module",
9135 format!("module_id '{module_id}' is not registered"),
9136 )?]);
9137 };
9138
9139 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
9152 {
9153 return Ok(vec![control_error_frame(
9154 &frame,
9155 "health_not_advertised",
9156 format!("module_id '{module_id}' did not advertise health.check"),
9157 )?]);
9158 }
9159
9160 self.observe_control_wait(
9161 ctx,
9162 Some(®istration.manifest.module_id),
9163 ops::SUPERVISOR_HEALTH_PROBE,
9164 async {
9165 let deadline = Instant::now() + self.health_probe_timeout;
9166 let pending = match self.forwarding.begin_module_control_rpc_for(
9167 &module_id,
9168 MODULE_CONTROL_OP_HEALTH_CHECK,
9169 deadline,
9170 ) {
9171 Ok(pending) => pending,
9172 Err(err) => {
9173 return Ok(vec![control_error_frame(
9174 &frame,
9175 forwarding_error_code(&err),
9176 err.to_string(),
9177 )?])
9178 }
9179 };
9180
9181 let PendingModuleControlRpc {
9182 endpoint,
9183 module_sink,
9184 negotiated_ver,
9185 corr: probe_corr,
9186 receiver,
9187 } = pending;
9188 let mut guard =
9189 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
9190 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
9191 .map_err(|err| {
9192 RouterError::backend(
9193 0,
9194 frame.header.corr,
9195 format!("failed to encode health.check request: {err}"),
9196 )
9197 })?;
9198 let probe_frame = Frame::build_with_version(
9199 negotiated_ver,
9200 FrameType::Request,
9201 control_flags(),
9202 0,
9203 0,
9204 probe_corr,
9205 probe_body,
9206 )
9207 .map_err(RouterError::FrameBuild)?;
9208
9209 match timeout_at(deadline, module_sink.send(probe_frame)).await {
9212 Ok(Ok(())) => {}
9213 Ok(Err(err)) => {
9214 return Ok(vec![control_error_frame(
9215 &frame,
9216 "target_unavailable",
9217 err.to_string(),
9218 )?]);
9219 }
9220 Err(_) => {
9221 return Ok(vec![control_error_frame(
9222 &frame,
9223 "module_timeout",
9224 format!(
9225 "module_id '{module_id}' did not answer health.check within {:?}",
9226 self.health_probe_timeout
9227 ),
9228 )?]);
9229 }
9230 }
9231
9232 match timeout_at(deadline, receiver).await {
9233 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => {
9234 Ok(vec![control_error_frame(
9235 &frame,
9236 "invalid_control_body",
9237 "health.check returned an unexpected config response",
9238 )?])
9239 }
9240 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
9241 guard.disarm();
9242 let Some(report) = response.health_report() else {
9243 return Ok(vec![control_error_frame(
9244 &frame,
9245 "invalid_control_body",
9246 "health.check RPC returned a non-health response",
9247 )?]);
9248 };
9249 let HealthReport {
9254 status,
9255 detail,
9256 metrics,
9257 } = report;
9258 let capability_detail = self
9259 .capability_evaluator
9260 .required_problem_detail(&module_id);
9261 let response = ClientControlResponse::SupervisorHealthProbe {
9262 module_id,
9263 status,
9264 detail: append_capability_problem_detail(detail, capability_detail),
9265 metrics,
9266 };
9267 Ok(vec![control_response_body_frame(
9268 &frame,
9269 &response,
9270 "ClientControlResponse::SupervisorHealthProbe",
9271 )?])
9272 }
9273 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
9274 guard.disarm();
9275 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
9280 }
9281 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
9282 guard.disarm();
9283 Ok(vec![control_error_frame(
9284 &frame,
9285 "target_unavailable",
9286 message,
9287 )?])
9288 }
9289 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
9290 guard.disarm();
9291 Ok(vec![control_error_frame(
9292 &frame,
9293 "invalid_control_body",
9294 message,
9295 )?])
9296 }
9297 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
9298 guard.disarm();
9299 Ok(vec![control_error_frame(
9300 &frame,
9301 "invalid_control_body",
9302 format!("expected module-control op '{expected}', got '{actual}'"),
9303 )?])
9304 }
9305 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
9306 guard.disarm();
9307 Ok(vec![control_error_frame(
9308 &frame,
9309 "module_timeout",
9310 format!(
9311 "module_id '{module_id}' answered health.check after {:?}",
9312 self.health_probe_timeout
9313 ),
9314 )?])
9315 }
9316 Ok(Err(_)) => Ok(vec![control_error_frame(
9317 &frame,
9318 "target_unavailable",
9319 "health.check waiter was canceled before the module responded",
9320 )?]),
9321 Err(_) => Ok(vec![control_error_frame(
9322 &frame,
9323 "module_timeout",
9324 format!(
9325 "module_id '{module_id}' did not answer health.check within {:?}",
9326 self.health_probe_timeout
9327 ),
9328 )?]),
9329 }
9330 },
9331 )
9332 .await
9333 }
9334
9335 fn supervisor_status(
9336 &self,
9337 module_id: &str,
9338 corr: u64,
9339 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
9340 self.supervisor
9341 .get(module_id)
9342 .map(|module| {
9343 let warming = module.is_warming_for_control("status").map_err(|err| {
9344 RouterError::backend(
9345 0,
9346 corr,
9347 format!(
9348 "failed to read supervisor warming state for module_id '{module_id}': {err}"
9349 ),
9350 )
9351 })?;
9352 module.status_for_control("status").map_err(|err| {
9353 RouterError::backend(
9354 0,
9355 corr,
9356 format!(
9357 "failed to read supervisor status for module_id '{module_id}': {err}"
9358 ),
9359 )
9360 }).map(|status| (status, warming))
9361 })
9362 .transpose()
9363 }
9364
9365 fn guard_module_control_op(
9366 &self,
9367 frame: &Frame,
9368 module_id: &str,
9369 op: &str,
9370 ) -> Result<Option<Frame>, RouterError> {
9371 if self.module_grants_op(module_id, op, frame.header.corr)? {
9372 return Ok(None);
9373 }
9374
9375 Ok(Some(control_error_frame(
9376 frame,
9377 "op_not_allowed",
9378 format!("module_id '{module_id}' did not grant control op '{op}'"),
9379 )?))
9380 }
9381
9382 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
9383 let Some(registration) = self
9384 .registry
9385 .get_module(module_id)
9386 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
9387 else {
9388 return Ok(false);
9389 };
9390 Ok(module_registration_grants_op(®istration.control_ops, op))
9391 }
9392
9393 fn handle_status_update(
9394 &self,
9395 endpoint: ModuleEndpointId,
9396 frame: Frame,
9397 ) -> Result<Vec<Frame>, RouterError> {
9398 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
9399 Ok(update) => update,
9400 Err(err) => {
9401 if is_known_module_push_op(&frame.body) {
9406 return Ok(vec![control_error_frame(
9407 &frame,
9408 "invalid_control_body",
9409 format!("malformed module control push body: {err}"),
9410 )?]);
9411 }
9412 return Ok(Vec::new());
9413 }
9414 };
9415
9416 match update {
9417 ModuleControlPush::RouteStatus {
9418 route_channel,
9419 route_epoch,
9420 status,
9421 } => {
9422 self.forwarding
9423 .cache_status(endpoint, route_channel, route_epoch, status)
9424 .map_err(RouterError::Forwarding)?;
9425 }
9426 }
9427 Ok(Vec::new())
9428 }
9429
9430 fn handle_route_poll(
9431 &self,
9432 ctx: &RouteCtx,
9433 frame: Frame,
9434 route_channel: u16,
9435 route_epoch: u32,
9436 kind: PollKind,
9437 ) -> Result<Vec<Frame>, RouterError> {
9438 let snapshot = self
9439 .forwarding
9440 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
9441 .map_err(RouterError::Forwarding)?;
9442 let response = match (kind, snapshot) {
9443 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
9444 ClientControlResponse::RoutePoll {
9445 route_channel,
9446 route_epoch,
9447 status,
9448 live: None,
9449 }
9450 }
9451 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9452 route_channel,
9453 route_epoch,
9454 status: None,
9455 live: None,
9456 },
9457 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
9458 let live = self
9484 .process_liveness
9485 .as_ref()
9486 .and_then(|source| source.process_live(&module_id))
9487 .unwrap_or(true);
9488 ClientControlResponse::RoutePoll {
9489 route_channel,
9490 route_epoch,
9491 status: None,
9492 live: Some(live),
9493 }
9494 }
9495 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9496 route_channel,
9497 route_epoch,
9498 status: None,
9499 live: Some(false),
9500 },
9501 };
9502
9503 Ok(vec![control_response_body_frame(
9504 &frame,
9505 &response,
9506 "ClientControlResponse::RoutePoll",
9507 )?])
9508 }
9509
9510 pub(crate) fn observe_module_control_completion(
9511 &self,
9512 completion: ModuleControlRpcCompletion,
9513 ) -> bool {
9514 match completion {
9515 ModuleControlRpcCompletion::Unknown => false,
9516 ModuleControlRpcCompletion::Settled => true,
9517 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
9518 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
9519 info!(
9520 module_id = %module_id,
9521 latency_ms,
9522 "late health.check answer proves the module is alive"
9523 );
9524 match self
9525 .supervisor
9526 .record_late_health_answer(&module_id, latency_ms)
9527 {
9528 Ok(true) => {}
9529 Ok(false) => debug!(
9530 module_id = %module_id,
9531 latency_ms,
9532 "late health.check answer has no active supervisor snapshot"
9533 ),
9534 Err(err) => warn!(
9535 module_id = %module_id,
9536 latency_ms,
9537 error = %err,
9538 "failed to record late health.check answer"
9539 ),
9540 }
9541 true
9542 }
9543 }
9544 }
9545
9546 fn refuse_to_end_module_connection_for_a_client(
9567 &self,
9568 module_connection_id: ConnectionId,
9569 corr: u64,
9570 err: ForwardingError,
9571 ) -> Result<(), RouterError> {
9572 if let ForwardingError::ConnectionClosing { connection_id } = err {
9573 if connection_id != module_connection_id {
9574 warn!(
9575 module_connection_id = module_connection_id.get(),
9576 client_connection_id = connection_id.get(),
9577 corr,
9578 "dropping a route.bind response for a closing client; the module connection keeps serving"
9579 );
9580 return Ok(());
9581 }
9582 }
9583 Err(RouterError::Forwarding(err))
9584 }
9585
9586 fn handle_module_relay_response(
9587 &self,
9588 connection_id: ConnectionId,
9589 frame: Frame,
9590 ) -> Result<Vec<Frame>, RouterError> {
9591 let mut secondary_error = None;
9592 let outcome = match frame.header.ty {
9593 FrameType::Response => match module_control_response_probe(&frame.body) {
9594 Ok(probe) if probe.op == "route.bind" => {
9595 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9596 Ok(ModuleControlResponse::RouteBindAck {}) => {
9597 RouteBindRelayOutcome::Accepted
9598 }
9599 Ok(other) => {
9600 let message =
9601 format!("route.bind response carried unexpected body: {other:?}");
9602 secondary_error = Some(control_error_frame(
9603 &frame,
9604 "invalid_control_body",
9605 message.clone(),
9606 )?);
9607 RouteBindRelayOutcome::ModuleGone(message)
9608 }
9609 Err(err) => {
9610 let message = format!("malformed route.bind response body: {err}");
9611 secondary_error = Some(control_error_frame(
9612 &frame,
9613 "invalid_control_body",
9614 message.clone(),
9615 )?);
9616 RouteBindRelayOutcome::ModuleGone(message)
9617 }
9618 }
9619 }
9620 Ok(probe) => {
9621 let outcome = if matches!(
9622 probe.op.as_str(),
9623 "config.review" | "consent/v1" | "projects.enumerate"
9624 ) {
9625 ModuleControlRpcOutcome::RawResponse(frame.body.clone())
9626 } else {
9627 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9628 Ok(response) => ModuleControlRpcOutcome::Response(response),
9629 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9630 "malformed {} response body: {err}",
9631 probe.op
9632 )),
9633 }
9634 };
9635 let completion = self
9636 .forwarding
9637 .complete_module_control_rpc(
9638 connection_id,
9639 frame.header.corr,
9640 Some(&probe.op),
9641 outcome,
9642 )
9643 .map_err(RouterError::Forwarding)?;
9644 if !self.observe_module_control_completion(completion) {
9645 debug!(
9646 connection_id = connection_id.get(),
9647 corr = frame.header.corr,
9648 op = %probe.op,
9649 "dropping late or unknown module-control RPC response"
9650 );
9651 }
9652 return Ok(Vec::new());
9653 }
9654 Err(err) => {
9655 if let Some(expected_op) = self
9656 .forwarding
9657 .pending_module_control_op(connection_id, frame.header.corr)
9658 .map_err(RouterError::Forwarding)?
9659 {
9660 let completion = self
9661 .forwarding
9662 .complete_module_control_rpc(
9663 connection_id,
9664 frame.header.corr,
9665 None,
9666 ModuleControlRpcOutcome::MalformedResponse(format!(
9667 "malformed {expected_op} response body: {err}"
9668 )),
9669 )
9670 .map_err(RouterError::Forwarding)?;
9671 if !self.observe_module_control_completion(completion) {
9672 debug!(
9673 connection_id = connection_id.get(),
9674 corr = frame.header.corr,
9675 "dropping late malformed module-control RPC response"
9676 );
9677 }
9678 return Ok(Vec::new());
9679 }
9680 let message = format!("malformed route.bind response body: {err}");
9681 secondary_error = Some(control_error_frame(
9682 &frame,
9683 "invalid_control_body",
9684 message.clone(),
9685 )?);
9686 RouteBindRelayOutcome::ModuleGone(message)
9687 }
9688 },
9689 FrameType::Error => {
9690 if self
9691 .forwarding
9692 .pending_module_control_op(connection_id, frame.header.corr)
9693 .map_err(RouterError::Forwarding)?
9694 .is_some()
9695 {
9696 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
9697 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
9698 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9699 "malformed module-control ERROR body: {err}"
9700 )),
9701 };
9702 let completion = self
9703 .forwarding
9704 .complete_module_control_rpc(
9705 connection_id,
9706 frame.header.corr,
9707 None,
9708 outcome,
9709 )
9710 .map_err(RouterError::Forwarding)?;
9711 if !self.observe_module_control_completion(completion) {
9712 debug!(
9713 connection_id = connection_id.get(),
9714 corr = frame.header.corr,
9715 "dropping late or unknown module-control RPC error"
9716 );
9717 }
9718 return Ok(Vec::new());
9719 }
9720 match serde_json::from_slice::<ErrorBody>(&frame.body) {
9721 Ok(body) => RouteBindRelayOutcome::Rejected(body),
9722 Err(err) => {
9723 let message = format!("malformed route.bind ERROR body: {err}");
9724 secondary_error = Some(control_error_frame(
9725 &frame,
9726 "invalid_control_body",
9727 message.clone(),
9728 )?);
9729 RouteBindRelayOutcome::ModuleGone(message)
9730 }
9731 }
9732 }
9733 ty => {
9734 return Ok(vec![control_error_frame(
9735 &frame,
9736 "unsupported_control_frame",
9737 format!("unsupported module channel-0 frame {ty:?}"),
9738 )?])
9739 }
9740 };
9741
9742 let settled =
9743 self.forwarding
9744 .complete_pending_relay(connection_id, frame.header.corr, outcome);
9745 let completion = match settled {
9746 Ok(completion) => completion,
9747 Err(err) => {
9748 self.refuse_to_end_module_connection_for_a_client(
9749 connection_id,
9750 frame.header.corr,
9751 err,
9752 )?;
9753 return Ok(secondary_error.into_iter().collect());
9754 }
9755 };
9756 if let Some(target) = completion.abandoned.as_ref() {
9757 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
9758 }
9759 if !completion.settled {
9760 debug!(
9761 connection_id = connection_id.get(),
9762 corr = frame.header.corr,
9763 frame_type = ?frame.header.ty,
9764 "dropping late or unknown route.bind relay response"
9765 );
9766 }
9767 Ok(secondary_error.into_iter().collect())
9768 }
9769
9770 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
9771 debug!(connection_id = connection_id.get(), "handling GOODBYE");
9772 self.cleanup_connection_with_end_reason(
9776 connection_id,
9777 RegistrationEndReason::ExplicitGoodbye,
9778 )
9779 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
9780 Ok(Vec::new())
9781 }
9782}
9783
9784impl Default for ControlHandler {
9785 fn default() -> Self {
9786 Self::new(Arc::new(Registry::default()))
9787 }
9788}
9789
9790impl crate::supervise::SwapPromotionObserver for ControlHandler {
9791 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
9792 self.apply_registration_capabilities(registration);
9793 }
9794}
9795
9796fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
9797 CapabilityRequirementStatus {
9798 consumer: status.consumer,
9799 capability: status.capability,
9800 need: match status.need {
9801 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
9802 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
9803 },
9804 verdict: status.verdict.as_str().to_string(),
9805 episode_seq: status.episode_seq,
9806 config_satisfiable: status.config_satisfiable,
9807 runtime_available: status.runtime_available,
9808 detail: status.detail,
9809 }
9810}
9811
9812fn append_capability_problem_detail(
9813 detail: Option<String>,
9814 capability_detail: Option<String>,
9815) -> Option<String> {
9816 match (detail, capability_detail) {
9817 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
9818 (Some(detail), None) => Some(detail),
9819 (None, Some(capability_detail)) => Some(capability_detail),
9820 (None, None) => None,
9821 }
9822}
9823
9824fn subc_ops() -> Vec<String> {
9825 SUBC_CONTROL_OPS
9826 .iter()
9827 .map(|op| (*op).to_string())
9828 .collect()
9829}
9830
9831fn module_subc_ops() -> Vec<String> {
9832 SUBC_CONTROL_OPS
9833 .iter()
9834 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
9835 .map(|op| (*op).to_string())
9836 .collect()
9837}
9838
9839#[cfg(test)]
9840fn module_baseline_control_ops() -> Vec<String> {
9841 MODULE_BASELINE_CONTROL_OPS
9842 .iter()
9843 .map(|op| (*op).to_string())
9844 .collect()
9845}
9846
9847fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
9848 let mut seen = HashSet::new();
9849 let mut effective = Vec::new();
9850 for op in MODULE_BASELINE_CONTROL_OPS {
9851 if seen.insert((*op).to_string()) {
9852 effective.push((*op).to_string());
9853 }
9854 }
9855 for op in declared.unwrap_or_default() {
9856 if seen.insert(op.clone()) {
9857 effective.push(op);
9858 }
9859 }
9860 effective
9861}
9862
9863fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
9864 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
9865}
9866
9867fn target_module_id(target: &RouteTarget) -> &str {
9868 match target {
9869 RouteTarget::ToolProvider { module_id }
9870 | RouteTarget::ManagementSurface { module_id }
9871 | RouteTarget::InternalService { module_id, .. } => module_id,
9872 }
9873}
9874
9875fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
9876 roles.iter().any(|role| match (target, role) {
9877 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
9878 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
9879 (
9880 RouteTarget::InternalService { service_id, .. },
9881 ProviderRole::InternalService {
9882 service_id: provided,
9883 ..
9884 },
9885 ) => service_id == provided,
9886 _ => false,
9887 })
9888}
9889
9890fn is_routable_role(role: &ProviderRole) -> bool {
9891 matches!(
9892 role,
9893 ProviderRole::ToolProvider { .. }
9894 | ProviderRole::ManagementSurface { .. }
9895 | ProviderRole::InternalService { .. }
9896 )
9897}
9898
9899#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9900enum ControlRequestBodyError {
9901 UnknownOp,
9902 InvalidBody,
9903}
9904
9905#[derive(Debug, Deserialize)]
9906struct ControlOpProbe {
9907 op: String,
9908}
9909
9910fn module_control_response_probe(body: &[u8]) -> Result<ControlOpProbe, serde_json::Error> {
9911 serde_json::from_slice(body).or_else(|original| {
9912 #[derive(Deserialize)]
9913 struct ConsentProbe {
9914 protocol: String,
9915 }
9916 match serde_json::from_slice::<ConsentProbe>(body) {
9917 Ok(probe) if probe.protocol == subc_protocol::consent::CONSENT_V1 => {
9918 Ok(ControlOpProbe { op: probe.protocol })
9919 }
9920 _ => Err(original),
9921 }
9922 })
9923}
9924
9925const MODULE_PUSH_OPS: &[&str] = &["route.status"];
9928
9929fn is_known_module_push_op(body: &[u8]) -> bool {
9930 serde_json::from_slice::<ControlOpProbe>(body)
9931 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
9932 .unwrap_or(false)
9933}
9934
9935fn is_known_module_request_op(body: &[u8]) -> bool {
9936 serde_json::from_slice::<ControlOpProbe>(body)
9937 .map(|probe| is_module_to_subc_op(&probe.op))
9938 .unwrap_or(false)
9939}
9940
9941fn is_module_to_subc_op(op: &str) -> bool {
9942 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
9943}
9944
9945fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
9946 debug!(
9947 op = %op,
9948 connection_id = connection_id.get(),
9949 corr,
9950 "control dispatch"
9951 );
9952}
9953
9954fn log_slow_control_dispatch(
9955 dispatch_started_at: Option<StdInstant>,
9956 op: &'static str,
9957 connection_id: ConnectionId,
9958 corr: u64,
9959 route_open_timings: Option<&RouteOpenTimings>,
9960) {
9961 let Some(dispatch_started_at) = dispatch_started_at else {
9962 return;
9963 };
9964 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
9966 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
9967 if let Some(timings) = route_open_timings {
9968 warn!(
9969 op = %op,
9970 connection_id = connection_id.get(),
9971 corr,
9972 elapsed_ms = elapsed.as_millis() as u64,
9973 module_id = ?timings.module_id,
9974 principal = %timings.principal,
9975 admission_ms = timings.admission.as_millis() as u64,
9976 project_root_ms = timings.project_root.as_millis() as u64,
9977 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
9978 bind_send_ms = timings.bind_send.as_millis() as u64,
9979 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
9980 "slow control dispatch"
9981 );
9982 return;
9983 }
9984 warn!(
9985 op = %op,
9986 connection_id = connection_id.get(),
9987 corr,
9988 elapsed_ms = elapsed.as_millis() as u64,
9989 "slow control dispatch"
9990 );
9991 }
9992}
9993
9994#[cfg(test)]
9996type TestCallerLookup =
9997 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
9998
9999fn privileged_operation(request: &impl PrivilegedRequest) -> Option<PrivilegedOperation<'_>> {
10001 request.privileged_operation()
10002}
10003
10004trait PrivilegedRequest {
10005 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>>;
10006}
10007
10008impl PrivilegedRequest for ClientControlRequest {
10009 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>> {
10010 let request = self;
10011 match request {
10012 ClientControlRequest::SupervisorRestart { module_id, .. } => {
10013 Some(PrivilegedOperation::Restart { module_id })
10014 }
10015 ClientControlRequest::SupervisorSwap { module_id, .. } => {
10016 Some(PrivilegedOperation::Swap { module_id })
10017 }
10018 ClientControlRequest::SupervisorReload { module_id } => {
10019 Some(PrivilegedOperation::Reload { module_id })
10020 }
10021 ClientControlRequest::SupervisorRescan { preview } => {
10022 if *preview {
10023 None
10024 } else {
10025 Some(PrivilegedOperation::Rescan)
10026 }
10027 }
10028 ClientControlRequest::SupervisorRescanChecked {
10033 preview,
10034 candidate,
10035 expect,
10036 } => (!*preview && candidate.is_none() && expect.is_some())
10037 .then_some(PrivilegedOperation::Rescan),
10038 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
10039 Some(PrivilegedOperation::SetEnabled {
10040 module_id,
10041 enabled: *enabled,
10042 })
10043 }
10044 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
10045 Some(PrivilegedOperation::ReleaseReserved { module_id })
10046 }
10047 ClientControlRequest::ServerDescribe {}
10048 | ClientControlRequest::DaemonShutdownIntent { .. }
10051 | ClientControlRequest::CatalogList { .. }
10052 | ClientControlRequest::RouteOpen { .. }
10053 | ClientControlRequest::RoutePoll { .. }
10054 | ClientControlRequest::SupervisorList {}
10055 | ClientControlRequest::SupervisorHealth {}
10056 | ClientControlRequest::SupervisorHealthProbe { .. }
10057 | ClientControlRequest::SupervisorRoutes { .. }
10058 | ClientControlRequest::SupervisorProvenance { .. }
10059 | ClientControlRequest::SupervisorStderrTail { .. }
10060 | ClientControlRequest::SupervisorTerminals { .. }
10061 | ClientControlRequest::SupervisorSpawnSnapshot {}
10062 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
10063 }
10064 }
10065}
10066
10067fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
10068 match request {
10069 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
10070 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
10071 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
10072 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
10073 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
10074 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
10075 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
10076 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
10077 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
10078 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
10079 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
10080 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
10081 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
10082 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
10083 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
10084 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
10085 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
10086 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
10087 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
10088 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
10089 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
10090 }
10091}
10092
10093struct RouteOpenCaller {
10101 principal: String,
10105 scope: String,
10108 harness: String,
10111}
10112
10113impl RouteOpenCaller {
10114 fn new(
10115 principal: Option<&Principal>,
10116 scope: Option<&ScopeSelector>,
10117 identity: &BindIdentity,
10118 ) -> Self {
10119 Self {
10120 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
10121 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
10122 harness: bounded_log_text(&identity.harness),
10123 }
10124 }
10125
10126 fn unknown() -> Self {
10129 Self {
10130 principal: "unknown".to_string(),
10131 scope: "unknown".to_string(),
10132 harness: "unknown".to_string(),
10133 }
10134 }
10135}
10136
10137const MAX_LOGGED_TEXT_CHARS: usize = 64;
10140
10141fn bounded_log_text(value: &str) -> String {
10145 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
10146 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
10147 bounded.push('…');
10148 }
10149 format!("{bounded:?}")
10150}
10151
10152fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
10158 use subc_protocol::scope::ScopeKind;
10159 let kind = match stamp.kind {
10160 ScopeKind::Head => "head",
10161 ScopeKind::Worker => "worker",
10162 ScopeKind::Ephemeral => "ephemeral",
10163 ScopeKind::Federated => "federated",
10164 _ => "unknown",
10165 };
10166 format!(
10167 "{kind}/{}/{}@{}",
10168 principal_label(&stamp.owner),
10169 bounded_log_text(&stamp.scope_ref),
10170 stamp.scope_epoch
10171 )
10172}
10173
10174fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
10181 let epoch = selector
10182 .scope_epoch
10183 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
10184 format!(
10185 "requested/{}/{}@{epoch}",
10186 bounded_log_text(&principal_label(&selector.owner)),
10187 bounded_log_text(&selector.scope_ref),
10188 )
10189}
10190
10191fn principal_label(principal: &Principal) -> String {
10192 match principal {
10193 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
10194 Principal::Direct => "direct".to_string(),
10195 other => format!("{other:?}"),
10196 }
10197}
10198
10199fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
10200 match request {
10201 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
10202 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
10203 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
10204 }
10205 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
10206 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
10207 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
10208 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
10209 ModuleControlRequestFromModule::RouteAuthorize { .. } => {
10210 subc_protocol::scope::ROUTE_AUTHORIZE_OP
10211 }
10212 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
10213 _ => "unsupported",
10214 }
10215}
10216
10217fn parse_client_control_request(
10218 body: &[u8],
10219) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
10220 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
10221 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10222 Ok(probe)
10223 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
10224 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
10225 {
10226 ControlRequestBodyError::InvalidBody
10227 }
10228 Ok(_) => ControlRequestBodyError::UnknownOp,
10229 Err(_) => ControlRequestBodyError::InvalidBody,
10230 };
10231 (err, classification)
10232 })
10233}
10234
10235fn parse_module_control_request_from_module(
10236 body: &[u8],
10237) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
10238 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
10239 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10240 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
10241 Ok(_) => ControlRequestBodyError::UnknownOp,
10242 Err(_) => ControlRequestBodyError::InvalidBody,
10243 };
10244 (err, classification)
10245 })
10246}
10247
10248#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
10249enum ProviderRoleKind {
10250 ToolProvider,
10251 PipelineStage,
10252 ManagementSurface,
10253 InternalService,
10254}
10255
10256fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
10257 match role {
10258 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
10259 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
10260 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
10261 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
10262 }
10263}
10264
10265fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
10266 roles.iter().map(provider_role_kind).collect()
10267}
10268
10269const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
10272
10273#[derive(Debug, Default, PartialEq, Eq)]
10275struct ScopeOutcomeCounts {
10276 created: usize,
10277 replaced: usize,
10278 updated: usize,
10279 unchanged: usize,
10280 refused: usize,
10281}
10282
10283impl ScopeOutcomeCounts {
10284 fn of(results: &[ScopeRecordResult]) -> Self {
10285 let mut counts = Self::default();
10286 for result in results {
10287 let slot = match result.outcome {
10288 ScopeRecordOutcome::Created => &mut counts.created,
10289 ScopeRecordOutcome::Replaced => &mut counts.replaced,
10290 ScopeRecordOutcome::Updated => &mut counts.updated,
10291 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
10292 ScopeRecordOutcome::Refused => &mut counts.refused,
10293 };
10294 *slot += 1;
10295 }
10296 counts
10297 }
10298}
10299
10300#[cfg(test)]
10301mod scope_outcome_count_tests {
10302 use super::*;
10303
10304 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
10305 ScopeRecordResult {
10306 scope_ref: "r".to_string(),
10307 scope_epoch: 1,
10308 outcome,
10309 code: None,
10310 message: None,
10311 version: None,
10312 parent_state: None,
10313 }
10314 }
10315
10316 #[test]
10319 fn every_outcome_is_counted_in_its_own_field() {
10320 let results = [
10321 result(ScopeRecordOutcome::Created),
10322 result(ScopeRecordOutcome::Created),
10323 result(ScopeRecordOutcome::Replaced),
10324 result(ScopeRecordOutcome::Updated),
10325 result(ScopeRecordOutcome::Unchanged),
10326 result(ScopeRecordOutcome::Refused),
10327 result(ScopeRecordOutcome::Refused),
10328 result(ScopeRecordOutcome::Refused),
10329 ];
10330 assert_eq!(
10331 ScopeOutcomeCounts::of(&results),
10332 ScopeOutcomeCounts {
10333 created: 2,
10334 replaced: 1,
10335 updated: 1,
10336 unchanged: 1,
10337 refused: 3,
10338 }
10339 );
10340 }
10341}
10342
10343fn capability_census_trigger(
10347 old: Option<&CapabilityDeclarations>,
10348 new: Option<&CapabilityDeclarations>,
10349) -> bool {
10350 let old_provides = old
10351 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
10352 .unwrap_or_default();
10353 let old_denies = old
10354 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
10355 .unwrap_or_default();
10356 let new = new.cloned().unwrap_or(CapabilityDeclarations {
10357 provides: Vec::new(),
10358 requires: Vec::new(),
10359 must_never_reach: Vec::new(),
10360 });
10361
10362 new.provides
10363 .iter()
10364 .any(|capability| !old_provides.contains(capability))
10365 || new
10366 .must_never_reach
10367 .iter()
10368 .any(|capability| !old_denies.contains(capability))
10369}
10370
10371fn catalog_update_frozen_field_message(
10372 registered: &ModuleManifest,
10373 provides: &[ProviderRole],
10374) -> Option<String> {
10375 let old_has_provides = !registered.provides.is_empty();
10376 let new_has_provides = !provides.is_empty();
10377 if old_has_provides != new_has_provides {
10378 return Some(format!(
10379 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
10380 registered.module_id
10381 ));
10382 }
10383
10384 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
10385 return Some(format!(
10386 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
10387 registered.module_id
10388 ));
10389 }
10390
10391 let registered_concurrency = manifest_concurrency(registered);
10392 let mut candidate = registered.clone();
10393 candidate.provides = provides.to_vec();
10394 let candidate_concurrency = manifest_concurrency(&candidate);
10395 if candidate_concurrency != registered_concurrency {
10396 return Some(format!(
10397 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
10398 registered.module_id, registered_concurrency, candidate_concurrency
10399 ));
10400 }
10401
10402 None
10405}
10406
10407fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
10408 manifest.provides.iter().any(is_routable_role)
10409}
10410
10411fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
10417 manifest
10418 .provides
10419 .iter()
10420 .find_map(|provider| match provider {
10421 ProviderRole::ToolProvider { concurrency, .. }
10422 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
10423 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
10424 })
10425 .unwrap_or(Concurrency::ModuleManaged)
10426}
10427
10428fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
10435 let has_management_surface = manifest
10436 .provides
10437 .iter()
10438 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
10439 if !has_management_surface {
10440 return false;
10441 }
10442 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
10443 return false;
10444 };
10445 let Some(provides) = raw
10446 .get("manifest")
10447 .and_then(|manifest| manifest.get("provides"))
10448 .and_then(serde_json::Value::as_array)
10449 else {
10450 return false;
10451 };
10452 provides.iter().any(|role| {
10457 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
10458 && role.get("concurrency").is_none()
10459 })
10460}
10461
10462fn negotiate_version(peer_version: u8) -> Result<u8, String> {
10463 if peer_version != PROTOCOL_VERSION {
10464 return Err(format!(
10465 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
10466 ));
10467 }
10468 Ok(PROTOCOL_VERSION)
10469}
10470
10471fn pong(frame: &Frame) -> Result<Frame, RouterError> {
10472 Frame::build_with_version(
10473 response_version(frame),
10474 FrameType::Pong,
10475 frame.header.flags,
10476 0,
10477 0,
10478 frame.header.corr,
10479 Vec::new(),
10480 )
10481 .map_err(RouterError::FrameBuild)
10482}
10483
10484pub(crate) fn control_error_frame(
10485 frame: &Frame,
10486 code: &'static str,
10487 message: impl Into<String>,
10488) -> Result<Frame, RouterError> {
10489 control_error_body_frame(
10490 frame,
10491 ErrorBody {
10492 code: code.to_string(),
10493 message: message.into(),
10494 detail: None,
10495 },
10496 )
10497}
10498
10499pub(crate) fn control_error_body_frame(
10507 frame: &Frame,
10508 error: ErrorBody,
10509) -> Result<Frame, RouterError> {
10510 control_error_frame_with_flags(frame, error, daemon_error_flags())
10511}
10512
10513fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
10519 control_error_frame_with_flags(frame, error, control_flags())
10520}
10521
10522fn control_error_frame_with_flags(
10523 frame: &Frame,
10524 error: ErrorBody,
10525 flags: Flags,
10526) -> Result<Frame, RouterError> {
10527 let body = serde_json::to_vec(&error).map_err(|err| {
10528 RouterError::backend(
10529 0,
10530 frame.header.corr,
10531 format!("failed to encode control ERROR: {err}"),
10532 )
10533 })?;
10534
10535 Frame::build_with_version(
10536 response_version(frame),
10537 FrameType::Error,
10538 flags,
10539 0,
10540 0,
10541 frame.header.corr,
10542 body,
10543 )
10544 .map_err(RouterError::FrameBuild)
10545}
10546
10547pub(crate) fn control_response_body_frame<T: Serialize>(
10548 frame: &Frame,
10549 reply: &T,
10550 label: &'static str,
10551) -> Result<Frame, RouterError> {
10552 let body = serde_json::to_vec(reply).map_err(|err| {
10553 RouterError::backend(
10554 0,
10555 frame.header.corr,
10556 format!("failed to encode {label}: {err}"),
10557 )
10558 })?;
10559
10560 Frame::build_with_version(
10561 response_version(frame),
10562 FrameType::Response,
10563 control_flags(),
10564 0,
10565 0,
10566 frame.header.corr,
10567 body,
10568 )
10569 .map_err(RouterError::FrameBuild)
10570}
10571
10572struct HelloAuthority {
10605 record: Arc<crate::spawn_authority::SpawnAuthorityRecord>,
10606 requested: ProviderDeclarations,
10607 froze: std::sync::atomic::AtomicBool,
10610}
10611
10612fn forwarding_error_code(err: &ForwardingError) -> &'static str {
10613 match err {
10614 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
10615 ForwardingError::NoModuleConnection => "target_unavailable",
10616 ForwardingError::ModuleReloading { .. } => "module_reloading",
10617 ForwardingError::ClientRouteChannelExhausted { .. }
10618 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
10619 ForwardingError::StaleModuleEndpoint
10620 | ForwardingError::UnknownReservation { .. }
10621 | ForwardingError::ConnectionClosing { .. }
10622 | ForwardingError::ClientEgressClosed { .. }
10623 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
10624 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
10627 ForwardingError::RelayCorrelationExhausted
10628 | ForwardingError::RouteOpenBuild(_)
10629 | ForwardingError::Poisoned => "forwarding_error",
10630 }
10631}
10632
10633fn response_version(frame: &Frame) -> u8 {
10634 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
10635 frame.header.ver
10636 } else {
10637 PROTOCOL_VERSION
10638 }
10639}
10640
10641pub(crate) fn control_flags() -> Flags {
10642 Flags::new(false, Priority::Passive, false)
10643}
10644
10645fn send_goodbye_target_best_effort(
10650 counters: &DaemonCounters,
10651 target: &GoodbyeTarget,
10652 context: &'static str,
10653) {
10654 let Ok(frame) = Frame::build_with_version(
10655 target.negotiated_ver,
10656 FrameType::Goodbye,
10657 control_flags(),
10658 target.channel,
10659 target.epoch,
10660 0,
10661 Vec::new(),
10662 ) else {
10663 return;
10664 };
10665 crate::forwarding::send_module_route_goodbye(
10666 counters,
10667 &target.sink,
10668 frame,
10669 target.module_id.as_deref(),
10670 context,
10671 );
10672}
10673
10674fn route_close_reason_for_opener(
10675 reason: RouteCloseReason,
10676 accepts_close_reasons: Option<&[String]>,
10677) -> RouteCloseReason {
10678 if reason == RouteCloseReason::AuthorityChanged
10679 && !accepts_close_reasons
10680 .is_some_and(|reasons| reasons.iter().any(|reason| reason == "authority_changed"))
10681 {
10682 RouteCloseReason::Reload
10685 } else {
10686 reason
10687 }
10688}
10689
10690pub(crate) fn send_route_control_pushes(
10691 forwarding: &ForwardingTable,
10692 routes: Vec<EndpointRoute>,
10693 push: ClientControlPush,
10694) {
10695 let reason = match &push {
10696 ClientControlPush::RouteClosing { reason, .. }
10697 | ClientControlPush::RouteClosed { reason, .. } => *reason,
10698 };
10699 let mut targets: Vec<(GoodbyeTarget, RouteCloseReason, Vec<u16>)> = Vec::new();
10700 for route in routes {
10701 let recipient_reason =
10702 route_close_reason_for_opener(reason, route.accepts_close_reasons.as_deref());
10703 let target = route.goodbye_target;
10704 if let Some((existing, _, channels)) = targets.iter_mut().find(|(existing, reason, _)| {
10705 existing.connection_id == target.connection_id && *reason == recipient_reason
10706 }) {
10707 debug_assert_eq!(
10708 existing.negotiated_ver, target.negotiated_ver,
10709 "one connection cannot negotiate multiple frame versions"
10710 );
10711 if !channels.contains(&target.channel) {
10712 channels.push(target.channel);
10713 }
10714 continue;
10715 }
10716 let channel = target.channel;
10717 targets.push((target, recipient_reason, vec![channel]));
10718 }
10719 for (target, recipient_reason, mut channels) in targets {
10720 channels.sort_unstable();
10721 let mut push = push.clone();
10722 match &mut push {
10723 ClientControlPush::RouteClosing {
10724 channels: covered,
10725 reason,
10726 ..
10727 }
10728 | ClientControlPush::RouteClosed {
10729 channels: covered,
10730 reason,
10731 ..
10732 } => {
10733 *covered = channels;
10734 *reason = recipient_reason;
10735 }
10736 }
10737 let body = match serde_json::to_vec(&push) {
10738 Ok(body) => body,
10739 Err(err) => {
10740 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
10741 continue;
10742 }
10743 };
10744 let frame = match Frame::build_with_version(
10745 target.negotiated_ver,
10746 FrameType::Push,
10747 control_flags(),
10748 0,
10749 0,
10750 0,
10751 body.clone(),
10752 ) {
10753 Ok(frame) => frame,
10754 Err(err) => {
10755 warn!(
10756 route_channel = target.channel,
10757 error = %err,
10758 "failed to build route lifecycle control PUSH frame"
10759 );
10760 continue;
10761 }
10762 };
10763 if let Err(err) = target.sink.try_send(frame) {
10764 if target.close_on_delivery_failure() {
10765 warn!(
10766 target_connection_id = target.connection_id.get(),
10767 route_channel = target.channel,
10768 error = %err,
10769 "route lifecycle control PUSH was not delivered to client; closing target connection"
10770 );
10771 let _ = forwarding.escalate_client_delivery_failure(
10772 target.connection_id,
10773 target.channel,
10774 target.epoch,
10775 CloseReason::new(
10776 "route_lifecycle_push_delivery_failed",
10777 format!(
10778 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
10779 target.channel
10780 ),
10781 ),
10782 crate::forwarding::UndeliveredFrame {
10783 module_id: target.module_id.as_deref(),
10784 sink: &target.sink,
10785 },
10786 );
10787 }
10788 }
10789 }
10790}
10791
10792#[cfg(test)]
10793mod tests {
10794 include!("control/agent_commands_tests.rs");
10795 include!("control/resource_revision_tests.rs");
10796 include!("control/route_open_recovery_tests.rs");
10797 #[tokio::test]
10798 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
10799 let registry = Arc::new(Registry::default());
10800 let handler = ControlHandler::new(registry);
10801 assert_eq!(
10802 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10803 None
10804 );
10805 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
10806 hello_via_sink(
10807 &handler,
10808 &ctx,
10809 &mut rx,
10810 hello_frame("daemon", PROTOCOL_VERSION, 1),
10811 )
10812 .await;
10813 assert_eq!(
10814 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10815 Some("daemon".into())
10816 );
10817 assert_eq!(
10818 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
10819 None
10820 );
10821 assert_eq!(
10822 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
10823 None,
10824 "a registered module is not necessarily supervised"
10825 );
10826 assert_eq!(
10827 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
10828 None,
10829 "aggregate provenance has no single target"
10830 );
10831 assert_eq!(
10832 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
10833 None
10834 );
10835 }
10836
10837 mod peer_manifest {
10838 include!("peer_manifest_tests.rs");
10839 }
10840 mod catalog_declarations {
10841 include!("control/catalog_tests.rs");
10842 }
10843 mod activity_read {
10844 include!("activity_read_tests.rs");
10845 }
10846
10847 mod lifecycle_activity {
10848 include!("control_lifecycle_tests.rs");
10849 }
10850 #[cfg(unix)]
10851 #[tokio::test]
10852 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
10853 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
10854 let path = dir.join("subc.jsonc");
10855 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
10856 "program":"/bin/sleep","args":["60"],"protocol":"none",
10857 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
10858 }}}).to_string()).unwrap();
10859 let mut configured = crate::daemon_config::load(&path)
10860 .unwrap()
10861 .unwrap()
10862 .modules
10863 .pop()
10864 .unwrap();
10865 let registry = std::sync::Arc::new(crate::Registry::default());
10866 let handle = crate::SupervisorHandle::new();
10867 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
10868 .with_handle(handle.clone());
10869 let module = supervisor
10870 .supervise_configured_with_health(
10871 configured.module_spec(),
10872 true,
10873 configured.health.clone(),
10874 None,
10875 configured.restart,
10876 )
10877 .unwrap();
10878 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
10879 let before = module.status().unwrap().pid;
10880 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
10881 configured.health.cadence = std::time::Duration::from_secs(3600);
10882 let health_only = handler
10883 .reconcile_supervised_modules(
10884 &supervisor,
10885 vec![configured.clone()],
10886 false,
10887 Principal::Direct,
10888 )
10889 .await
10890 .unwrap();
10891 assert!(
10892 health_only.0.changed_pending_reload.is_empty(),
10893 "health policy is already applied live"
10894 );
10895 assert_eq!(module.status().unwrap().pid, before);
10896 assert_eq!(
10897 module.configuration().unwrap().1.http,
10898 configured.health.http
10899 );
10900 configured.args = vec!["61".into()];
10901 let launch = handler
10902 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
10903 .await
10904 .unwrap();
10905 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
10906 assert_eq!(
10907 module.status().unwrap().pid,
10908 before,
10909 "a launch edit is stored until reload"
10910 );
10911 module.drain().await.unwrap();
10912 }
10913 use cortexkit_test_support::ScratchDir;
10914 use std::{
10915 collections::BTreeMap,
10916 fmt,
10917 path::PathBuf,
10918 sync::{Arc, Mutex},
10919 time::Duration,
10920 };
10921
10922 use serde_json::{json, Value};
10923 use subc_protocol::{
10924 manifest::{
10925 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
10926 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
10927 },
10928 session::HealthStatus,
10929 FrameType,
10930 };
10931
10932 use super::*;
10933 use crate::{
10934 forwarding::{DataRoute, DataRouteState},
10935 registry::ChannelState,
10936 router::FrameSink,
10937 stderr_tail::DEFAULT_MAX_LINE_BYTES,
10938 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
10939 RouteCtx, Router,
10940 };
10941 use tokio::{
10942 sync::mpsc,
10943 time::{sleep, Instant},
10944 };
10945 use tracing::{
10946 field::{Field, Visit},
10947 Event, Subscriber,
10948 };
10949 use tracing_subscriber::{layer::Context, prelude::*, Layer};
10950
10951 fn fake_aft_stub_path() -> PathBuf {
10970 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
10971 assert!(
10972 path.exists(),
10973 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
10974 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
10975 path.display()
10976 );
10977 path
10978 }
10979
10980 fn client_retries(code: &str) -> bool {
10985 subc_protocol::error_codes::is_retryable_route_open(code)
10986 }
10987
10988 #[test]
10999 fn retryability_of_forwarding_codes_matches_the_failure() {
11000 let transient = [
11003 ForwardingError::NoModuleConnection,
11004 ForwardingError::ModuleReloading {
11005 module_id: "m".into(),
11006 },
11007 ForwardingError::StaleModuleEndpoint,
11008 ForwardingError::UnknownReservation {
11009 client_channel: 1,
11010 module_channel: 1,
11011 },
11012 ForwardingError::ConnectionClosing {
11013 connection_id: ConnectionId::new(1),
11014 },
11015 ForwardingError::ClientEgressClosed {
11016 connection_id: ConnectionId::new(1),
11017 },
11018 ForwardingError::ModuleEgressUnavailable {
11019 connection_id: ConnectionId::new(1),
11020 },
11021 ];
11022 for err in transient {
11023 let code = forwarding_error_code(&err);
11024 assert!(
11025 client_retries(code),
11026 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
11027 );
11028 }
11029
11030 let permanent = [
11035 ForwardingError::ConnectionRoleConflict {
11036 connection_id: ConnectionId::new(1),
11037 },
11038 ForwardingError::ClientRouteChannelExhausted {
11039 connection_id: ConnectionId::new(1),
11040 },
11041 ForwardingError::ModuleRouteChannelExhausted {
11042 endpoint: ModuleEndpointId {
11043 connection_id: ConnectionId::new(1),
11044 generation: 1,
11045 },
11046 },
11047 ForwardingError::RelayCorrelationExhausted,
11048 ForwardingError::RouteOpenBuild("x".into()),
11049 ForwardingError::Poisoned,
11050 ];
11051 for err in permanent {
11052 let code = forwarding_error_code(&err);
11053 assert!(
11054 !client_retries(code),
11055 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
11056 );
11057 }
11058 }
11059
11060 #[tokio::test]
11088 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
11089 let handler = ControlHandler::default();
11090 let frame =
11091 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
11092
11093 let stamped = handler
11097 .route_open_principal(ConnectionId::new(1), &frame, None)
11098 .unwrap()
11099 .unwrap();
11100 assert_eq!(
11101 stamped,
11102 Principal::Direct,
11103 "a caller that proved nothing must not be stamped as a supervised module"
11104 );
11105
11106 let forged = handler
11111 .route_open_principal(
11112 ConnectionId::new(1),
11113 &frame,
11114 Some(ConsumerIdentity {
11115 module_id: "aft".to_string(),
11116 launch_nonce: "not-a-real-nonce".to_string(),
11117 }),
11118 )
11119 .unwrap();
11120 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
11121 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
11122 }
11123
11124 #[test]
11141 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
11142 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
11143 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
11144 let ClientControlRequest::RouteOpen {
11145 consumer_identity, ..
11146 } = parsed
11147 else {
11148 panic!("route.open body must parse as RouteOpen");
11149 };
11150 assert_eq!(
11151 consumer_identity,
11152 Some(ConsumerIdentity {
11153 module_id: "aft".to_string(),
11154 launch_nonce: "n".to_string(),
11155 }),
11156 "the wire field name must reach the value route_open_principal reads"
11157 );
11158 }
11159
11160 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
11161 ModuleManifest::builder(module_id, "0.1.0")
11162 .protocol_ver(protocol_ver)
11163 .provides(vec![ProviderRole::ToolProvider {
11164 tools: vec![Tool {
11165 name: "read".to_string(),
11166 description: None,
11167 execution_mode: ExecutionMode::Pure,
11168 schema: json!({"type": "object"}),
11169 }],
11170 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
11171 concurrency: Concurrency::ModuleManaged,
11172 emits_push: true,
11173 sub_supervises: true,
11174 }])
11175 .build()
11176 }
11177
11178 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
11179 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
11180 }
11181
11182 fn hello_frame_with_control_ops(
11183 module_id: &str,
11184 protocol_ver: u8,
11185 corr: u64,
11186 control_ops: Option<Vec<String>>,
11187 ) -> Frame {
11188 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
11189 }
11190
11191 fn hello_frame_with_nonce(
11192 module_id: &str,
11193 protocol_ver: u8,
11194 corr: u64,
11195 launch_nonce: Option<&str>,
11196 ) -> Frame {
11197 hello_frame_full(
11198 module_id,
11199 protocol_ver,
11200 corr,
11201 None,
11202 launch_nonce.map(ToOwned::to_owned),
11203 )
11204 }
11205
11206 fn hello_frame_full(
11207 module_id: &str,
11208 protocol_ver: u8,
11209 corr: u64,
11210 control_ops: Option<Vec<String>>,
11211 launch_nonce: Option<String>,
11212 ) -> Frame {
11213 let body = serde_json::to_vec(&ModuleHelloBody {
11214 manifest: manifest(module_id, protocol_ver),
11215 protocol_ver,
11216 control_ops,
11217 launch_nonce,
11218 })
11219 .unwrap();
11220 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11221 }
11222
11223 fn hello_frame_with_rich_ui(
11224 module_id: &str,
11225 protocol_ver: u8,
11226 corr: u64,
11227 launch_nonce: Option<&str>,
11228 rich_ui: Value,
11229 ) -> Frame {
11230 let mut body = serde_json::to_value(ModuleHelloBody {
11231 manifest: manifest(module_id, protocol_ver),
11232 protocol_ver,
11233 control_ops: None,
11234 launch_nonce: launch_nonce.map(str::to_string),
11235 })
11236 .unwrap();
11237 body["manifest"]["rich_ui"] = rich_ui;
11238 Frame::build(
11239 FrameType::Hello,
11240 control_flags(),
11241 0,
11242 0,
11243 corr,
11244 serde_json::to_vec(&body).unwrap(),
11245 )
11246 .unwrap()
11247 }
11248
11249 fn non_routable_hello_frame_with_control_ops(
11250 module_id: &str,
11251 corr: u64,
11252 control_ops: Option<Vec<String>>,
11253 ) -> Frame {
11254 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11255 manifest.provides.clear();
11256 let body = serde_json::to_vec(&ModuleHelloBody {
11257 manifest,
11258 protocol_ver: PROTOCOL_VERSION,
11259 control_ops,
11260 launch_nonce: None,
11261 })
11262 .unwrap();
11263 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11264 }
11265
11266 fn capability_grammar_hello_frame(
11267 capabilities: Value,
11268 runtime_computed: Option<Value>,
11269 corr: u64,
11270 ) -> Frame {
11271 let mut body = serde_json::to_value(ModuleHelloBody {
11272 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
11273 protocol_ver: PROTOCOL_VERSION,
11274 control_ops: None,
11275 launch_nonce: None,
11276 })
11277 .expect("HELLO body serializes");
11278 body["manifest"]["capabilities"] = capabilities;
11279 if let Some(runtime_computed) = runtime_computed {
11280 body["runtime_computed"] = runtime_computed;
11281 }
11282 Frame::build(
11283 FrameType::Hello,
11284 control_flags(),
11285 0,
11286 0,
11287 corr,
11288 serde_json::to_vec(&body).expect("HELLO body reserializes"),
11289 )
11290 .expect("HELLO frame builds")
11291 }
11292
11293 fn channel_request(channel: u16, corr: u64) -> Frame {
11294 Frame::build(
11295 FrameType::Request,
11296 Flags::new(true, Priority::Interactive, false),
11297 channel,
11298 0,
11299 corr,
11300 b"opaque".to_vec(),
11301 )
11302 .unwrap()
11303 }
11304
11305 fn route_ctx(
11306 connection_id: ConnectionId,
11307 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11308 let (tx, rx) = mpsc::channel(8);
11309 (
11310 RouteCtx {
11311 connection_id,
11312 egress: FrameSink::new(tx),
11313 },
11314 rx,
11315 )
11316 }
11317
11318 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
11319 serde_json::from_slice(&frame.body).unwrap()
11320 }
11321
11322 async fn hello_via_sink(
11327 handler: &ControlHandler,
11328 ctx: &RouteCtx,
11329 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11330 hello: Frame,
11331 ) -> Frame {
11332 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
11333 assert!(
11334 replies.is_empty(),
11335 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
11336 );
11337 let ack = rx
11338 .try_recv()
11339 .expect("HELLO_ACK is queued on the module sink")
11340 .frame;
11341 assert_eq!(ack.header.ty, FrameType::HelloAck);
11342 ack
11343 }
11344
11345 fn parse_error(frame: &Frame) -> Value {
11346 serde_json::from_slice(&frame.body).unwrap()
11347 }
11348
11349 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
11350 serde_json::from_slice(&frame.body).unwrap()
11351 }
11352
11353 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
11354 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
11355 route_channel,
11356 route_epoch: 0,
11357 kind,
11358 })
11359 .unwrap();
11360 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11361 }
11362
11363 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
11364 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
11365 module_id: module_id.to_string(),
11366 })
11367 .unwrap();
11368 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11369 }
11370
11371 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
11372 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
11373 }
11374
11375 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
11377 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
11378 let mut frame = route_open_frame(corr, module_id, root);
11379 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
11380 body["identity"]["project_root"] = serde_json::json!(path.as_path());
11381 frame.body = serde_json::to_vec(&body).unwrap();
11382 frame
11383 }
11384
11385 fn route_open_frame_with_consumer_capabilities(
11386 corr: u64,
11387 module_id: &str,
11388 _project_root: ScratchDir,
11389 consumer_capabilities: Option<Vec<String>>,
11390 ) -> Frame {
11391 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11392 busy_refusals: None,
11393 target: RouteTarget::ToolProvider {
11394 module_id: module_id.to_string(),
11395 },
11396 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
11397 consumer_identity: None,
11398 consumer_capabilities,
11399 role_versions: None,
11400 admission_facts: None,
11401 scope: None,
11402 })
11403 .unwrap();
11404 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11405 }
11406
11407 fn route_open_frame_with_role_versions(
11408 corr: u64,
11409 module_id: &str,
11410 _project_root: ScratchDir,
11411 role_versions: Option<BTreeMap<String, String>>,
11412 ) -> Frame {
11413 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11414 busy_refusals: None,
11415 target: RouteTarget::ToolProvider {
11416 module_id: module_id.to_string(),
11417 },
11418 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11419 consumer_identity: None,
11420 consumer_capabilities: None,
11421 role_versions,
11422 admission_facts: None,
11423 scope: None,
11424 })
11425 .unwrap();
11426 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11427 }
11428
11429 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
11430 entries
11431 .iter()
11432 .map(|(role, version)| (role.to_string(), version.to_string()))
11433 .collect()
11434 }
11435
11436 fn route_open_frame_with_admission_facts(
11437 corr: u64,
11438 module_id: &str,
11439 _project_root: ScratchDir,
11440 consumer_identity: Option<subc_control::ConsumerIdentity>,
11441 facts: Option<Value>,
11442 ) -> Frame {
11443 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11444 busy_refusals: None,
11445 target: RouteTarget::ToolProvider {
11446 module_id: module_id.to_string(),
11447 },
11448 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11449 consumer_identity,
11450 consumer_capabilities: None,
11451 role_versions: None,
11452 admission_facts: facts,
11453 scope: None,
11454 })
11455 .unwrap();
11456 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11457 }
11458
11459 #[derive(Clone, Default)]
11460 struct EventCapture {
11461 events: Arc<Mutex<Vec<CapturedEvent>>>,
11462 }
11463
11464 #[derive(Clone, Debug)]
11465 struct CapturedEvent {
11466 target: String,
11467 level: tracing::Level,
11468 fields: BTreeMap<String, String>,
11469 }
11470
11471 impl EventCapture {
11472 fn events(&self) -> Vec<CapturedEvent> {
11473 self.events.lock().unwrap().clone()
11474 }
11475 }
11476
11477 impl<S> Layer<S> for EventCapture
11478 where
11479 S: Subscriber,
11480 {
11481 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
11482 let mut visitor = EventFieldVisitor::default();
11483 event.record(&mut visitor);
11484 self.events.lock().unwrap().push(CapturedEvent {
11485 target: event.metadata().target().to_string(),
11486 level: *event.metadata().level(),
11487 fields: visitor.fields,
11488 });
11489 }
11490 }
11491
11492 #[derive(Default)]
11493 struct EventFieldVisitor {
11494 fields: BTreeMap<String, String>,
11495 }
11496
11497 impl Visit for EventFieldVisitor {
11498 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
11499 self.fields
11500 .insert(field.name().to_string(), format!("{value:?}"));
11501 }
11502 }
11503
11504 fn health_response(corr: u64, status: HealthStatus) -> Frame {
11505 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
11506 status,
11507 detail: Some("warming".to_string()),
11508 metrics: Some(json!({"queue_depth": 3})),
11509 })
11510 .unwrap();
11511 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11512 }
11513
11514 fn route_bind_ack(corr: u64) -> Frame {
11515 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
11516 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11517 }
11518
11519 fn unique_project_root(label: &str) -> ScratchDir {
11520 ScratchDir::new(label)
11521 }
11522
11523 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
11524 match parse_route_poll(frame) {
11525 ClientControlResponse::RoutePoll {
11526 status: None,
11527 live: Some(live),
11528 ..
11529 } => assert_eq!(live, expected_live),
11530 other => panic!("unexpected route.poll response: {other:?}"),
11531 }
11532 }
11533
11534 fn bind_liveness_route(
11535 registry: &Registry,
11536 forwarding: &ForwardingTable,
11537 module_id: &str,
11538 ) -> (RouteCtx, u16, u32) {
11539 let module_connection = ConnectionId::new(101);
11540 let client_connection = ConnectionId::new(202);
11541 let registration = registry
11542 .register_with_control_ops(
11543 manifest(module_id, PROTOCOL_VERSION),
11544 PROTOCOL_VERSION,
11545 module_connection,
11546 module_baseline_control_ops(),
11547 )
11548 .unwrap();
11549 let (module_tx, _module_rx) = mpsc::channel(8);
11550 let endpoint = forwarding
11551 .register_module_connection(
11552 module_connection,
11553 module_id.to_string(),
11554 PROTOCOL_VERSION,
11555 manifest_concurrency(®istration.manifest),
11556 FrameSink::new(module_tx),
11557 )
11558 .unwrap();
11559 let (client_ctx, _client_rx) = route_ctx(client_connection);
11560 let pending = forwarding
11561 .begin_route_bind_relay_for_test(
11562 client_connection,
11563 client_ctx.egress.clone(),
11564 1,
11565 module_id,
11566 )
11567 .unwrap();
11568 assert_eq!(pending.endpoint, endpoint);
11569 let route_channel = pending.client_channel;
11570 let route_epoch = pending.client_epoch;
11571 forwarding
11572 .complete_pending_relay(
11573 module_connection,
11574 pending.corr,
11575 RouteBindRelayOutcome::Accepted,
11576 )
11577 .unwrap();
11578 (client_ctx, route_channel, route_epoch)
11579 }
11580
11581 struct FakeProcessLiveness {
11582 live: Option<bool>,
11583 }
11584
11585 impl ModuleProcessLiveness for FakeProcessLiveness {
11586 fn process_live(&self, _module_id: &str) -> Option<bool> {
11587 self.live
11588 }
11589 }
11590
11591 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11592 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
11593 {
11594 let registry = Arc::new(Registry::default());
11595 let supervisor_handle = SupervisorHandle::new();
11596 let supervisor = Supervisor::new_for_test(
11597 Arc::clone(®istry),
11598 RestartPolicy::new(1, Duration::from_millis(10)),
11599 )
11600 .with_handle(supervisor_handle.clone());
11601 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
11602 let module = supervisor
11603 .spawn(ModuleSpec {
11604 script: None,
11605 carry_containment: false,
11606 module_id: "stderr-tail-wire".to_string(),
11607 program: fake_aft_stub_path(),
11608 args: Vec::new(),
11609 env: vec![
11610 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
11611 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
11612 ],
11613 reserved: false,
11614 reserved_prefixes: Vec::new(),
11615 protocol: ModuleProtocol::Subc,
11616 overlap: Default::default(),
11617 })
11618 .unwrap();
11619
11620 let deadline = Instant::now() + Duration::from_secs(5);
11621 loop {
11622 let tail = module.stderr_tail(None, None);
11623 if tail
11624 .entries
11625 .iter()
11626 .any(|entry| matches!(entry, TailEntry::ProcessStart))
11627 && tail.entries.iter().any(|entry| {
11628 matches!(
11629 entry,
11630 TailEntry::Line {
11631 truncated: true,
11632 ..
11633 }
11634 )
11635 })
11636 {
11637 break;
11638 }
11639 assert!(
11640 Instant::now() < deadline,
11641 "module did not produce a truncated line and restart boundary: {tail:?}"
11642 );
11643 sleep(Duration::from_millis(10)).await;
11644 }
11645
11646 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11647 let request = ClientControlRequest::SupervisorStderrTail {
11648 module_id: "stderr-tail-wire".to_string(),
11649 max_lines: None,
11650 max_bytes: None,
11651 };
11652 let frame = Frame::build(
11653 FrameType::Request,
11654 control_flags(),
11655 0,
11656 0,
11657 1,
11658 serde_json::to_vec(&request).unwrap(),
11659 )
11660 .unwrap();
11661 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11662 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11663 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
11664 serde_json::from_slice(&responses[0].body).unwrap()
11665 else {
11666 panic!("expected supervisor.stderr_tail response");
11667 };
11668
11669 assert!(
11670 tail.entries
11671 .iter()
11672 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
11673 "the control response lost the restart boundary"
11674 );
11675 let Some(StderrTailEntry::Line {
11676 text,
11677 truncated,
11678 at_ms,
11679 }) = tail.entries.iter().find(|entry| {
11680 matches!(
11681 entry,
11682 StderrTailEntry::Line {
11683 truncated: true,
11684 ..
11685 }
11686 )
11687 })
11688 else {
11689 panic!("the control response lost the truncated line");
11690 };
11691 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
11692 assert!(*truncated);
11693 assert!(
11694 at_ms.is_some(),
11695 "the control response lost the line's capture time"
11696 );
11697 }
11698
11699 #[tokio::test(flavor = "current_thread")]
11704 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
11705 let dir = ScratchDir::new("terminals-off-worker");
11706 let journal_path = dir.join("terminals.jsonl");
11707 let registry = Arc::new(Registry::default());
11708 let supervisor_handle = SupervisorHandle::new();
11709 let supervisor =
11710 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11711 .with_handle(supervisor_handle.clone())
11712 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
11713 let module = supervisor
11714 .spawn(ModuleSpec {
11715 script: None,
11716 carry_containment: false,
11717 module_id: "terminal-off-worker".to_string(),
11718 program: fake_aft_stub_path(),
11719 args: Vec::new(),
11720 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11721 reserved: false,
11722 reserved_prefixes: Vec::new(),
11723 protocol: ModuleProtocol::Subc,
11724 overlap: Default::default(),
11725 })
11726 .unwrap();
11727 let deadline = Instant::now() + Duration::from_secs(5);
11728 while module.terminal_history().entries.len() != 2 {
11729 assert!(Instant::now() < deadline, "module did not record two exits");
11730 sleep(Duration::from_millis(10)).await;
11731 }
11732
11733 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
11734 let handler =
11735 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
11736 let frame = Frame::build(
11737 FrameType::Request,
11738 control_flags(),
11739 0,
11740 0,
11741 1,
11742 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
11743 module_id: "terminal-off-worker".to_string(),
11744 })
11745 .unwrap(),
11746 )
11747 .unwrap();
11748 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11749 let spawned_at = std::time::Instant::now();
11750 let read = tokio::spawn({
11751 let handler = Arc::clone(&handler);
11752 async move { handler.handle_control_frame(&ctx, frame).await }
11753 });
11754 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
11757 .await
11758 .unwrap()
11759 .expect("the history read reached its pause");
11760 let elapsed = spawned_at.elapsed();
11761 assert!(
11762 elapsed < Duration::from_secs(2) && !read.is_finished(),
11763 "this task could not run while the history read was paused \
11764 (resumed after {elapsed:?}, read finished: {})",
11765 read.is_finished()
11766 );
11767
11768 drop(release);
11769 let responses = read.await.unwrap().unwrap();
11770 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11771 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
11772 panic!("expected supervisor.terminals response");
11773 };
11774 assert_eq!(terminals.entries.len(), 2);
11775 assert_eq!(terminals.journal_skipped_lines, 0);
11776 assert_eq!(terminals.journal_read_errors, 0);
11777 }
11778
11779 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11780 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
11781 let registry = Arc::new(Registry::default());
11782 let supervisor_handle = SupervisorHandle::new();
11783 let supervisor =
11784 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11785 .with_handle(supervisor_handle.clone());
11786 let module = supervisor
11787 .spawn(ModuleSpec {
11788 script: None,
11789 carry_containment: false,
11790 module_id: "terminal-golden".to_string(),
11791 program: fake_aft_stub_path(),
11792 args: Vec::new(),
11793 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11794 reserved: false,
11795 reserved_prefixes: Vec::new(),
11796 protocol: ModuleProtocol::Subc,
11797 overlap: Default::default(),
11798 })
11799 .unwrap();
11800
11801 let deadline = Instant::now() + Duration::from_secs(5);
11802 while module.terminal_history().entries.len() != 2 {
11803 assert!(
11804 Instant::now() < deadline,
11805 "module did not retain two terminal exits: {:?}",
11806 module.terminal_history()
11807 );
11808 sleep(Duration::from_millis(10)).await;
11809 }
11810
11811 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11812 let request = ClientControlRequest::SupervisorTerminals {
11813 module_id: "terminal-golden".to_string(),
11814 };
11815 let frame = Frame::build(
11816 FrameType::Request,
11817 control_flags(),
11818 0,
11819 0,
11820 1,
11821 serde_json::to_vec(&request).unwrap(),
11822 )
11823 .unwrap();
11824 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11825 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11826 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11827 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
11828 panic!("expected supervisor.terminals response");
11829 };
11830 assert_eq!(terminals.entries.len(), 2);
11831 assert_eq!(terminals.dropped, 0);
11832
11833 let mut rendered = serde_json::to_value(response).unwrap();
11834 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
11837 for (index, entry) in rendered["entries"]
11838 .as_array_mut()
11839 .expect("terminal response entries array")
11840 .iter_mut()
11841 .enumerate()
11842 {
11843 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
11844 }
11845
11846 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11847 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
11848 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
11849 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11850 std::fs::write(&golden_path, &serialized).unwrap();
11851 }
11852 let expected: Value =
11853 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
11854 assert_eq!(rendered, expected);
11855 }
11856
11857 #[test]
11858 fn hello_registers_manifest_and_returns_ack() {
11859 let registry = Arc::new(Registry::default());
11860 let handler = ControlHandler::new(Arc::clone(®istry));
11861 let conn = ConnectionId::new(1);
11862
11863 let responses = handler
11864 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
11865 .unwrap();
11866
11867 assert_eq!(responses.len(), 1);
11868 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11869 assert_eq!(responses[0].header.channel, 0);
11870 assert_eq!(responses[0].header.corr, 7);
11871 let ack = parse_ack(&responses[0]);
11872 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11873 assert!(ack
11874 .subc_capabilities
11875 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
11876 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
11877 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
11878 assert!(ack
11879 .subc_ops
11880 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
11881 assert!(ack
11882 .subc_ops
11883 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
11884 assert!(ack
11885 .subc_ops
11886 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
11887
11888 let registration = registry.get_module("aft").unwrap().unwrap();
11889 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
11890 assert_eq!(registration.state, ChannelState::Active);
11891 assert_eq!(registration.connection_id, conn);
11892 assert_eq!(registration.control_ops, module_baseline_control_ops());
11893 assert!(registration.manifest.display_name.is_none());
11894 }
11895
11896 #[test]
11897 fn hello_display_name_refuses_invalid_manifest_by_field() {
11898 let registry = Arc::new(Registry::default());
11899 let handler = ControlHandler::new(Arc::clone(®istry));
11900 for (index, invalid) in [
11901 json!(""),
11902 json!("a".repeat(41)),
11903 json!(" Routing"),
11904 json!("a\nb"),
11905 json!("a\u{202e}b"),
11906 json!(7),
11907 ]
11908 .into_iter()
11909 .enumerate()
11910 {
11911 let module_id = format!("bad-display-name-{index}");
11912 let mut body = serde_json::to_value(ModuleHelloBody {
11913 manifest: manifest(&module_id, PROTOCOL_VERSION),
11914 protocol_ver: PROTOCOL_VERSION,
11915 control_ops: None,
11916 launch_nonce: None,
11917 })
11918 .unwrap();
11919 body["manifest"]["display_name"] = invalid;
11920 let hello = Frame::build(
11921 FrameType::Hello,
11922 control_flags(),
11923 0,
11924 0,
11925 7,
11926 serde_json::to_vec(&body).unwrap(),
11927 )
11928 .unwrap();
11929 let response = handler
11930 .handle_control(ConnectionId::new(100 + index as u64), hello)
11931 .unwrap();
11932 assert_eq!(response[0].header.ty, FrameType::Error);
11933 let error = parse_error(&response[0]);
11934 assert_eq!(error["code"], "invalid_manifest");
11935 assert!(error["message"].as_str().unwrap().contains("display_name"));
11936 assert!(registry.get_module(&module_id).unwrap().is_none());
11937 }
11938 }
11939
11940 #[test]
11941 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
11942 let invalid_identifiers = [
11943 ("case_change", "credentials-Provider/v1"),
11944 ("leading_zero", "credentials-provider/v01"),
11945 ("trailing_hyphen", "credentials-provider-/v1"),
11946 ("consecutive_hyphens", "credentials--provider/v1"),
11947 ("uppercase", "Credentials-provider/v1"),
11948 ("missing_v", "credentials-provider/1"),
11949 ("whitespace", "credentials provider/v1"),
11950 ("zero_version", "credentials-provider/v0"),
11951 ("out_of_range_version", "credentials-provider/v4294967296"),
11952 (
11953 "overlength_name",
11954 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
11955 ),
11956 ];
11957 let mut cases = invalid_identifiers
11958 .into_iter()
11959 .map(|(name, identifier)| {
11960 (
11961 format!("identifier_{name}"),
11962 "capabilities.provides[0]".to_string(),
11963 identifier.to_string(),
11964 json!({ "provides": [identifier] }),
11965 None,
11966 )
11967 })
11968 .collect::<Vec<_>>();
11969 cases.extend([
11970 (
11971 "unknown_need".to_string(),
11972 "capabilities.requires[0].need".to_string(),
11973 "deferred".to_string(),
11974 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
11975 None,
11976 ),
11977 (
11978 "duplicate_provides".to_string(),
11979 "capabilities.provides[1]".to_string(),
11980 "credentials-provider/v1".to_string(),
11981 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
11982 None,
11983 ),
11984 (
11985 "duplicate_must_never_reach".to_string(),
11986 "capabilities.must_never_reach[1]".to_string(),
11987 "credentials-provider/v1".to_string(),
11988 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
11989 None,
11990 ),
11991 (
11992 "duplicate_requires_same_need".to_string(),
11993 "capabilities.requires[1]".to_string(),
11994 "credentials-provider/v1".to_string(),
11995 json!({ "requires": [
11996 { "capability": "credentials-provider/v1", "need": "required" },
11997 { "capability": "credentials-provider/v1", "need": "required" }
11998 ] }),
11999 None,
12000 ),
12001 (
12002 "duplicate_requires_conflicting_need".to_string(),
12003 "capabilities.requires[1]".to_string(),
12004 "credentials-provider/v1".to_string(),
12005 json!({ "requires": [
12006 { "capability": "credentials-provider/v1", "need": "required" },
12007 { "capability": "credentials-provider/v1", "need": "optional" }
12008 ] }),
12009 None,
12010 ),
12011 (
12012 "capabilities_root_pointer".to_string(),
12013 "runtime_computed[0]".to_string(),
12014 "/capabilities".to_string(),
12015 json!({}),
12016 Some(json!(["/capabilities"])),
12017 ),
12018 (
12019 "capabilities_descendant_pointer".to_string(),
12020 "runtime_computed[0]".to_string(),
12021 "/capabilities/provides".to_string(),
12022 json!({}),
12023 Some(json!(["/capabilities/provides"])),
12024 ),
12025 (
12026 "malformed_pointer_without_leading_slash".to_string(),
12027 "runtime_computed[0]".to_string(),
12028 "capabilities".to_string(),
12029 json!({}),
12030 Some(json!(["capabilities"])),
12031 ),
12032 (
12033 "malformed_pointer_escape".to_string(),
12034 "runtime_computed[0]".to_string(),
12035 "/roles/~2/tools".to_string(),
12036 json!({}),
12037 Some(json!(["/roles/~2/tools"])),
12038 ),
12039 (
12040 "unknown_capabilities_field".to_string(),
12041 "capabilities.future".to_string(),
12042 "<array>".to_string(),
12043 json!({ "future": [] }),
12044 None,
12045 ),
12046 ]);
12047
12048 for (index, (name, field, value, capabilities, runtime_computed)) in
12049 cases.into_iter().enumerate()
12050 {
12051 let registry = Arc::new(Registry::default());
12052 let handler = ControlHandler::new(Arc::clone(®istry));
12053 let response = handler
12054 .handle_control(
12055 ConnectionId::new((index + 1) as u64),
12056 capability_grammar_hello_frame(
12057 capabilities,
12058 runtime_computed,
12059 index as u64 + 1,
12060 ),
12061 )
12062 .expect("invalid HELLO returns a refusal");
12063
12064 assert_eq!(response.len(), 1, "{name} must emit one refusal");
12065 let error = parse_error(&response[0]);
12066 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
12067 let message = error["message"]
12068 .as_str()
12069 .expect("error message is a string");
12070 assert!(
12071 message.contains(&field),
12072 "{name}: field missing from {message}"
12073 );
12074 assert!(
12075 message.contains(&value),
12076 "{name}: value missing from {message}"
12077 );
12078 assert_eq!(
12079 registry
12080 .active_registration_count()
12081 .expect("registry reads"),
12082 0,
12083 "{name}: refused HELLO must not create a catalog entry"
12084 );
12085 }
12086 }
12087
12088 #[test]
12089 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
12090 let registry = Arc::new(Registry::default());
12091 let handler = ControlHandler::new(Arc::clone(®istry));
12092 let capabilities = json!({
12093 "provides": ["credentials-provider/v1"],
12094 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
12095 "must_never_reach": ["federation-transport/v1"]
12096 });
12097 let response = handler
12098 .handle_control(
12099 ConnectionId::new(99),
12100 capability_grammar_hello_frame(
12101 capabilities.clone(),
12102 Some(json!(["/roles/0/tools"])),
12103 99,
12104 ),
12105 )
12106 .expect("valid HELLO registers");
12107 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12108
12109 let request = Frame::build(
12110 FrameType::Request,
12111 control_flags(),
12112 0,
12113 0,
12114 100,
12115 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12116 .expect("catalog request serializes"),
12117 )
12118 .expect("catalog request frame builds");
12119 let response = handler
12120 .handle_catalog_list(request, None)
12121 .expect("catalog list succeeds");
12122 let ClientControlResponse::CatalogList { modules, .. } =
12123 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
12124 else {
12125 panic!("catalog request must return catalog.list");
12126 };
12127 assert_eq!(modules.len(), 1);
12128 assert_eq!(
12129 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
12130 capabilities
12131 );
12132 }
12133
12134 #[test]
12135 fn catalog_list_mirrors_management_operation_description() {
12136 let registry = Arc::new(Registry::default());
12137 let handler = ControlHandler::new(Arc::clone(®istry));
12138 let description = "List managed records and return their identifiers and metadata.";
12139 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
12140 manifest.provides = vec![ProviderRole::ManagementSurface {
12141 operations: vec![ManagementOperation {
12142 name: "records.list".to_string(),
12143 kind: ManagementOperationKind::Query,
12144 description: Some(description.to_string()),
12145 }],
12146 config_schema: json!({"type": "object"}),
12147 observability: vec![ObservabilitySurface {
12148 name: "records.stats".to_string(),
12149 kind: ObservabilityKind::Snapshot,
12150 }],
12151 identity_scope: vec![IdentityScope::Project],
12152 concurrency: Concurrency::ModuleManaged,
12153 }];
12154 registry
12155 .register_with_control_ops(
12156 manifest,
12157 PROTOCOL_VERSION,
12158 ConnectionId::new(99),
12159 Vec::new(),
12160 )
12161 .expect("described management manifest registers");
12162
12163 let request = Frame::build(
12164 FrameType::Request,
12165 control_flags(),
12166 0,
12167 0,
12168 100,
12169 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12170 .expect("catalog request serializes"),
12171 )
12172 .expect("catalog request frame builds");
12173 let response = handler
12174 .handle_catalog_list(request, None)
12175 .expect("catalog list succeeds");
12176 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
12177 assert_eq!(
12178 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
12179 "catalog.list must preserve the declared operation description verbatim"
12180 );
12181 }
12182
12183 #[test]
12184 fn catalog_list_mirrors_registered_operation_delivery() {
12185 let registry = Arc::new(Registry::default());
12186 let handler = ControlHandler::new(Arc::clone(®istry));
12187 let delivery =
12188 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
12189 let mut body = serde_json::to_value(ModuleHelloBody {
12190 manifest: manifest("watch-management", PROTOCOL_VERSION),
12191 protocol_ver: PROTOCOL_VERSION,
12192 control_ops: None,
12193 launch_nonce: None,
12194 })
12195 .unwrap();
12196 body["manifest"]["provides"] = json!([{
12197 "role": "management_surface",
12198 "operations": [
12199 {"name": "status", "kind": "query"},
12200 {"name": "future", "kind": "query"},
12201 {"name": "reset", "kind": "mutate"}
12202 ],
12203 "config_schema": {}, "observability": [], "identity_scope": []
12204 }]);
12205 body["manifest"]["operation_delivery"] = delivery.clone();
12206 let hello = Frame::build(
12207 FrameType::Hello,
12208 control_flags(),
12209 0,
12210 0,
12211 99,
12212 serde_json::to_vec(&body).unwrap(),
12213 )
12214 .unwrap();
12215 let response = handler
12216 .handle_control(ConnectionId::new(99), hello)
12217 .unwrap();
12218 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12219
12220 let (_, registrations) = registry.list_modules().unwrap();
12221 let registered = ®istrations[0].manifest;
12222 assert!(registered.is_watch_operation("status"));
12223 assert!(!registered.is_watch_operation("future"));
12224 assert!(!registered.is_watch_operation("reset"));
12225 assert!(!registered.is_watch_operation("missing"));
12226
12227 let request = Frame::build(
12228 FrameType::Request,
12229 control_flags(),
12230 0,
12231 0,
12232 100,
12233 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12234 )
12235 .unwrap();
12236 let response = handler.handle_catalog_list(request, None).unwrap();
12237 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
12238 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
12239 let ClientControlResponse::CatalogList { modules, .. } =
12240 serde_json::from_slice(&response[0].body).unwrap()
12241 else {
12242 panic!("expected catalog.list");
12243 };
12244 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
12245 }
12246
12247 #[test]
12248 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
12249 let registry = Arc::new(Registry::default());
12250 let supervisor = SupervisorHandle::new();
12251 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12252 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12253 let descriptor = json!({
12254 "schema": "ckios.richui.manifest/1",
12255 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12256 "bytes": 4096,
12257 "chunks": 4
12258 });
12259 let connection = ConnectionId::new(201);
12260 let hello = handler
12261 .handle_control(
12262 connection,
12263 hello_frame_with_rich_ui(
12264 "vault",
12265 PROTOCOL_VERSION,
12266 201,
12267 Some("the-real-nonce"),
12268 descriptor.clone(),
12269 ),
12270 )
12271 .unwrap();
12272 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12273
12274 let request =
12275 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
12276 let provides = registry
12277 .get_module_by_connection(connection)
12278 .unwrap()
12279 .unwrap()
12280 .manifest
12281 .provides;
12282 handler
12283 .handle_catalog_update(connection, request, provides, None, Some(false))
12284 .unwrap();
12285
12286 let request = Frame::build(
12287 FrameType::Request,
12288 control_flags(),
12289 0,
12290 0,
12291 203,
12292 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12293 )
12294 .unwrap();
12295 let response = handler.handle_catalog_list(request, None).unwrap();
12296 let ClientControlResponse::CatalogList { modules, .. } =
12297 serde_json::from_slice(&response[0].body).unwrap()
12298 else {
12299 panic!("expected catalog.list");
12300 };
12301 assert_eq!(
12302 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
12303 descriptor
12304 );
12305 assert!(modules[0].nonce_verified);
12306 }
12307
12308 fn hello_with_role_declarations(declarations: Value) -> Frame {
12309 let mut body = serde_json::to_value(ModuleHelloBody {
12310 manifest: manifest("role-test", PROTOCOL_VERSION),
12311 protocol_ver: PROTOCOL_VERSION,
12312 control_ops: None,
12313 launch_nonce: None,
12314 })
12315 .unwrap();
12316 body["manifest"]["role_declarations"] = declarations;
12317 Frame::build(
12318 FrameType::Hello,
12319 control_flags(),
12320 0,
12321 0,
12322 301,
12323 serde_json::to_vec(&body).unwrap(),
12324 )
12325 .unwrap()
12326 }
12327
12328 fn update_role_declarations_frame(declarations: Value) -> Frame {
12329 Frame::build(
12330 FrameType::Request,
12331 control_flags(),
12332 0,
12333 0,
12334 303,
12335 serde_json::to_vec(&json!({
12336 "op": "catalog.update_role_declarations",
12337 "role_declarations": declarations,
12338 }))
12339 .unwrap(),
12340 )
12341 .unwrap()
12342 }
12343
12344 #[tokio::test]
12345 async fn role_declarations_update_replaces_then_clears_catalog() {
12346 let registry = Arc::new(Registry::default());
12347 let handler = ControlHandler::new(Arc::clone(®istry));
12348 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12349 let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
12350 hello_via_sink(
12351 &handler,
12352 &ctx,
12353 &mut rx,
12354 hello_with_role_declarations(initial.clone()),
12355 )
12356 .await;
12357 assert_eq!(
12358 role_declarations_catalog(&handler).0["role_declarations"],
12359 initial
12360 );
12361 let before = registry.get_module("role-test").unwrap().unwrap();
12362 for declarations in [
12363 json!({"media-provider/v1": {"engines": ["new"]}}),
12364 json!({}),
12365 ] {
12366 let response = handler
12367 .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
12368 .await
12369 .unwrap();
12370 assert_eq!(response[0].header.ty, FrameType::Response);
12371 assert_eq!(
12372 serde_json::from_slice::<Value>(&response[0].body).unwrap(),
12373 json!({"op": "catalog.update_role_declarations"})
12374 );
12375 let (raw, entry) = role_declarations_catalog(&handler);
12376 assert_eq!(
12377 serde_json::to_value(entry.role_declarations).unwrap(),
12378 declarations
12379 );
12380 if declarations.as_object().unwrap().is_empty() {
12381 assert!(raw.get("role_declarations").is_none());
12382 } else {
12383 assert_eq!(raw["role_declarations"], declarations);
12384 }
12385 let after = registry.get_module("role-test").unwrap().unwrap();
12386 let mut expected = before.manifest.clone();
12387 expected.role_declarations = serde_json::from_value(declarations).unwrap();
12388 assert_eq!(after.manifest, expected);
12389 assert_eq!(after.ready, before.ready);
12390 }
12391 }
12392
12393 #[tokio::test]
12394 async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
12395 let registry = Arc::new(Registry::default());
12396 let handler = ControlHandler::new(registry);
12397 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12398 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12399 hello_via_sink(
12400 &handler,
12401 &ctx,
12402 &mut rx,
12403 hello_with_role_declarations(initial.clone()),
12404 )
12405 .await;
12406 for (invalid, reason) in [
12407 (json!(null), "role_declarations must be an object"),
12408 (json!([]), "role_declarations must be an object"),
12409 (
12410 json!({"bad key": {}}),
12411 "role_declarations key must use capability-name grammar",
12412 ),
12413 (
12414 json!({"media-provider/v1": []}),
12415 "role_declarations values must be objects",
12416 ),
12417 (
12418 json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
12419 "role_declarations exceeds 16384 compact JSON bytes",
12420 ),
12421 ] {
12422 let response = handler
12423 .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
12424 .await
12425 .unwrap();
12426 let error = parse_error(&response[0]);
12427 assert_eq!(error["code"], "invalid_role_declarations");
12428 assert_eq!(error["message"], reason);
12429 assert_eq!(
12430 role_declarations_catalog(&handler).0["role_declarations"],
12431 initial
12432 );
12433 }
12434 }
12435
12436 #[tokio::test]
12437 async fn role_declarations_update_is_owned_by_the_registered_connection() {
12438 let registry = Arc::new(Registry::default());
12439 let handler = ControlHandler::new(Arc::clone(®istry));
12440 let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
12441 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12442 hello_via_sink(
12443 &handler,
12444 &owner,
12445 &mut owner_rx,
12446 hello_with_role_declarations(initial.clone()),
12447 )
12448 .await;
12449 let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
12450 let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
12451 let response = handler
12452 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12453 .await
12454 .unwrap();
12455 assert_eq!(parse_error(&response[0])["code"], "not_registered");
12456 hello_via_sink(
12457 &handler,
12458 &other,
12459 &mut other_rx,
12460 hello_frame("other", PROTOCOL_VERSION, 304),
12461 )
12462 .await;
12463 let response = handler
12464 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12465 .await
12466 .unwrap();
12467 assert_eq!(response[0].header.ty, FrameType::Response);
12468 assert_eq!(
12469 serde_json::to_value(
12470 registry
12471 .get_module("role-test")
12472 .unwrap()
12473 .unwrap()
12474 .manifest
12475 .role_declarations
12476 )
12477 .unwrap(),
12478 initial
12479 );
12480 assert_eq!(
12481 serde_json::to_value(
12482 registry
12483 .get_module("other")
12484 .unwrap()
12485 .unwrap()
12486 .manifest
12487 .role_declarations
12488 )
12489 .unwrap(),
12490 replacement
12491 );
12492 }
12493
12494 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
12495 let request = Frame::build(
12496 FrameType::Request,
12497 control_flags(),
12498 0,
12499 0,
12500 302,
12501 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12502 )
12503 .unwrap();
12504 let response = handler.handle_catalog_list(request, None).unwrap();
12505 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
12506 let ClientControlResponse::CatalogList { mut modules, .. } =
12507 serde_json::from_slice(&response[0].body).unwrap()
12508 else {
12509 panic!("expected catalog.list");
12510 };
12511 (raw["modules"][0].clone(), modules.remove(0))
12512 }
12513
12514 #[test]
12515 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
12516 let registry = Arc::new(Registry::default());
12517 let handler = ControlHandler::new(Arc::clone(®istry));
12518 let declarations = json!({
12519 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
12520 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
12521 });
12522 let connection = ConnectionId::new(301);
12523 let response = handler
12524 .handle_control(
12525 connection,
12526 hello_with_role_declarations(declarations.clone()),
12527 )
12528 .unwrap();
12529 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12530 let registered = registry
12531 .get_module_by_connection(connection)
12532 .unwrap()
12533 .unwrap()
12534 .manifest;
12535 assert_eq!(
12539 serde_json::to_value(®istered.role_declarations).unwrap(),
12540 declarations
12541 );
12542 let (raw, entry) = role_declarations_catalog(&handler);
12543 assert_eq!(raw["role_declarations"], declarations);
12544 assert_eq!(entry.role_declarations, registered.role_declarations);
12545
12546 let request =
12547 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
12548 handler
12549 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
12550 .unwrap();
12551 let (raw, entry) = role_declarations_catalog(&handler);
12552 assert_eq!(raw["role_declarations"], declarations);
12553 assert_eq!(entry.role_declarations, registered.role_declarations);
12554 }
12555
12556 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
12557 let registry = Arc::new(Registry::default());
12558 let handler = ControlHandler::new(Arc::clone(®istry));
12559 let capture = EventCapture::default();
12560 let _subscriber =
12561 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12562 let response = handler
12563 .handle_control(
12564 ConnectionId::new(301),
12565 hello_with_role_declarations(declarations),
12566 )
12567 .unwrap();
12568 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12569 assert!(registry
12570 .get_module_by_connection(ConnectionId::new(301))
12571 .unwrap()
12572 .unwrap()
12573 .manifest
12574 .role_declarations
12575 .is_empty());
12576 let (raw, entry) = role_declarations_catalog(&handler);
12577 assert!(raw.get("role_declarations").is_none());
12578 assert!(entry.role_declarations.is_empty());
12579 let warnings: Vec<_> = capture
12580 .events()
12581 .into_iter()
12582 .filter(|event| {
12583 event.fields.get("message").map(String::as_str)
12584 == Some("dropping whole role_declarations map from HELLO")
12585 })
12586 .collect();
12587 assert_eq!(warnings.len(), 1);
12588 assert_eq!(warnings[0].level, tracing::Level::WARN);
12589 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
12590 assert_eq!(warnings[0].fields["reason"], reason);
12591 }
12592
12593 #[test]
12594 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
12595 for value in [
12596 json!(null),
12597 json!(false),
12598 json!(0),
12599 json!("opaque"),
12600 json!([]),
12601 ] {
12602 assert_hello_drops_role_declarations_with_warning(
12603 json!({"routing/v1": {}, "broken/v1": value}),
12604 "role_declarations values must be objects",
12605 );
12606 }
12607 assert_hello_drops_role_declarations_with_warning(
12608 json!([]),
12609 "role_declarations must be an object",
12610 );
12611 }
12612
12613 #[test]
12614 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
12615 assert_hello_drops_role_declarations_with_warning(
12616 json!({"routing/v1": {}, "bad key": {}}),
12617 "role_declarations key must use capability-name grammar",
12618 );
12619 }
12620
12621 #[test]
12622 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
12623 assert_hello_drops_role_declarations_with_warning(
12624 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
12625 "role_declarations exceeds 16384 compact JSON bytes",
12626 );
12627 }
12628
12629 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
12630 let mut body = serde_json::to_value(ModuleHelloBody {
12631 manifest: manifest("flow-ops", PROTOCOL_VERSION),
12632 protocol_ver: PROTOCOL_VERSION,
12633 control_ops: None,
12634 launch_nonce: None,
12635 })
12636 .unwrap();
12637 body["manifest"]["provides"] = json!([{
12638 "role": "management_surface",
12639 "operations": [
12640 {"name": "diagnostics.findings", "kind": "query"},
12641 {"name": "reset", "kind": "mutate"}
12642 ],
12643 "config_schema": {}, "observability": [], "identity_scope": []
12644 }]);
12645 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
12646 if let Some(flow_operations) = flow_operations {
12647 body["manifest"]["flow_operations"] = flow_operations;
12648 }
12649 Frame::build(
12650 FrameType::Hello,
12651 control_flags(),
12652 0,
12653 0,
12654 401,
12655 serde_json::to_vec(&body).unwrap(),
12656 )
12657 .unwrap()
12658 }
12659
12660 #[test]
12661 fn catalog_list_mirrors_flow_operations_unchanged() {
12662 for (capability, flow_operations) in [
12665 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
12666 ("call-modes/v1", json!(["diagnostics.findings"])),
12667 ("flow-scopes/v1", json!([])),
12668 ] {
12669 let registry = Arc::new(Registry::default());
12670 let handler = ControlHandler::new(Arc::clone(®istry));
12671 let connection = ConnectionId::new(401);
12672 let response = handler
12673 .handle_control(
12674 connection,
12675 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
12676 )
12677 .unwrap();
12678 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12679 let registered = registry
12680 .get_module_by_connection(connection)
12681 .unwrap()
12682 .unwrap()
12683 .manifest;
12684 assert_eq!(
12685 serde_json::to_value(®istered.flow_operations).unwrap(),
12686 flow_operations
12687 );
12688 let (raw, entry) = role_declarations_catalog(&handler);
12689 assert_eq!(raw["flow_operations"], flow_operations);
12690 assert_eq!(entry.flow_operations, registered.flow_operations);
12691 }
12692
12693 let registry = Arc::new(Registry::default());
12694 let handler = ControlHandler::new(Arc::clone(®istry));
12695 let response = handler
12696 .handle_control(
12697 ConnectionId::new(401),
12698 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
12699 )
12700 .unwrap();
12701 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12702 let (raw, entry) = role_declarations_catalog(&handler);
12703 assert!(raw.get("flow_operations").is_none());
12704 assert_eq!(entry.flow_operations, None);
12705 }
12706
12707 #[test]
12708 fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
12709 let oversized: serde_json::Map<String, Value> = (0..257)
12710 .map(|n| (format!("tool:t{n}"), json!(true)))
12711 .collect();
12712 for (capabilities, map, valid) in [
12713 (
12714 json!(["scope-federated/v1"]),
12715 json!({"tool:read":true,"management:reset":true}),
12716 true,
12717 ),
12718 (json!(["scope-federated/v1"]), json!({}), true),
12719 (json!([]), json!({"tool:read":true}), false),
12720 (
12721 json!(["scope-federated/v1"]),
12722 json!({"tool:valid":true,"tool:bad":false}),
12723 false,
12724 ),
12725 (
12726 json!(["scope-federated/v1"]),
12727 json!({"tool:valid":true,"management:missing":true}),
12728 false,
12729 ),
12730 (
12731 json!(["scope-federated/v1"]),
12732 Value::Object(oversized),
12733 false,
12734 ),
12735 ] {
12736 let capture = EventCapture::default();
12737 let _subscriber = tracing::subscriber::set_default(
12738 tracing_subscriber::registry().with(capture.clone()),
12739 );
12740 let registry = Arc::new(Registry::default());
12741 let handler = ControlHandler::new(Arc::clone(®istry));
12742 let hello = hello_with_flow_operations(capabilities, None);
12743 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12744 body["manifest"]["operation_deferral"] = map.clone();
12745 let hello = Frame::build(
12746 FrameType::Hello,
12747 control_flags(),
12748 0,
12749 0,
12750 401,
12751 serde_json::to_vec(&body).unwrap(),
12752 )
12753 .unwrap();
12754 let response = handler
12755 .handle_control(ConnectionId::new(401), hello)
12756 .unwrap();
12757 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12758 let (raw, entry) = role_declarations_catalog(&handler);
12759 if valid {
12760 assert_eq!(raw["operation_deferral"], map);
12761 assert_eq!(
12762 serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
12763 map
12764 );
12765 } else {
12766 assert!(entry.operation_deferral.is_none());
12767 assert!(raw.get("operation_deferral").is_none());
12768 assert_eq!(
12769 capture
12770 .events()
12771 .iter()
12772 .filter(|event| event.fields.get("message").map(String::as_str)
12773 == Some("dropping operation_deferral from HELLO"))
12774 .count(),
12775 1
12776 );
12777 }
12778 }
12779 }
12780
12781 fn assert_hello_drops_flow_operations_with_warning(
12782 capabilities: Value,
12783 flow_operations: Value,
12784 reason: &str,
12785 ) {
12786 let registry = Arc::new(Registry::default());
12787 let handler = ControlHandler::new(Arc::clone(®istry));
12788 let capture = EventCapture::default();
12789 let _subscriber =
12790 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12791 let response = handler
12792 .handle_control(
12793 ConnectionId::new(401),
12794 hello_with_flow_operations(capabilities, Some(flow_operations)),
12795 )
12796 .unwrap();
12797 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12798 let registered = registry
12799 .get_module_by_connection(ConnectionId::new(401))
12800 .unwrap()
12801 .expect("the module still registers")
12802 .manifest;
12803 assert_eq!(registered.module_id, "flow-ops");
12804 assert_eq!(registered.flow_operations, None);
12805 let (raw, entry) = role_declarations_catalog(&handler);
12806 assert_eq!(raw["module_id"], "flow-ops");
12807 assert!(raw.get("flow_operations").is_none());
12808 assert_eq!(entry.flow_operations, None);
12809 let warnings: Vec<_> = capture
12810 .events()
12811 .into_iter()
12812 .filter(|event| {
12813 event.fields.get("message").map(String::as_str)
12814 == Some("dropping flow_operations from HELLO")
12815 })
12816 .collect();
12817 assert_eq!(warnings.len(), 1);
12818 assert_eq!(warnings[0].level, tracing::Level::WARN);
12819 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
12820 assert_eq!(warnings[0].fields["reason"], reason);
12821 }
12822
12823 #[test]
12824 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
12825 assert_hello_drops_flow_operations_with_warning(
12826 json!(["flow-scopes/v1"]),
12827 json!(["diagnostics.findings", "missing"]),
12828 "flow_operations names 'missing', which is not a declared management operation",
12829 );
12830 }
12831
12832 #[test]
12833 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
12834 assert_hello_drops_flow_operations_with_warning(
12835 json!([]),
12836 json!(["diagnostics.findings"]),
12837 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12838 );
12839 assert_hello_drops_flow_operations_with_warning(
12841 json!(["agent-run-scopes/v1"]),
12842 json!([]),
12843 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12844 );
12845 }
12846
12847 #[test]
12848 fn hello_malformed_flow_operations_is_dropped_with_warning() {
12849 assert_hello_drops_flow_operations_with_warning(
12850 json!(["flow-scopes/v1"]),
12851 json!(["reset", "reset"]),
12852 "flow_operations names 'reset' more than once",
12853 );
12854 assert_hello_drops_flow_operations_with_warning(
12855 json!(["flow-scopes/v1"]),
12856 json!(vec!["reset"; 257]),
12857 "flow_operations lists 257 names, more than 256",
12858 );
12859 assert_hello_drops_flow_operations_with_warning(
12860 json!(["flow-scopes/v1"]),
12861 json!("reset"),
12862 "flow_operations must be an array of operation names",
12863 );
12864 }
12865
12866 #[test]
12867 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
12868 let registry = Arc::new(Registry::default());
12869 let handler = ControlHandler::new(registry.clone());
12870 let descriptor = json!({
12871 "schema": "ckios.richui.manifest/1",
12872 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12873 "bytes": 128,
12874 "chunks": 2
12875 });
12876 let malformed = [
12877 ("schema", json!("")),
12878 ("schema", json!("a".repeat(65))),
12879 ("schema", json!("Bad schema")),
12880 (
12881 "manifest_digest",
12882 json!(format!("sha256:{}", "A".repeat(64))),
12883 ),
12884 ("manifest_digest", json!("sha256:abc")),
12885 ("bytes", json!(0)),
12886 ("bytes", json!(16_777_217)),
12887 ("bytes", json!("128")),
12888 ("chunks", json!(0)),
12889 ("chunks", json!(1025)),
12890 ("chunks", json!("2")),
12891 ];
12892 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
12893 let mut invalid = descriptor.clone();
12894 invalid[field] = invalid_value;
12895 let response = handler
12896 .handle_control(
12897 ConnectionId::new(210 + index as u64),
12898 hello_frame_with_rich_ui(
12899 &format!("bad-rich-ui-{index}"),
12900 PROTOCOL_VERSION,
12901 210 + index as u64,
12902 None,
12903 invalid,
12904 ),
12905 )
12906 .unwrap();
12907 assert_eq!(response[0].header.ty, FrameType::Error);
12908 let error = parse_error(&response[0]);
12909 assert_eq!(error["code"], "invalid_manifest");
12910 assert!(error["message"]
12911 .as_str()
12912 .unwrap()
12913 .contains(&format!("rich_ui.{field}")));
12914 assert!(registry
12915 .get_module(&format!("bad-rich-ui-{index}"))
12916 .unwrap()
12917 .is_none());
12918 }
12919 }
12920
12921 #[test]
12922 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
12923 let registry = Arc::new(Registry::default());
12924 let handler = ControlHandler::new(Arc::clone(®istry));
12925 let connection = ConnectionId::new(219);
12926 registry
12927 .register_with_control_ops(
12928 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
12929 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
12930 "bad schema",
12931 format!("sha256:{}", "a".repeat(64)),
12932 128,
12933 2,
12934 )))
12935 .build(),
12936 PROTOCOL_VERSION,
12937 connection,
12938 Vec::new(),
12939 )
12940 .unwrap();
12941 let request =
12942 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
12943 let response = handler
12944 .handle_catalog_update(connection, request, Vec::new(), None, None)
12945 .unwrap();
12946 assert_eq!(response[0].header.ty, FrameType::Error);
12947 let error = parse_error(&response[0]);
12948 assert_eq!(error["code"], "invalid_manifest");
12949 assert!(error["message"]
12950 .as_str()
12951 .unwrap()
12952 .contains("rich_ui.schema"));
12953 }
12954
12955 #[test]
12956 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
12957 let registry = Arc::new(Registry::default());
12958 let supervisor = SupervisorHandle::new();
12959 supervisor.open_swap("provider", "candidate-nonce".to_string());
12960 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12961 let connection = ConnectionId::new(218);
12962 let hello = handler
12963 .handle_control(
12964 connection,
12965 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
12966 )
12967 .unwrap();
12968 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12969 assert!(registry.list_modules().unwrap().1.is_empty());
12970
12971 registry.promote_candidate("provider").unwrap().unwrap();
12972 let request = Frame::build(
12973 FrameType::Request,
12974 control_flags(),
12975 0,
12976 0,
12977 219,
12978 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12979 )
12980 .unwrap();
12981 let response = handler.handle_catalog_list(request, None).unwrap();
12982 let ClientControlResponse::CatalogList { modules, .. } =
12983 serde_json::from_slice(&response[0].body).unwrap()
12984 else {
12985 panic!("expected catalog.list");
12986 };
12987 assert_eq!(modules.len(), 1);
12988 assert!(modules[0].nonce_verified);
12989 }
12990
12991 #[test]
12992 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
12993 for provenance_json in [
12994 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
12995 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
12996 ] {
12997 let registry = Arc::new(Registry::default());
12998 let handler = ControlHandler::new(registry);
12999 let hello = Frame::build(
13000 FrameType::Hello,
13001 control_flags(),
13002 0,
13003 0,
13004 220,
13005 serde_json::to_vec(&ModuleHelloBody {
13006 manifest: ModuleManifest::builder("built", "0.1.0")
13007 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
13008 .build(),
13009 protocol_ver: PROTOCOL_VERSION,
13010 control_ops: None,
13011 launch_nonce: None,
13012 })
13013 .unwrap(),
13014 )
13015 .unwrap();
13016 assert_eq!(
13017 handler
13018 .handle_control(ConnectionId::new(220), hello)
13019 .unwrap()[0]
13020 .header
13021 .ty,
13022 FrameType::HelloAck
13023 );
13024 let request = Frame::build(
13025 FrameType::Request,
13026 control_flags(),
13027 0,
13028 0,
13029 221,
13030 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13031 )
13032 .unwrap();
13033 let response = handler.handle_catalog_list(request, None).unwrap();
13034 let ClientControlResponse::CatalogList { modules, .. } =
13035 serde_json::from_slice(&response[0].body).unwrap()
13036 else {
13037 panic!("expected catalog.list");
13038 };
13039 assert_eq!(modules.len(), 1);
13040 assert_eq!(
13041 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13042 format!(
13043 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
13044 )
13045 .as_bytes()
13046 );
13047 }
13048 }
13049
13050 #[test]
13051 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
13052 let registry = Arc::new(Registry::default());
13053 let handler = ControlHandler::new(Arc::clone(®istry));
13054 let hello = Frame::build(
13055 FrameType::Hello,
13056 control_flags(),
13057 0,
13058 0,
13059 220,
13060 serde_json::to_vec(&ModuleHelloBody {
13061 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
13062 protocol_ver: PROTOCOL_VERSION,
13063 control_ops: None,
13064 launch_nonce: Some("unverified".to_string()),
13065 })
13066 .unwrap(),
13067 )
13068 .unwrap();
13069 assert_eq!(
13070 handler
13071 .handle_control(ConnectionId::new(220), hello)
13072 .unwrap()[0]
13073 .header
13074 .ty,
13075 FrameType::HelloAck
13076 );
13077 let request = Frame::build(
13078 FrameType::Request,
13079 control_flags(),
13080 0,
13081 0,
13082 221,
13083 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13084 )
13085 .unwrap();
13086 let response = handler.handle_catalog_list(request, None).unwrap();
13087 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
13088 assert!(wire["modules"][0].get("provenance").is_none());
13089 let ClientControlResponse::CatalogList { modules, .. } =
13090 serde_json::from_slice(&response[0].body).unwrap()
13091 else {
13092 panic!("expected catalog.list");
13093 };
13094 assert_eq!(
13095 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13096 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
13097 );
13098 assert!(!modules[0].nonce_verified);
13099 assert!(modules[0].provenance.is_none());
13100 }
13101
13102 fn catalog_legacy_projection(mut entry: CatalogEntry) -> CatalogEntry {
13104 assert_eq!(entry.first_party, Some(false));
13105 entry.incarnation = None;
13106 entry.spawn = None;
13107 entry.first_party = None;
13108 entry.origin = None;
13109 entry.requested_declarations_digest = None;
13110 entry.approved_declarations_digest = None;
13111 entry
13112 }
13113
13114 #[test]
13115 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
13116 let registry = Arc::new(Registry::default());
13117 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13118 [("vault".to_string(), true), ("squatter".to_string(), true)],
13119 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13120 );
13121 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
13122 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13123 provides: vec!["credentials-provider/v1".to_string()],
13124 requires: Vec::new(),
13125 must_never_reach: Vec::new(),
13126 });
13127 let frame = Frame::build(
13128 FrameType::Hello,
13129 control_flags(),
13130 0,
13131 0,
13132 77,
13133 serde_json::to_vec(&ModuleHelloBody {
13134 manifest: squatter,
13135 protocol_ver: PROTOCOL_VERSION,
13136 control_ops: None,
13137 launch_nonce: None,
13138 })
13139 .expect("HELLO serializes"),
13140 )
13141 .expect("HELLO frame builds");
13142 let response = handler
13143 .handle_control(ConnectionId::new(77), frame)
13144 .expect("reserved claim receives a typed refusal");
13145 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13146 assert_eq!(
13147 registry
13148 .active_registration_count()
13149 .expect("registry reads"),
13150 0,
13151 "a reserved capability refusal must not leave a catalog entry"
13152 );
13153 }
13154
13155 #[test]
13156 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
13157 for settlement in ["timeout", "inconclusive", "drop"] {
13158 let breakers = RouteBindBreakers::default();
13159 let RouteBindAdmission::Admitted {
13160 guard: mut old,
13161 probe: false,
13162 } = breakers.admit("prov")
13163 else {
13164 panic!("ordinary relay admitted")
13165 };
13166 let RouteBindAdmission::Admitted {
13167 guard: mut opener, ..
13168 } = breakers.admit("prov")
13169 else {
13170 panic!("second relay admitted")
13171 };
13172 assert!(
13173 !opener
13174 .record_timeout(1, Duration::ZERO)
13175 .unwrap()
13176 .reopened_after_probe
13177 );
13178 let RouteBindAdmission::Admitted {
13179 guard: mut probe,
13180 probe: true,
13181 } = breakers.admit("prov")
13182 else {
13183 panic!("one cooldown probe admitted")
13184 };
13185 match settlement {
13186 "timeout" => assert!(
13187 !old.record_timeout(1, Duration::ZERO)
13188 .unwrap()
13189 .reopened_after_probe
13190 ),
13191 "inconclusive" => old.record_inconclusive(),
13192 "drop" => drop(old),
13193 _ => unreachable!(),
13194 }
13195 assert!(
13196 matches!(
13197 breakers.admit("prov"),
13198 RouteBindAdmission::Refused {
13199 probe_in_flight: true,
13200 ..
13201 }
13202 ),
13203 "{settlement} of a pre-open relay cannot release the real probe"
13204 );
13205 assert!(
13206 probe
13207 .record_timeout(1, Duration::ZERO)
13208 .unwrap()
13209 .reopened_after_probe
13210 );
13211 assert!(matches!(
13212 breakers.admit("prov"),
13213 RouteBindAdmission::Admitted { probe: true, .. }
13214 ));
13215 }
13216 let breakers = RouteBindBreakers::default();
13217 let admit = || match breakers.admit("prov") {
13218 RouteBindAdmission::Admitted { guard, .. } => guard,
13219 _ => panic!("relay admitted"),
13220 };
13221 admit().record_timeout(1, Duration::ZERO);
13222 let mut old_probe = admit();
13223 breakers.reset_for_new_module_connection("prov");
13224 admit().record_timeout(1, Duration::ZERO);
13225 let _new_probe = admit();
13226 old_probe.record_inconclusive();
13227 assert!(matches!(
13228 breakers.admit("prov"),
13229 RouteBindAdmission::Refused {
13230 probe_in_flight: true,
13231 ..
13232 }
13233 ));
13234 }
13235
13236 #[tokio::test]
13237 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
13238 for candidate in [false, true] {
13239 let registry = Arc::new(Registry::default());
13240 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13241 [("vault".to_string(), true), ("squatter".to_string(), true)],
13242 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13243 );
13244 let conn = ConnectionId::new(77);
13245 let (ctx, mut rx) = route_ctx(conn);
13246 let initial = capability_manifest("squatter", &[], &[]);
13247 if candidate {
13248 registry
13249 .register_candidate_with_control_ops(
13250 initial.clone(),
13251 PROTOCOL_VERSION,
13252 conn,
13253 module_baseline_control_ops(),
13254 )
13255 .unwrap();
13256 handler
13257 .forwarding
13258 .register_candidate_module_connection(
13259 conn,
13260 "squatter".to_string(),
13261 PROTOCOL_VERSION,
13262 manifest_concurrency(&initial),
13263 ctx.egress.clone(),
13264 )
13265 .unwrap();
13266 } else {
13267 hello_via_sink(
13268 &handler,
13269 &ctx,
13270 &mut rx,
13271 hello_frame_with_manifest(initial.clone(), 1),
13272 )
13273 .await;
13274 }
13275 let response = handler
13276 .handle_control_frame(
13277 &ctx,
13278 catalog_update_with_capabilities_frame(
13279 2,
13280 capability_manifest("squatter", &["credentials-provider/v1"], &[])
13281 .capabilities
13282 .unwrap(),
13283 ),
13284 )
13285 .await
13286 .unwrap();
13287 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13288 assert_eq!(
13289 registry
13290 .get_module_by_connection(conn)
13291 .unwrap()
13292 .unwrap()
13293 .manifest,
13294 initial
13295 );
13296 }
13297 }
13298
13299 #[test]
13300 #[cfg(not(feature = "dev-desktop-approval"))]
13301 fn server_describe_default_build_advertises_no_config_capability() {
13302 let assert_default = |handler: &ControlHandler| {
13303 let request = Frame::build(
13304 FrameType::Request,
13305 control_flags(),
13306 0,
13307 0,
13308 79,
13309 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13310 )
13311 .unwrap();
13312 let responses = handler.handle_server_describe(request).unwrap();
13313 assert_eq!(responses.len(), 1);
13314 let response: Value = serde_json::from_slice(&responses[0].body).unwrap();
13315 assert_eq!(response["op"], "server.describe");
13316 let capabilities = response["capabilities"]
13317 .as_array()
13318 .expect("capability array");
13319 assert!(
13320 !capabilities.is_empty(),
13321 "exercise an actual describe response, not an empty fixture"
13322 );
13323 assert!(
13324 capabilities
13325 .iter()
13326 .all(|capability| !capability.as_str().unwrap().starts_with("config.")),
13327 "a default build must not announce configuration authority: {capabilities:?}"
13328 );
13329 };
13330 assert_default(&ControlHandler::new(Arc::new(Registry::default())));
13331 #[cfg(unix)]
13332 {
13333 let (_rig, _owner, _supervisor, handler, _spec) =
13337 crate::config_gate::runtime::RuntimeOwner::unactivated_supervisor_fixture_for_test(
13338 );
13339 assert_default(&handler);
13340 }
13341 }
13342
13343 #[test]
13344 fn server_describe_surfaces_required_capability_verdict_fields() {
13345 let registry = Arc::new(Registry::default());
13346 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13347 [
13348 ("consumer".to_string(), true),
13349 ("provider".to_string(), false),
13350 ],
13351 BTreeMap::new(),
13352 );
13353 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13354 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13355 provides: Vec::new(),
13356 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13357 capability: "credentials-provider/v1".to_string(),
13358 need: subc_protocol::manifest::CapabilityNeed::Required,
13359 }],
13360 must_never_reach: Vec::new(),
13361 });
13362 let hello = Frame::build(
13363 FrameType::Hello,
13364 control_flags(),
13365 0,
13366 0,
13367 78,
13368 serde_json::to_vec(&ModuleHelloBody {
13369 manifest: consumer,
13370 protocol_ver: PROTOCOL_VERSION,
13371 control_ops: None,
13372 launch_nonce: None,
13373 })
13374 .expect("HELLO serializes"),
13375 )
13376 .expect("HELLO frame builds");
13377 handler
13378 .handle_control(ConnectionId::new(78), hello)
13379 .expect("consumer registers");
13380 let describe = Frame::build(
13381 FrameType::Request,
13382 control_flags(),
13383 0,
13384 0,
13385 79,
13386 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
13387 .expect("request serializes"),
13388 )
13389 .expect("describe frame builds");
13390 let response = handler
13391 .handle_server_describe(describe)
13392 .expect("server.describe succeeds");
13393 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
13394 let requirement = &rendered["capability_requirements"][0];
13395 assert_eq!(requirement["consumer"], "consumer");
13396 assert_eq!(requirement["verdict"], "never_provided");
13397 assert_eq!(requirement["episode_seq"], 1);
13398 assert_eq!(requirement["config_satisfiable"], false);
13399 assert_eq!(requirement["runtime_available"], false);
13400 assert!(requirement["detail"]
13401 .as_str()
13402 .expect("detail string")
13403 .contains("credentials-provider/v1"));
13404 }
13405
13406 fn deregister_required_provider(
13412 daemon_shutdown_begun: bool,
13413 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
13414 const CAPABILITY: &str = "project-identity/v1";
13415 let registry = Arc::new(Registry::default());
13416 let daemon_shutdown = DaemonShutdownFlag::default();
13417 let handler = ControlHandler::new(Arc::clone(®istry))
13418 .with_capability_config(
13419 [
13420 ("consumer".to_string(), true),
13421 ("provider".to_string(), true),
13422 ],
13423 BTreeMap::new(),
13424 )
13425 .with_daemon_shutdown_flag(daemon_shutdown.clone());
13426 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13427 consumer.capabilities = Some(CapabilityDeclarations {
13428 provides: Vec::new(),
13429 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13430 capability: CAPABILITY.to_string(),
13431 need: subc_protocol::manifest::CapabilityNeed::Required,
13432 }],
13433 must_never_reach: Vec::new(),
13434 });
13435 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
13436 let provider_conn = ConnectionId::new(182);
13437 for (conn, manifest, corr) in [
13438 (ConnectionId::new(181), consumer, 181),
13439 (provider_conn, provider, 182),
13440 ] {
13441 let hello = handler
13442 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
13443 .expect("module registers");
13444 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13445 }
13446 assert_eq!(
13447 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13448 Some(CapabilityVerdict::Provided),
13449 "the registered provider must satisfy the requirement first"
13450 );
13451 if daemon_shutdown_begun {
13452 daemon_shutdown.set_for_test();
13453 }
13454
13455 let capture = EventCapture::default();
13456 let _subscriber =
13457 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13458 handler
13459 .cleanup_connection(provider_conn)
13460 .expect("provider deregisters");
13461 let events = capture
13462 .events()
13463 .into_iter()
13464 .filter(|event| {
13465 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
13466 })
13467 .collect();
13468 (
13469 events,
13470 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13471 )
13472 }
13473
13474 #[test]
13475 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
13476 let (events, verdict) = deregister_required_provider(true);
13477 assert_eq!(
13478 verdict,
13479 Some(CapabilityVerdict::NeverProvided),
13480 "shutdown must not change the verdict route.open refusals read"
13481 );
13482 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13483 let event = &events[0];
13484 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
13485 assert_eq!(
13486 event.fields.get("cause").map(String::as_str),
13487 Some("\"daemon_shutdown\""),
13488 "{event:?}"
13489 );
13490 assert_eq!(
13491 event.fields.get("verdict").map(String::as_str),
13492 Some("\"never_provided\""),
13493 "{event:?}"
13494 );
13495 }
13496
13497 #[test]
13498 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
13499 let (events, verdict) = deregister_required_provider(false);
13500 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
13501 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13502 let event = &events[0];
13503 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
13504 assert_eq!(event.fields.get("cause"), None, "{event:?}");
13505 assert_eq!(
13506 event.fields.get("verdict").map(String::as_str),
13507 Some("\"never_provided\""),
13508 "{event:?}"
13509 );
13510 }
13511
13512 #[test]
13513 fn catalog_list_omits_capabilities_for_legacy_manifest() {
13514 let registry = Arc::new(Registry::default());
13515 let handler = ControlHandler::new(Arc::clone(®istry));
13516 let hello = handler
13517 .handle_control(
13518 ConnectionId::new(101),
13519 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
13520 )
13521 .expect("legacy HELLO registers");
13522 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13523
13524 let request = Frame::build(
13525 FrameType::Request,
13526 control_flags(),
13527 0,
13528 0,
13529 102,
13530 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
13531 .expect("catalog request serializes"),
13532 )
13533 .expect("catalog request frame builds");
13534 let response = handler
13535 .handle_catalog_list(request, None)
13536 .expect("catalog list succeeds");
13537 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
13538 assert!(
13539 body["modules"][0].get("capabilities").is_none(),
13540 "legacy manifest must retain an absent capabilities field on catalog.list"
13541 );
13542 }
13543
13544 #[test]
13545 fn hello_ack_omits_storage_when_no_storage_config() {
13546 let registry = Arc::new(Registry::default());
13547 let handler = ControlHandler::new(Arc::clone(®istry));
13548 let responses = handler
13549 .handle_control(
13550 ConnectionId::new(1),
13551 hello_frame("aft", PROTOCOL_VERSION, 7),
13552 )
13553 .unwrap();
13554 let ack = parse_ack(&responses[0]);
13555 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
13556 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
13557 }
13558
13559 #[tokio::test]
13560 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
13561 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
13562 let registry = Arc::new(Registry::default());
13563 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
13564 let responses = handler
13565 .handle_control(
13566 ConnectionId::new(1),
13567 hello_frame("aft", PROTOCOL_VERSION, 7),
13568 )
13569 .unwrap();
13570 let ack = parse_ack(&responses[0]);
13571 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
13572
13573 let described = handler
13574 .handle_control_frame(
13575 &route_ctx(ConnectionId::new(2)).0,
13576 Frame::build(
13577 FrameType::Request,
13578 control_flags(),
13579 0,
13580 0,
13581 9,
13582 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13583 )
13584 .unwrap(),
13585 )
13586 .await
13587 .unwrap();
13588 let ClientControlResponse::ServerDescribe { machine_id, .. } =
13589 serde_json::from_slice(&described[0].body).unwrap()
13590 else {
13591 panic!("server.describe answered with another shape");
13592 };
13593 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
13594 }
13595
13596 #[test]
13597 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
13598 let registry = Arc::new(Registry::default());
13601 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
13602 crate::daemon_config::StorageConfig::Sqlite {
13603 data_home: std::path::PathBuf::from("/data"),
13604 },
13605 ));
13606
13607 let responses = handler
13608 .handle_control(
13609 ConnectionId::new(1),
13610 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
13611 )
13612 .unwrap();
13613 let ack = parse_ack(&responses[0]);
13614 assert_eq!(
13615 ack.storage,
13616 Some(serde_json::json!({
13617 "module_id": "alfonso-routing",
13618 "storage_namespace": "default",
13619 "isolation": { "kind": "module" },
13620 "backend": {
13621 "backend": "sqlite",
13622 "path": "/data/cortexkit/alfonso-routing/store.db"
13623 }
13624 })),
13625 "the delivered descriptor is the module's own sqlite store path"
13626 );
13627 }
13628
13629 #[test]
13630 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
13631 let registry = Arc::new(Registry::default());
13632 let handler = ControlHandler::new(Arc::clone(®istry));
13633 let conn = ConnectionId::new(1);
13634 let responses = handler
13635 .handle_control(
13636 conn,
13637 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13638 )
13639 .unwrap();
13640 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
13641 let registration = registry.get_module("aft").unwrap().unwrap();
13642 assert_eq!(registration.control_ops, module_baseline_control_ops());
13643
13644 let frame =
13645 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
13646 assert!(handler
13647 .guard_module_control_op(&frame, "aft", "route.bind")
13648 .unwrap()
13649 .is_none());
13650 let error = handler
13651 .guard_module_control_op(&frame, "aft", "test.synthetic")
13652 .unwrap()
13653 .expect("synthetic ungranted op should be rejected");
13654 assert_eq!(error.header.ty, FrameType::Error);
13655 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
13656 }
13657
13658 #[test]
13659 fn hello_control_ops_some_adds_optional_grants() {
13660 let registry = Arc::new(Registry::default());
13661 let handler = ControlHandler::new(Arc::clone(®istry));
13662 handler
13663 .handle_control(
13664 ConnectionId::new(1),
13665 hello_frame_with_control_ops(
13666 "aft",
13667 PROTOCOL_VERSION,
13668 7,
13669 Some(vec![
13670 "future.synthetic".to_string(),
13671 "route.bind".to_string(),
13672 ]),
13673 ),
13674 )
13675 .unwrap();
13676 let registration = registry.get_module("aft").unwrap().unwrap();
13677 assert_eq!(
13678 registration.control_ops,
13679 vec![
13680 "route.bind".to_string(),
13681 "route.status".to_string(),
13682 "future.synthetic".to_string(),
13683 ]
13684 );
13685 let frame =
13686 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
13687 assert!(handler
13688 .guard_module_control_op(&frame, "aft", "future.synthetic")
13689 .unwrap()
13690 .is_none());
13691 }
13692
13693 #[tokio::test]
13694 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
13695 let registry = Arc::new(Registry::default());
13696 let forwarding = Arc::new(ForwardingTable::default());
13697 let handler =
13698 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13699 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
13700 hello_via_sink(
13701 &handler,
13702 &module_ctx,
13703 &mut module_rx,
13704 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13705 )
13706 .await;
13707
13708 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
13709 let responses = handler
13710 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
13711 .await
13712 .unwrap();
13713 assert_eq!(responses.len(), 1);
13714 assert_eq!(responses[0].header.ty, FrameType::Error);
13715 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
13716 assert!(module_rx.try_recv().is_err());
13717 }
13718
13719 #[tokio::test]
13720 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
13721 let registry = Arc::new(Registry::default());
13722 let forwarding = Arc::new(ForwardingTable::default());
13723 let handler =
13724 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13725 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
13726 hello_via_sink(
13727 &handler,
13728 &module_ctx,
13729 &mut module_rx,
13730 hello_frame_with_control_ops(
13731 "aft",
13732 PROTOCOL_VERSION,
13733 7,
13734 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13735 ),
13736 )
13737 .await;
13738
13739 let project_root = unique_project_root("demux");
13740 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
13741 let route_handler = handler.clone();
13742 let route_task = tokio::spawn(async move {
13743 route_handler
13744 .handle_control_frame(
13745 &route_client_ctx,
13746 route_open_frame(100, "aft", project_root),
13747 )
13748 .await
13749 .unwrap()
13750 });
13751 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13752 .await
13753 .unwrap()
13754 .unwrap();
13755 assert!(matches!(
13756 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
13757 ModuleControlRequest::RouteBind { .. }
13758 ));
13759
13760 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
13761 let health_handler = handler.clone();
13762 let health_task = tokio::spawn(async move {
13763 health_handler
13764 .handle_control_frame(
13765 &health_client_ctx,
13766 supervisor_health_probe_frame(101, "aft"),
13767 )
13768 .await
13769 .unwrap()
13770 });
13771 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13772 .await
13773 .unwrap()
13774 .unwrap();
13775 assert_eq!(
13776 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13777 ModuleControlRequest::HealthCheck {}
13778 );
13779
13780 handler
13781 .handle_control_frame(
13782 &module_ctx,
13783 health_response(health_frame.header.corr, HealthStatus::Degraded),
13784 )
13785 .await
13786 .unwrap();
13787 let health_response = health_task.await.unwrap();
13788 assert_eq!(health_response.len(), 1);
13789 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13790 ClientControlResponse::SupervisorHealthProbe {
13791 module_id,
13792 status,
13793 detail,
13794 metrics,
13795 } => {
13796 assert_eq!(module_id, "aft");
13797 assert_eq!(status, HealthStatus::Degraded);
13798 assert_eq!(detail.as_deref(), Some("warming"));
13799 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
13800 }
13801 other => panic!("unexpected health response: {other:?}"),
13802 }
13803
13804 handler
13805 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13806 .await
13807 .unwrap();
13808 let route_response = route_task.await.unwrap();
13809 assert!(route_response.is_empty());
13810 let published = route_client_rx.recv().await.unwrap();
13811 assert!(matches!(
13812 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13813 ClientControlResponse::RouteOpen { .. }
13814 ));
13815 }
13816
13817 async fn relay_route_open(
13822 handler: &ControlHandler,
13823 client_connection: ConnectionId,
13824 client_egress: &FrameSink,
13825 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13826 corr: u64,
13827 module_id: &str,
13828 project_root_label: &str,
13829 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
13830 let ctx = RouteCtx {
13831 connection_id: client_connection,
13832 egress: client_egress.clone(),
13833 };
13834 let handler = handler.clone();
13835 let project_root = unique_project_root(project_root_label);
13836 let module_id = module_id.to_string();
13837 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
13838 let task = tokio::spawn(async move {
13839 let _guard = tracing::dispatcher::set_default(&dispatch);
13840 handler
13841 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
13842 .await
13843 .unwrap()
13844 });
13845 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13846 .await
13847 .expect("module receives the relayed route.bind")
13848 .expect("module egress is open");
13849 (task, bind.frame)
13850 }
13851
13852 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
13853 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
13854 ModuleControlRequest::RouteBind {
13855 route_channel,
13856 epoch,
13857 ..
13858 } => (route_channel, epoch),
13859 other => panic!("expected a route.bind request, got {other:?}"),
13860 }
13861 }
13862
13863 fn published_route(frame: &Frame) -> (u16, u32) {
13864 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
13865 ClientControlResponse::RouteOpen {
13866 route_channel,
13867 route_epoch,
13868 ..
13869 } => (route_channel, route_epoch),
13870 other => panic!("expected a route.open response, got {other:?}"),
13871 }
13872 }
13873
13874 #[tokio::test]
13894 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
13895 let registry = Arc::new(Registry::default());
13896 let forwarding = Arc::new(ForwardingTable::default());
13897 let handler =
13898 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13899
13900 let module_connection = ConnectionId::new(30);
13901 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13902 hello_via_sink(
13903 &handler,
13904 &module_ctx,
13905 &mut module_rx,
13906 hello_frame("aft", PROTOCOL_VERSION, 7),
13907 )
13908 .await;
13909
13910 let dying_client = ConnectionId::new(31);
13911 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
13912
13913 let (first_task, first_bind) = relay_route_open(
13916 &handler,
13917 dying_client,
13918 &dying_ctx.egress,
13919 &mut module_rx,
13920 100,
13921 "aft",
13922 "closing-first",
13923 )
13924 .await;
13925 handler
13926 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
13927 .await
13928 .unwrap();
13929 assert!(first_task.await.unwrap().is_empty());
13930 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
13931
13932 let (second_task, second_bind) = relay_route_open(
13934 &handler,
13935 dying_client,
13936 &dying_ctx.egress,
13937 &mut module_rx,
13938 101,
13939 "aft",
13940 "closing-second",
13941 )
13942 .await;
13943 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
13944
13945 assert!(forwarding
13948 .escalate_client_delivery_failure(
13949 dying_client,
13950 first_channel,
13951 first_epoch,
13952 CloseReason::new(
13953 "module_to_client_delivery_failed",
13954 "client egress refused a module frame",
13955 ),
13956 crate::forwarding::UndeliveredFrame {
13957 module_id: None,
13958 sink: &dying_ctx.egress,
13959 },
13960 )
13961 .unwrap());
13962 assert!(!dying_ctx.egress.is_closed());
13963
13964 let ack = handler
13966 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
13967 .await;
13968 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13969 if module_loop_error.is_some() {
13970 handler.cleanup_connection(module_connection).unwrap();
13974 }
13975 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13979 .await
13980 .ok()
13981 .flatten();
13982
13983 assert!(
13985 registry
13986 .get_module_by_connection(module_connection)
13987 .unwrap()
13988 .is_some(),
13989 "one client's closing connection ended the shared module connection: \
13990 {module_loop_error:?}"
13991 );
13992 let cotenant = ConnectionId::new(32);
13994 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13995 let (cotenant_task, cotenant_bind) = relay_route_open(
13996 &handler,
13997 cotenant,
13998 &cotenant_ctx.egress,
13999 &mut module_rx,
14000 102,
14001 "aft",
14002 "closing-cotenant",
14003 )
14004 .await;
14005 handler
14006 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
14007 .await
14008 .unwrap();
14009 assert!(cotenant_task.await.unwrap().is_empty());
14010 let (cotenant_channel, cotenant_epoch) =
14011 published_route(&cotenant_rx.recv().await.unwrap());
14012 assert!(matches!(
14013 forwarding
14014 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14015 .unwrap(),
14016 DataRoute::Client(DataRouteState::Bound(_))
14017 ));
14018
14019 let goodbye = post_ack_module_frame
14022 .expect("module receives a GOODBYE for the abandoned route channel");
14023 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14024 assert_eq!(goodbye.header.channel, abandoned_channel);
14025 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14026
14027 assert!(dying_rx.try_recv().is_err());
14031 let second_response = second_task.await.unwrap();
14032 assert_eq!(second_response.len(), 1);
14033 assert_eq!(
14034 parse_error(&second_response[0])["code"],
14035 "target_unavailable"
14036 );
14037 }
14038
14039 async fn catalog_snapshot(
14042 handler: &ControlHandler,
14043 client_ctx: &RouteCtx,
14044 corr: u64,
14045 ) -> (u64, Vec<(String, bool)>) {
14046 let body =
14047 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
14048 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
14049 let reply = handler
14050 .handle_control_frame(client_ctx, request)
14051 .await
14052 .unwrap()
14053 .pop()
14054 .unwrap();
14055 let ClientControlResponse::CatalogList {
14056 generation,
14057 modules,
14058 ..
14059 } = serde_json::from_slice(&reply.body).unwrap()
14060 else {
14061 panic!("not a catalog.list reply");
14062 };
14063 (
14064 generation,
14065 modules
14066 .into_iter()
14067 .map(|entry| (entry.module_id, entry.ready))
14068 .collect(),
14069 )
14070 }
14071
14072 fn hello_held_before_forwarding() -> (
14078 ControlHandler,
14079 Arc<std::sync::Barrier>,
14080 std::thread::JoinHandle<Vec<Frame>>,
14081 RouteCtx,
14082 mpsc::Receiver<crate::router::OutboundFrame>,
14083 ) {
14084 let hold = Arc::new(std::sync::Barrier::new(2));
14085 let handler = ControlHandler::with_forwarding(
14086 Arc::new(Registry::default()),
14087 Arc::new(ForwardingTable::default()),
14088 )
14089 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14090 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14091 let hello = {
14092 let handler = handler.clone();
14093 let module_ctx = module_ctx.clone();
14094 std::thread::spawn(move || {
14095 handler
14096 .handle_hello(
14097 module_ctx.connection_id,
14098 Some(module_ctx.egress.clone()),
14099 hello_frame("aft", PROTOCOL_VERSION, 7),
14100 )
14101 .unwrap()
14102 })
14103 };
14104 (handler, hold, hello, module_ctx, module_rx)
14105 }
14106
14107 fn lookup_during_hello_publication<T: Send + 'static>(
14113 registry: &Registry,
14114 hold: &std::sync::Barrier,
14115 lookup: impl FnOnce() -> T + Send + 'static,
14116 ) -> T {
14117 let attempted = registry.observe_next_lock_attempt();
14118 let reader = std::thread::spawn(lookup);
14119 if attempted.recv().unwrap() {
14120 let result = reader.join().unwrap();
14121 hold.wait();
14122 result
14123 } else {
14124 hold.wait();
14125 reader.join().unwrap()
14126 }
14127 }
14128
14129 #[test]
14130 fn catalog_read_after_hello_ack_waits_for_publication() {
14131 let registry = Arc::new(Registry::default());
14132 let hold = Arc::new(std::sync::Barrier::new(2));
14133 let mut handler = ControlHandler::with_forwarding(
14134 Arc::clone(®istry),
14135 Arc::new(ForwardingTable::default()),
14136 );
14137 handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
14138 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14139 let hello = {
14140 let handler = handler.clone();
14141 std::thread::spawn(move || {
14142 handler
14143 .handle_hello(
14144 module_ctx.connection_id,
14145 Some(module_ctx.egress),
14146 hello_frame("aft", PROTOCOL_VERSION, 7),
14147 )
14148 .unwrap()
14149 })
14150 };
14151 hold.wait();
14152 assert_eq!(
14153 module_rx.try_recv().unwrap().frame.header.ty,
14154 FrameType::HelloAck
14155 );
14156 let replies = lookup_during_hello_publication(®istry, &hold, move || {
14157 let request = Frame::build(
14160 FrameType::Request,
14161 control_flags(),
14162 0,
14163 0,
14164 300,
14165 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
14166 )
14167 .unwrap();
14168 handler.handle_catalog_list(request, None).unwrap()
14169 });
14170 assert!(hello.join().unwrap().is_empty());
14171 let ClientControlResponse::CatalogList {
14172 generation,
14173 modules,
14174 ..
14175 } = serde_json::from_slice(&replies[0].body).unwrap()
14176 else {
14177 panic!("not a catalog.list reply");
14178 };
14179 assert_eq!(
14180 modules
14181 .iter()
14182 .map(|entry| entry.module_id.as_str())
14183 .collect::<Vec<_>>(),
14184 vec!["aft"]
14185 );
14186 assert_eq!(generation, 1);
14187 }
14188
14189 #[tokio::test]
14198 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
14199 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
14200 let client = ConnectionId::new(41);
14201 let (client_ctx, mut client_rx) = route_ctx(client);
14202
14203 hold.wait();
14204 assert!(handler.registry.get_module("aft").unwrap().is_some());
14206 assert!(!handler
14207 .forwarding
14208 .has_live_module_connection("aft")
14209 .unwrap());
14210 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
14211 assert!(
14212 listed.is_empty(),
14213 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
14214 );
14215 let refused = handler
14216 .handle_control_frame(
14217 &client_ctx,
14218 route_open_frame(301, "aft", unique_project_root("listed-window")),
14219 )
14220 .await
14221 .unwrap();
14222 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
14223 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
14224 let duplicate = handler
14225 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
14226 .await
14227 .unwrap();
14228 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14229
14230 hold.wait();
14231 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14232 let ack = module_rx.try_recv().unwrap().frame;
14233 assert_eq!(ack.header.ty, FrameType::HelloAck);
14234 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14235 assert_eq!(listed, vec![("aft".to_string(), true)]);
14236
14237 let (open_task, bind) = relay_route_open(
14238 &handler,
14239 client,
14240 &client_ctx.egress,
14241 &mut module_rx,
14242 303,
14243 "aft",
14244 "listed-after",
14245 )
14246 .await;
14247 assert!(handler
14248 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14249 .await
14250 .unwrap()
14251 .is_empty());
14252 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
14253 published_route(&client_rx.try_recv().unwrap().frame);
14254 }
14255
14256 #[tokio::test]
14263 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
14264 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
14265 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14266 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14267
14268 hold.wait();
14269 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
14270 assert!(listed.is_empty());
14271 assert_eq!(
14272 during, before,
14273 "the generation moved while the module was not listed"
14274 );
14275
14276 hold.wait();
14277 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14278 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14279 assert_eq!(listed, vec![("aft".to_string(), true)]);
14280 assert_eq!(
14281 after,
14282 before + 1,
14283 "the module appeared without exactly one generation bump"
14284 );
14285 }
14286
14287 #[tokio::test]
14292 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
14293 let registry = Arc::new(Registry::default());
14294 let handler = ControlHandler::with_forwarding(
14295 Arc::clone(®istry),
14296 Arc::new(ForwardingTable::default()),
14297 );
14298 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14299 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14300 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14301 drop(module_rx);
14302
14303 let replies = handler
14304 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
14305 .await
14306 .unwrap();
14307 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
14308 assert!(registry.get_module("aft").unwrap().is_none());
14309 assert!(registry
14310 .get_module_by_connection(module_ctx.connection_id)
14311 .unwrap()
14312 .is_none());
14313 assert_eq!(
14314 catalog_snapshot(&handler, &client_ctx, 301).await,
14315 (before, Vec::new())
14316 );
14317 }
14318
14319 async fn route_open_held_after_bind_send() -> (
14323 ControlHandler,
14324 Arc<tokio::sync::Barrier>,
14325 RouteCtx,
14326 mpsc::Receiver<crate::router::OutboundFrame>,
14327 ) {
14328 let hold = Arc::new(tokio::sync::Barrier::new(2));
14329 let handler = ControlHandler::with_forwarding(
14330 Arc::new(Registry::default()),
14331 Arc::new(ForwardingTable::default()),
14332 )
14333 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
14334 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14335 hello_via_sink(
14336 &handler,
14337 &module_ctx,
14338 &mut module_rx,
14339 hello_frame("aft", PROTOCOL_VERSION, 7),
14340 )
14341 .await;
14342 (handler, hold, module_ctx, module_rx)
14343 }
14344
14345 #[tokio::test]
14352 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
14353 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14354 let client = ConnectionId::new(41);
14355 let (client_ctx, mut client_rx) = route_ctx(client);
14356
14357 let (open_task, bind) = relay_route_open(
14358 &handler,
14359 client,
14360 &client_ctx.egress,
14361 &mut module_rx,
14362 200,
14363 "aft",
14364 "acked-before-mark",
14365 )
14366 .await;
14367 let (module_channel, module_epoch) = route_bind_channel(&bind);
14368 hold.wait().await;
14370
14371 assert!(handler
14373 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14374 .await
14375 .unwrap()
14376 .is_empty());
14377 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
14378
14379 hold.wait().await;
14380 assert!(
14381 open_task.await.unwrap().is_empty(),
14382 "route.open answers accepted"
14383 );
14384
14385 match module_rx.try_recv() {
14388 Err(mpsc::error::TryRecvError::Empty) => {}
14389 Ok(outbound) => panic!(
14390 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
14391 outbound.frame.header.ty,
14392 outbound.frame.header.channel,
14393 outbound.frame.header.epoch
14394 ),
14395 Err(err) => panic!("module egress closed: {err}"),
14396 }
14397
14398 let router = Router::with_control_handler(Arc::new(handler.clone()));
14401 let request = Frame::build(
14402 FrameType::Request,
14403 Flags::new(true, Priority::Interactive, false),
14404 client_channel,
14405 client_epoch,
14406 201,
14407 b"first request".to_vec(),
14408 )
14409 .unwrap();
14410 router
14411 .route_for_connection(&client_ctx, request)
14412 .await
14413 .unwrap();
14414 let delivered = module_rx.try_recv().unwrap().frame;
14415 assert_eq!(delivered.header.ty, FrameType::Request);
14416 assert_eq!(
14417 (delivered.header.channel, delivered.header.epoch),
14418 (module_channel, module_epoch)
14419 );
14420 assert_eq!(delivered.body, b"first request");
14421 }
14422
14423 #[tokio::test]
14429 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
14430 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14431 let client = ConnectionId::new(42);
14432 let (client_ctx, _client_rx) = route_ctx(client);
14433
14434 let (open_task, bind) = relay_route_open(
14435 &handler,
14436 client,
14437 &client_ctx.egress,
14438 &mut module_rx,
14439 300,
14440 "aft",
14441 "released-before-mark",
14442 )
14443 .await;
14444 let (module_channel, module_epoch) = route_bind_channel(&bind);
14445 hold.wait().await;
14446
14447 handler.cleanup_connection(client).unwrap();
14448 assert!(
14449 module_rx.try_recv().is_err(),
14450 "client teardown must not GOODBYE a bind route.open is still queueing"
14451 );
14452
14453 hold.wait().await;
14454 let response = open_task.await.unwrap();
14455 assert_eq!(response.len(), 1);
14456 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14457
14458 let goodbye = module_rx
14459 .try_recv()
14460 .expect("module is told to drop the binding the daemon abandoned")
14461 .frame;
14462 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14463 assert_eq!(
14464 (goodbye.header.channel, goodbye.header.epoch),
14465 (module_channel, module_epoch)
14466 );
14467 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14468 }
14469
14470 #[tokio::test]
14475 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
14476 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14477 let client = ConnectionId::new(43);
14478 let (client_ctx, client_rx) = route_ctx(client);
14479
14480 let (open_task, bind) = relay_route_open(
14481 &handler,
14482 client,
14483 &client_ctx.egress,
14484 &mut module_rx,
14485 400,
14486 "aft",
14487 "gone-client-before-mark",
14488 )
14489 .await;
14490 let (module_channel, module_epoch) = route_bind_channel(&bind);
14491 hold.wait().await;
14492
14493 drop(client_rx);
14494 assert!(client_ctx.egress.is_closed());
14495 handler
14496 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14497 .await
14498 .unwrap();
14499
14500 hold.wait().await;
14501 let response = open_task.await.unwrap();
14502 assert_eq!(response.len(), 1);
14503 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14504
14505 let goodbye = module_rx
14506 .try_recv()
14507 .expect("module is told to drop the binding it acked")
14508 .frame;
14509 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14510 assert_eq!(
14511 (goodbye.header.channel, goodbye.header.epoch),
14512 (module_channel, module_epoch)
14513 );
14514 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14515 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14516 }
14517
14518 #[test]
14525 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
14526 let handler = ControlHandler::default();
14527 let module_connection = ConnectionId::new(30);
14528 let client_connection = ConnectionId::new(31);
14529
14530 handler
14531 .refuse_to_end_module_connection_for_a_client(
14532 module_connection,
14533 77,
14534 ForwardingError::ConnectionClosing {
14535 connection_id: client_connection,
14536 },
14537 )
14538 .expect("a closing client must never end the module connection");
14539
14540 assert!(matches!(
14541 handler.refuse_to_end_module_connection_for_a_client(
14542 module_connection,
14543 78,
14544 ForwardingError::ConnectionClosing {
14545 connection_id: module_connection,
14546 },
14547 ),
14548 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
14549 connection_id
14550 })) if connection_id == module_connection
14551 ));
14552 assert!(matches!(
14553 handler.refuse_to_end_module_connection_for_a_client(
14554 module_connection,
14555 79,
14556 ForwardingError::Poisoned,
14557 ),
14558 Err(RouterError::Forwarding(ForwardingError::Poisoned))
14559 ));
14560 assert!(matches!(
14561 handler.refuse_to_end_module_connection_for_a_client(
14562 module_connection,
14563 80,
14564 ForwardingError::StaleModuleEndpoint,
14565 ),
14566 Err(RouterError::Forwarding(
14567 ForwardingError::StaleModuleEndpoint
14568 ))
14569 ));
14570 }
14571
14572 #[tokio::test]
14585 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
14586 let registry = Arc::new(Registry::default());
14587 let forwarding = Arc::new(ForwardingTable::default());
14588 let supervisor = SupervisorHandle::new();
14589 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14590 let handler =
14591 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14592 .with_supervisor(supervisor);
14593
14594 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14595 hello_via_sink(
14596 &handler,
14597 &target_ctx,
14598 &mut target_rx,
14599 hello_frame("target", PROTOCOL_VERSION, 1),
14600 )
14601 .await;
14602
14603 let wrong_nonce = handler
14607 .handle_control_frame(
14608 &route_ctx(ConnectionId::new(91)).0,
14609 route_open_frame_with_admission_facts(
14610 20,
14611 "target",
14612 unique_project_root("admission-facts"),
14613 Some(subc_control::ConsumerIdentity {
14614 module_id: "fed".to_string(),
14615 launch_nonce: "not-the-real-nonce".to_string(),
14616 }),
14617 None,
14618 ),
14619 )
14620 .await
14621 .unwrap();
14622 assert_eq!(
14623 parse_error(&wrong_nonce[0])["code"],
14624 "bad_consumer_identity",
14625 "a mismatched launch nonce must be refused, not stamped Reserved"
14626 );
14627
14628 let never_spawned = handler
14632 .handle_control_frame(
14633 &route_ctx(ConnectionId::new(92)).0,
14634 route_open_frame_with_admission_facts(
14635 21,
14636 "target",
14637 unique_project_root("admission-facts"),
14638 Some(subc_control::ConsumerIdentity {
14639 module_id: "never-spawned".to_string(),
14640 launch_nonce: "any-nonce".to_string(),
14641 }),
14642 None,
14643 ),
14644 )
14645 .await
14646 .unwrap();
14647 assert_eq!(
14648 parse_error(&never_spawned[0])["code"],
14649 "bad_consumer_identity",
14650 "an unspawned module_id must be refused rather than accepted for lack of a record"
14651 );
14652 }
14653
14654 #[tokio::test]
14677 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
14678 let registry = Arc::new(Registry::default());
14679 let forwarding = Arc::new(ForwardingTable::default());
14680 let supervisor = SupervisorHandle::new();
14681 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14682 let handler =
14683 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14684 .with_supervisor(supervisor);
14685
14686 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
14687 hello_via_sink(
14688 &handler,
14689 &target_ctx,
14690 &mut target_rx,
14691 hello_frame("target", PROTOCOL_VERSION, 1),
14692 )
14693 .await;
14694
14695 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
14696 let route_handler = handler.clone();
14697 let route_task = tokio::spawn(async move {
14698 route_handler
14699 .handle_control_frame(
14700 &client_ctx,
14701 route_open_frame_with_admission_facts(
14702 30,
14703 "target",
14704 unique_project_root("admission-facts"),
14705 Some(subc_control::ConsumerIdentity {
14706 module_id: "fed".to_string(),
14707 launch_nonce: "fed-nonce".to_string(),
14708 }),
14709 None,
14710 ),
14711 )
14712 .await
14713 .unwrap()
14714 });
14715
14716 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14723 .await
14724 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
14725 .expect("module control channel closed before route.bind");
14726 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14727 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14728 panic!("expected route.bind")
14729 };
14730 assert_eq!(
14731 principal,
14732 Some(Principal::Reserved {
14733 module_id: "fed".to_string()
14734 }),
14735 "a correctly attested consumer must be stamped Reserved for its own id"
14736 );
14737
14738 handler
14739 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14740 .await
14741 .unwrap();
14742 assert!(route_task.await.unwrap().is_empty());
14743 assert!(
14744 matches!(
14745 serde_json::from_slice::<ClientControlResponse>(
14746 &client_rx.recv().await.unwrap().body
14747 )
14748 .unwrap(),
14749 ClientControlResponse::RouteOpen { .. }
14750 ),
14751 "the route must actually open, not merely avoid an error"
14752 );
14753 }
14754
14755 #[tokio::test(start_paused = true)]
14756 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
14757 let registry = Arc::new(Registry::default());
14758 let forwarding = Arc::new(ForwardingTable::default());
14759 let supervisor = SupervisorHandle::new();
14760 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14761 let handler =
14762 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14763 .with_supervisor(supervisor);
14764
14765 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
14766 hello_via_sink(
14767 &handler,
14768 &target_ctx,
14769 &mut target_rx,
14770 hello_frame("target", PROTOCOL_VERSION, 1),
14771 )
14772 .await;
14773
14774 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
14775 let direct_handler = handler.clone();
14776 let direct_open = tokio::spawn(async move {
14777 direct_handler
14778 .handle_control_frame(
14779 &direct_ctx,
14780 route_open_frame(2, "target", unique_project_root("route-census-direct")),
14781 )
14782 .await
14783 .unwrap()
14784 });
14785 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14786 .await
14787 .expect("no direct route.bind within 5s")
14788 .expect("target control channel closed before direct route.bind");
14789 handler
14790 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
14791 .await
14792 .unwrap();
14793 assert!(direct_open.await.unwrap().is_empty());
14794 let _ = direct_rx.recv().await.unwrap();
14795
14796 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
14797 let reserved_handler = handler.clone();
14798 let reserved_open = tokio::spawn(async move {
14799 reserved_handler
14800 .handle_control_frame(
14801 &reserved_ctx,
14802 route_open_frame_with_admission_facts(
14803 3,
14804 "target",
14805 unique_project_root("admission-facts"),
14806 Some(ConsumerIdentity {
14807 module_id: "fed".to_string(),
14808 launch_nonce: "fed-nonce".to_string(),
14809 }),
14810 None,
14811 ),
14812 )
14813 .await
14814 .unwrap()
14815 });
14816 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14817 .await
14818 .expect("no reserved route.bind within 5s")
14819 .expect("target control channel closed before reserved route.bind");
14820 handler
14821 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
14822 .await
14823 .unwrap();
14824 assert!(reserved_open.await.unwrap().is_empty());
14825 let _ = reserved_rx.recv().await.unwrap();
14826
14827 forwarding
14828 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
14829 .unwrap();
14830 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
14831 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
14832 module_id: Some("target".to_string()),
14833 })
14834 .unwrap();
14835 let census_frame =
14836 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
14837 let response = handler
14838 .handle_control_frame(&census_ctx, census_frame)
14839 .await
14840 .unwrap()
14841 .pop()
14842 .unwrap();
14843 let actual: Value = serde_json::from_slice(&response.body).unwrap();
14844 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
14845 assert!(matches!(
14846 decoded,
14847 ClientControlResponse::SupervisorRoutes { .. }
14848 ));
14849 let routes = actual["modules"][0]["routes"].as_array().unwrap();
14850 assert_eq!(routes.len(), 2);
14851 assert!(routes.iter().all(|route| route["draining"] == true));
14852 assert!(
14855 routes.iter().all(|route| route["drain_reason"] == "reload"),
14856 "draining routes must name the drain's reason: {routes:?}"
14857 );
14858 assert!(routes.iter().any(|route| {
14859 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
14860 }));
14861 assert!(routes.iter().any(|route| {
14862 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
14863 }));
14864
14865 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14866 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
14867 if std::env::var_os("UPDATE_GOLDEN").is_some() {
14868 std::fs::write(
14869 &golden_path,
14870 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
14871 )
14872 .unwrap();
14873 }
14874 let expected: Value =
14875 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
14876 assert_eq!(actual, expected);
14877 }
14878
14879 async fn query_live_roots(
14880 handler: &ControlHandler,
14881 module_ctx: &RouteCtx,
14882 ) -> ModuleControlResponseToModule {
14883 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
14884 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
14885 let response = handler
14886 .handle_control_frame(module_ctx, frame)
14887 .await
14888 .unwrap()
14889 .pop()
14890 .unwrap();
14891 serde_json::from_slice(&response.body).unwrap()
14892 }
14893
14894 #[tokio::test(start_paused = true)]
14895 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
14896 let registry = Arc::new(Registry::default());
14897 let forwarding = Arc::new(ForwardingTable::default());
14898 let handler = ControlHandler::with_forwarding(registry, forwarding);
14899 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
14900 hello_via_sink(
14901 &handler,
14902 &target_ctx,
14903 &mut target_rx,
14904 hello_frame("target", PROTOCOL_VERSION, 1),
14905 )
14906 .await;
14907 let root = unique_project_root("live-roots-known");
14908 let path = ProjectRootId::from_path_allowing_missing(root.path())
14909 .unwrap()
14910 .as_path()
14911 .to_path_buf();
14912 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
14913 let open_handler = handler.clone();
14914 let opened = tokio::spawn(async move {
14915 open_handler
14916 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
14917 .await
14918 .unwrap()
14919 });
14920 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14921 .await
14922 .unwrap()
14923 .unwrap();
14924 handler
14925 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
14926 .await
14927 .unwrap();
14928 assert!(opened.await.unwrap().is_empty());
14929 let _ = client_rx.recv().await.unwrap();
14930
14931 let root = unique_project_root("live-roots-pending");
14932 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
14933 .unwrap()
14934 .as_path()
14935 .to_path_buf();
14936 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
14937 let open_handler = handler.clone();
14938 let pending = tokio::spawn(async move {
14939 open_handler
14940 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
14941 .await
14942 .unwrap()
14943 });
14944 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14945 .await
14946 .unwrap()
14947 .unwrap();
14948 let actual = query_live_roots(&handler, &target_ctx).await;
14949 let ModuleControlResponseToModule::LiveRoots {
14950 roots,
14951 unknown_root_bindings,
14952 total_bindings,
14953 } = actual
14954 else {
14955 panic!("expected live roots")
14956 };
14957 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
14958 assert_eq!(unknown_root_bindings, 0);
14959 assert_eq!(
14960 roots.len(),
14961 2,
14962 "root-known arm must retain each canonical root"
14963 );
14964 assert_eq!(
14965 total_bindings,
14966 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14967 );
14968 let counts = roots
14969 .iter()
14970 .map(|root| (root.project_root.clone(), root.bound, root.pending))
14971 .collect::<Vec<_>>();
14972 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
14973 expected.sort_by(|a, b| a.0.cmp(&b.0));
14974 assert_eq!(
14975 counts, expected,
14976 "roots must sort by path and count pending separately"
14977 );
14978 handler
14979 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
14980 .await
14981 .unwrap();
14982 assert!(pending.await.unwrap().is_empty());
14983 }
14984
14985 #[tokio::test(start_paused = true)]
14986 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
14987 let forwarding = Arc::new(ForwardingTable::default());
14988 let handler =
14989 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14990 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
14991 hello_via_sink(
14992 &handler,
14993 &target_ctx,
14994 &mut target_rx,
14995 hello_frame("target", PROTOCOL_VERSION, 1),
14996 )
14997 .await;
14998 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
14999 let pending = forwarding
15000 .begin_route_bind_relay_for_test(
15001 client_ctx.connection_id,
15002 client_ctx.egress.clone(),
15003 2,
15004 "target",
15005 )
15006 .unwrap();
15007 forwarding
15008 .complete_pending_relay(
15009 target_ctx.connection_id,
15010 pending.corr,
15011 RouteBindRelayOutcome::Accepted,
15012 )
15013 .unwrap();
15014 let actual = query_live_roots(&handler, &target_ctx).await;
15015 let ModuleControlResponseToModule::LiveRoots {
15016 roots,
15017 unknown_root_bindings,
15018 total_bindings,
15019 } = actual
15020 else {
15021 panic!("expected live roots")
15022 };
15023 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
15024 assert_eq!(
15025 unknown_root_bindings, 1,
15026 "unknown-root arm must not read as no bindings"
15027 );
15028 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
15029 assert_eq!(
15030 total_bindings,
15031 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15032 );
15033 }
15034
15035 #[tokio::test(start_paused = true)]
15043 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
15044 let forwarding = Arc::new(ForwardingTable::default());
15045 let handler =
15046 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
15047 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
15048 let replies = handler
15049 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
15050 .await
15051 .unwrap();
15052 let queued_by_hello = module_rx.len();
15053
15054 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
15055 let open_handler = handler.clone();
15056 let open = tokio::spawn(async move {
15057 open_handler
15058 .handle_control_frame(
15059 &client_ctx,
15060 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
15061 )
15062 .await
15063 .unwrap()
15064 });
15065 let mut spins = 0;
15067 while module_rx.len() == queued_by_hello {
15068 spins += 1;
15069 assert!(spins < 10_000, "route.open never queued a route.bind");
15070 tokio::task::yield_now().await;
15071 }
15072
15073 for reply in replies {
15075 module_ctx.egress.send(reply).await.unwrap();
15076 }
15077
15078 let first = module_rx.recv().await.unwrap().frame;
15079 assert_eq!(
15080 first.header.ty,
15081 FrameType::HelloAck,
15082 "the first frame a registering module reads must be its HELLO_ACK"
15083 );
15084 assert_eq!(first.header.corr, 7);
15085 let second = module_rx.recv().await.unwrap().frame;
15086 assert_eq!(second.header.ty, FrameType::Request);
15087 assert!(
15088 matches!(
15089 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
15090 ModuleControlRequest::RouteBind { .. }
15091 ),
15092 "the route.bind follows the ack"
15093 );
15094 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
15095
15096 handler
15097 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
15098 .await
15099 .unwrap();
15100 assert!(open.await.unwrap().is_empty());
15101 let _ = client_rx.recv().await.unwrap();
15102 }
15103
15104 #[tokio::test(start_paused = true)]
15105 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
15106 let handler = ControlHandler::with_forwarding(
15107 Arc::new(Registry::default()),
15108 Arc::new(ForwardingTable::default()),
15109 );
15110 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
15111 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
15112 hello_via_sink(
15113 &handler,
15114 &first_ctx,
15115 &mut first_rx,
15116 hello_frame("first", PROTOCOL_VERSION, 1),
15117 )
15118 .await;
15119 hello_via_sink(
15120 &handler,
15121 &second_ctx,
15122 &mut second_rx,
15123 hello_frame("second", PROTOCOL_VERSION, 2),
15124 )
15125 .await;
15126 let root = unique_project_root("second-only");
15127 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
15128 let cloned = handler.clone();
15129 let open = tokio::spawn(async move {
15130 cloned
15131 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
15132 .await
15133 .unwrap()
15134 });
15135 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
15136 .await
15137 .unwrap()
15138 .unwrap();
15139 let first = query_live_roots(&handler, &first_ctx).await;
15140 let second = query_live_roots(&handler, &second_ctx).await;
15141 assert!(
15142 matches!(
15143 first,
15144 ModuleControlResponseToModule::LiveRoots {
15145 total_bindings: 0,
15146 ..
15147 }
15148 ),
15149 "cross-module scope must not expose another module's roots"
15150 );
15151 assert!(
15152 matches!(
15153 second,
15154 ModuleControlResponseToModule::LiveRoots {
15155 total_bindings: 1,
15156 ..
15157 }
15158 ),
15159 "second module must see its pending route"
15160 );
15161 handler
15162 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
15163 .await
15164 .unwrap();
15165 assert!(open.await.unwrap().is_empty());
15166 }
15167
15168 #[tokio::test(start_paused = true)]
15169 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
15170 let handler = ControlHandler::with_forwarding(
15171 Arc::new(Registry::default()),
15172 Arc::new(ForwardingTable::default()),
15173 );
15174 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
15175 hello_via_sink(
15176 &handler,
15177 &target_ctx,
15178 &mut target_rx,
15179 hello_frame("target", PROTOCOL_VERSION, 1),
15180 )
15181 .await;
15182 let actual = query_live_roots(&handler, &target_ctx).await;
15183 let ModuleControlResponseToModule::LiveRoots {
15184 roots,
15185 unknown_root_bindings,
15186 total_bindings,
15187 } = actual
15188 else {
15189 panic!("expected live roots")
15190 };
15191 assert!(roots.is_empty());
15192 assert_eq!(unknown_root_bindings, 0);
15193 assert_eq!(total_bindings, 0);
15194 assert_eq!(
15195 total_bindings,
15196 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15197 );
15198 }
15199
15200 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
15207 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
15208 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
15209 let text = std::fs::read_to_string(&path)
15210 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
15211 let vectors: Vec<(String, Value)> = text
15212 .lines()
15213 .filter(|line| !line.trim().is_empty())
15214 .map(|line| {
15215 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
15216 let id = entry["corpus_id"]
15217 .as_str()
15218 .expect("every vector carries a corpus_id")
15219 .to_string();
15220 (id, entry["package"].clone())
15221 })
15222 .collect();
15223 assert_eq!(
15227 vectors.len(),
15228 3,
15229 "vendored fed corpus changed size; re-sync from subc-federation"
15230 );
15231
15232 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
15247 let richest = vectors
15248 .iter()
15249 .filter_map(|(_, package)| package.as_object())
15250 .map(|object| {
15251 object
15252 .keys()
15253 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
15254 .count()
15255 })
15256 .max()
15257 .unwrap_or(0);
15258 assert!(
15259 richest >= 2,
15260 "vendored corpus no longer carries a package with unmodelled fields, \
15261 so the relay test can no longer distinguish a verbatim relay from a lossy one"
15262 );
15263
15264 vectors
15265 }
15266
15267 #[tokio::test]
15275 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
15276 for (corpus_id, package) in fed_admission_facts_vectors() {
15277 let registry = Arc::new(Registry::default());
15278 let forwarding = Arc::new(ForwardingTable::default());
15279 let supervisor = SupervisorHandle::new();
15280 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15281 let handler =
15282 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15283 .with_supervisor(supervisor)
15284 .with_admission_facts_config(
15285 Some("fed".to_string()),
15286 Some(vec!["target".to_string()]),
15287 );
15288
15289 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
15290 hello_via_sink(
15291 &handler,
15292 &target_ctx,
15293 &mut target_rx,
15294 hello_frame("target", PROTOCOL_VERSION, 1),
15295 )
15296 .await;
15297
15298 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
15299 let route_handler = handler.clone();
15300 let expected = package.clone();
15301 let route_task = tokio::spawn(async move {
15302 route_handler
15303 .handle_control_frame(
15304 &client_ctx,
15305 route_open_frame_with_admission_facts(
15306 20,
15307 "target",
15308 unique_project_root("admission-facts"),
15309 Some(subc_control::ConsumerIdentity {
15310 module_id: "fed".to_string(),
15311 launch_nonce: "fed-nonce".to_string(),
15312 }),
15313 Some(package),
15314 ),
15315 )
15316 .await
15317 .unwrap()
15318 });
15319
15320 let bind_frame = target_rx.recv().await.unwrap();
15321 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15322 let ModuleControlRequest::RouteBind {
15323 admission_facts, ..
15324 } = bind
15325 else {
15326 panic!("{corpus_id}: expected route.bind")
15327 };
15328 assert_eq!(
15329 admission_facts,
15330 Some(expected),
15331 "{corpus_id}: relay must not add, drop or reshape any field"
15332 );
15333
15334 handler
15335 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15336 .await
15337 .unwrap();
15338 route_task.await.unwrap();
15339 }
15340 }
15341
15342 #[tokio::test]
15343 async fn route_bind_never_relays_the_openers_build_class() {
15344 let registry = Arc::new(Registry::default());
15345 let forwarding = Arc::new(ForwardingTable::default());
15346 let handler =
15347 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15348 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
15349 hello_via_sink(
15350 &handler,
15351 &target_ctx,
15352 &mut target_rx,
15353 hello_frame("target", PROTOCOL_VERSION, 1),
15354 )
15355 .await;
15356
15357 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
15358 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
15359 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15360 busy_refusals: None,
15361 target: RouteTarget::ToolProvider {
15362 module_id: "target".to_string(),
15363 },
15364 identity,
15365 consumer_identity: None,
15366 consumer_capabilities: None,
15367 role_versions: None,
15368 admission_facts: None,
15369 scope: None,
15370 })
15371 .unwrap();
15372 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
15373
15374 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
15375 let route_handler = handler.clone();
15376 let route_task = tokio::spawn(async move {
15377 route_handler
15378 .handle_control_frame(&client_ctx, open)
15379 .await
15380 .unwrap()
15381 });
15382 let bind_frame = target_rx.recv().await.unwrap();
15383 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15384 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
15385 panic!("expected route.bind")
15386 };
15387 assert_eq!(
15388 identity.build_class, None,
15389 "route.bind carried the build class"
15390 );
15391 assert!(
15392 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
15393 "the route.bind bytes still name build_class"
15394 );
15395 handler
15396 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15397 .await
15398 .unwrap();
15399 route_task.await.unwrap();
15400 }
15401
15402 #[tokio::test]
15403 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
15404 let registry = Arc::new(Registry::default());
15405 let forwarding = Arc::new(ForwardingTable::default());
15406 let supervisor = SupervisorHandle::new();
15407 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15408 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
15409 let handler =
15410 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15411 .with_supervisor(supervisor)
15412 .with_admission_facts_config(
15413 Some("fed".to_string()),
15414 Some(vec!["target".to_string()]),
15415 );
15416
15417 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
15418 hello_via_sink(
15419 &handler,
15420 &target_ctx,
15421 &mut target_rx,
15422 hello_frame("target", PROTOCOL_VERSION, 1),
15423 )
15424 .await;
15425 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
15426 hello_via_sink(
15427 &handler,
15428 &other_ctx,
15429 &mut other_rx,
15430 hello_frame("other", PROTOCOL_VERSION, 2),
15431 )
15432 .await;
15433
15434 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
15435 let expected_facts = facts.clone();
15436 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
15437 let route_handler = handler.clone();
15438 let route_task = tokio::spawn(async move {
15439 route_handler
15440 .handle_control_frame(
15441 &client_ctx,
15442 route_open_frame_with_admission_facts(
15443 10,
15444 "target",
15445 unique_project_root("admission-facts"),
15446 Some(subc_control::ConsumerIdentity {
15447 module_id: "fed".to_string(),
15448 launch_nonce: "fed-nonce".to_string(),
15449 }),
15450 Some(facts.clone()),
15451 ),
15452 )
15453 .await
15454 .unwrap()
15455 });
15456 let bind_frame = target_rx.recv().await.unwrap();
15457 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15458 let ModuleControlRequest::RouteBind {
15459 admission_facts, ..
15460 } = bind
15461 else {
15462 panic!("expected route.bind")
15463 };
15464 assert_eq!(admission_facts, Some(expected_facts));
15465 handler
15466 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15467 .await
15468 .unwrap();
15469 assert!(route_task.await.unwrap().is_empty());
15470 assert!(matches!(
15471 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
15472 .unwrap(),
15473 ClientControlResponse::RouteOpen { .. }
15474 ));
15475
15476 let direct = handler
15477 .handle_control_frame(
15478 &route_ctx(ConnectionId::new(73)).0,
15479 route_open_frame_with_admission_facts(
15480 11,
15481 "target",
15482 unique_project_root("admission-facts"),
15483 None,
15484 Some(json!({"x": 1})),
15485 ),
15486 )
15487 .await
15488 .unwrap();
15489 assert_eq!(
15490 parse_error(&direct[0])["code"],
15491 "admission_facts_not_permitted"
15492 );
15493
15494 let different_reserved = handler
15495 .handle_control_frame(
15496 &route_ctx(ConnectionId::new(77)).0,
15497 route_open_frame_with_admission_facts(
15498 15,
15499 "target",
15500 unique_project_root("admission-facts"),
15501 Some(subc_control::ConsumerIdentity {
15502 module_id: "other".to_string(),
15503 launch_nonce: "other-nonce".to_string(),
15504 }),
15505 Some(json!({"x": 1})),
15506 ),
15507 )
15508 .await
15509 .unwrap();
15510 assert_eq!(
15511 parse_error(&different_reserved[0])["code"],
15512 "admission_facts_not_permitted"
15513 );
15514
15515 let other_target = handler
15516 .handle_control_frame(
15517 &route_ctx(ConnectionId::new(74)).0,
15518 route_open_frame_with_admission_facts(
15519 12,
15520 "other",
15521 unique_project_root("admission-facts"),
15522 Some(subc_control::ConsumerIdentity {
15523 module_id: "fed".to_string(),
15524 launch_nonce: "fed-nonce".to_string(),
15525 }),
15526 Some(json!({"x": 1})),
15527 ),
15528 )
15529 .await
15530 .unwrap();
15531 assert_eq!(
15532 parse_error(&other_target[0])["code"],
15533 "admission_facts_target_not_allowed"
15534 );
15535
15536 let nonexistent = handler
15537 .handle_control_frame(
15538 &route_ctx(ConnectionId::new(75)).0,
15539 route_open_frame_with_admission_facts(
15540 13,
15541 "missing",
15542 unique_project_root("admission-facts"),
15543 None,
15544 Some(json!({"x": 1})),
15545 ),
15546 )
15547 .await
15548 .unwrap();
15549 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
15550
15551 let described = handler
15552 .handle_control_frame(
15553 &route_ctx(ConnectionId::new(76)).0,
15554 Frame::build(
15555 FrameType::Request,
15556 control_flags(),
15557 0,
15558 0,
15559 14,
15560 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15561 )
15562 .unwrap(),
15563 )
15564 .await
15565 .unwrap();
15566 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15567 serde_json::from_slice(&described[0].body).unwrap()
15568 else {
15569 panic!("expected server.describe response")
15570 };
15571 assert!(capabilities
15572 .iter()
15573 .any(|cap| cap == "admission_facts_relay_v1"));
15574 }
15575
15576 #[tokio::test]
15577 async fn admission_facts_without_configured_carrier_are_rejected() {
15578 let registry = Arc::new(Registry::default());
15579 let forwarding = Arc::new(ForwardingTable::default());
15580 let handler = ControlHandler::with_forwarding(registry, forwarding);
15581 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
15582 hello_via_sink(
15583 &handler,
15584 &target_ctx,
15585 &mut target_rx,
15586 hello_frame("target", PROTOCOL_VERSION, 1),
15587 )
15588 .await;
15589
15590 let responses = handler
15591 .handle_control_frame(
15592 &route_ctx(ConnectionId::new(79)).0,
15593 route_open_frame_with_admission_facts(
15594 16,
15595 "target",
15596 unique_project_root("admission-facts"),
15597 None,
15598 Some(json!({"x": 1})),
15599 ),
15600 )
15601 .await
15602 .unwrap();
15603 assert_eq!(
15604 parse_error(&responses[0])["code"],
15605 "admission_facts_not_permitted"
15606 );
15607 }
15608
15609 #[tokio::test]
15610 async fn route_open_relays_consumer_capabilities_verbatim() {
15611 let registry = Arc::new(Registry::default());
15612 let forwarding = Arc::new(ForwardingTable::default());
15613 let handler =
15614 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15615 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
15616 hello_via_sink(
15617 &handler,
15618 &module_ctx,
15619 &mut module_rx,
15620 hello_frame("aft", PROTOCOL_VERSION, 7),
15621 )
15622 .await;
15623
15624 let expected = vec!["elicitation".to_string(), "roots".to_string()];
15625 let expected_for_request = expected.clone();
15626 let project_root = unique_project_root("consumer-capabilities-present");
15627 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
15628 let route_handler = handler.clone();
15629 let route_task = tokio::spawn(async move {
15630 route_handler
15631 .handle_control_frame(
15632 &client_ctx,
15633 route_open_frame_with_consumer_capabilities(
15634 401,
15635 "aft",
15636 project_root,
15637 Some(expected_for_request),
15638 ),
15639 )
15640 .await
15641 .unwrap()
15642 });
15643 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15644 .await
15645 .unwrap()
15646 .unwrap();
15647 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15648 let ModuleControlRequest::RouteBind {
15649 consumer_capabilities,
15650 ..
15651 } = bind
15652 else {
15653 panic!("expected route.bind request, got {bind:?}");
15654 };
15655 assert_eq!(consumer_capabilities, Some(expected.clone()));
15656
15657 handler
15658 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15659 .await
15660 .unwrap();
15661 let route_response = route_task.await.unwrap();
15662 assert!(route_response.is_empty());
15663 let published = client_rx.recv().await.unwrap();
15664 assert!(matches!(
15665 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15666 ClientControlResponse::RouteOpen { .. }
15667 ));
15668 }
15669
15670 #[tokio::test]
15671 async fn route_open_without_consumer_capabilities_relays_none() {
15672 let registry = Arc::new(Registry::default());
15673 let forwarding = Arc::new(ForwardingTable::default());
15674 let handler =
15675 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15676 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
15677 hello_via_sink(
15678 &handler,
15679 &module_ctx,
15680 &mut module_rx,
15681 hello_frame("aft", PROTOCOL_VERSION, 7),
15682 )
15683 .await;
15684
15685 let project_root = unique_project_root("consumer-capabilities-absent");
15686 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
15687 let route_handler = handler.clone();
15688 let route_task = tokio::spawn(async move {
15689 route_handler
15690 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
15691 .await
15692 .unwrap()
15693 });
15694 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15695 .await
15696 .unwrap()
15697 .unwrap();
15698 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15699 let ModuleControlRequest::RouteBind {
15700 consumer_capabilities,
15701 ..
15702 } = bind
15703 else {
15704 panic!("expected route.bind request, got {bind:?}");
15705 };
15706 assert_eq!(consumer_capabilities, None);
15707
15708 handler
15709 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15710 .await
15711 .unwrap();
15712 let route_response = route_task.await.unwrap();
15713 assert!(route_response.is_empty());
15714 let published = client_rx.recv().await.unwrap();
15715 assert!(matches!(
15716 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15717 ClientControlResponse::RouteOpen { .. }
15718 ));
15719 }
15720
15721 async fn bind_role_versions_for(
15725 sent: Option<BTreeMap<String, String>>,
15726 connection: u64,
15727 ) -> Option<BTreeMap<String, String>> {
15728 let registry = Arc::new(Registry::default());
15729 let forwarding = Arc::new(ForwardingTable::default());
15730 let handler =
15731 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15732 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
15733 hello_via_sink(
15734 &handler,
15735 &module_ctx,
15736 &mut module_rx,
15737 hello_frame("aft", PROTOCOL_VERSION, 7),
15738 )
15739 .await;
15740 let project_root = unique_project_root("role-versions");
15741 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
15742 let route_handler = handler.clone();
15743 let route_task = tokio::spawn(async move {
15744 route_handler
15745 .handle_control_frame(
15746 &client_ctx,
15747 route_open_frame_with_role_versions(403, "aft", project_root, sent),
15748 )
15749 .await
15750 .unwrap()
15751 });
15752 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15753 .await
15754 .expect("a well-formed route.open reaches the module as a bind")
15755 .unwrap();
15756 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15757 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
15758 panic!("expected route.bind request, got {bind:?}");
15759 };
15760 handler
15761 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15762 .await
15763 .unwrap();
15764 assert!(route_task.await.unwrap().is_empty());
15765 let published = client_rx.recv().await.unwrap();
15766 assert!(matches!(
15767 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15768 ClientControlResponse::RouteOpen { .. }
15769 ));
15770 role_versions
15771 }
15772
15773 #[tokio::test]
15774 async fn route_open_relays_role_versions_verbatim() {
15775 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
15776 assert_eq!(
15777 bind_role_versions_for(Some(sent.clone()), 141).await,
15778 Some(sent)
15779 );
15780 }
15781
15782 #[tokio::test]
15785 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
15786 assert_eq!(bind_role_versions_for(None, 143).await, None);
15787 assert_eq!(
15788 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
15789 None
15790 );
15791 }
15792
15793 #[tokio::test]
15794 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
15795 let registry = Arc::new(Registry::default());
15796 let forwarding = Arc::new(ForwardingTable::default());
15797 let handler =
15798 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15799 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
15800 hello_via_sink(
15801 &handler,
15802 &module_ctx,
15803 &mut module_rx,
15804 hello_frame("aft", PROTOCOL_VERSION, 7),
15805 )
15806 .await;
15807
15808 let nine: BTreeMap<String, String> = (0..9)
15809 .map(|index| (format!("role-{index}"), "v1".to_string()))
15810 .collect();
15811 for (label, malformed) in [
15812 (
15813 "invalid role name",
15814 role_versions(&[("Tool_Provider", "v1")]),
15815 ),
15816 ("invalid version", role_versions(&[("tool-provider", "v0")])),
15817 ("nine entries", nine),
15818 ] {
15819 let responses = tokio::time::timeout(
15822 Duration::from_secs(1),
15823 handler.handle_control_frame(
15824 &route_ctx(ConnectionId::new(148)).0,
15825 route_open_frame_with_role_versions(
15826 404,
15827 "aft",
15828 unique_project_root("role-versions-malformed"),
15829 Some(malformed),
15830 ),
15831 ),
15832 )
15833 .await
15834 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
15835 .unwrap();
15836 assert_eq!(responses.len(), 1, "{label}");
15837 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
15838 let error = parse_error(&responses[0]);
15839 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
15840 assert_eq!(
15841 error["detail"]["field"], "role_versions",
15842 "{label}: {error}"
15843 );
15844 assert!(
15845 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
15846 "{label}: a malformed declaration is terminal"
15847 );
15848 assert!(
15849 module_rx.try_recv().is_err(),
15850 "{label}: the module must never see a bind"
15851 );
15852 }
15853 }
15854
15855 #[tokio::test]
15859 async fn route_role_versions_capability_is_advertised() {
15860 let handler = ControlHandler::new(Arc::new(Registry::default()));
15861 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
15862 let ack = hello_via_sink(
15863 &handler,
15864 &ctx,
15865 &mut rx,
15866 hello_frame("m", PROTOCOL_VERSION, 1),
15867 )
15868 .await;
15869 let ack = parse_ack(&ack);
15870 assert!(
15871 ack.subc_capabilities
15872 .iter()
15873 .any(|c| c == "route-role-versions/v1"),
15874 "{:?}",
15875 ack.subc_capabilities
15876 );
15877
15878 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15879 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15880 let reply = handler
15881 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
15882 .await
15883 .unwrap()
15884 .pop()
15885 .unwrap();
15886 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15887 serde_json::from_slice(&reply.body).unwrap()
15888 else {
15889 panic!("not a server.describe reply");
15890 };
15891 assert!(
15892 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
15893 "{capabilities:?}"
15894 );
15895 }
15896
15897 #[tokio::test]
15901 async fn catalog_events_capability_is_advertised() {
15902 let handler = ControlHandler::new(Arc::new(Registry::default()));
15903 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15904 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15905 let reply = handler
15906 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
15907 .await
15908 .unwrap()
15909 .pop()
15910 .unwrap();
15911 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15912 serde_json::from_slice(&reply.body).unwrap()
15913 else {
15914 panic!("not a server.describe reply");
15915 };
15916 assert!(
15917 capabilities
15918 .iter()
15919 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
15920 "{capabilities:?}"
15921 );
15922 }
15923
15924 #[tokio::test]
15925 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
15926 let handler = ControlHandler::new(Arc::new(Registry::default()));
15927 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
15928 let ack = hello_via_sink(
15929 &handler,
15930 &module,
15931 &mut module_rx,
15932 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
15933 )
15934 .await;
15935 let ack = parse_ack(&ack);
15936 assert!(
15937 ack.subc_capabilities
15938 .iter()
15939 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15940 "{:?}",
15941 ack.subc_capabilities
15942 );
15943 assert!(ack
15944 .subc_capabilities
15945 .iter()
15946 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15947
15948 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
15949 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15950 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15951 let reply = handler
15952 .handle_control_frame(&client, frame)
15953 .await
15954 .unwrap()
15955 .pop()
15956 .unwrap();
15957 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15958 serde_json::from_slice(&reply.body).unwrap()
15959 else {
15960 panic!("not a server.describe reply");
15961 };
15962 assert!(
15963 capabilities
15964 .iter()
15965 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15966 "{capabilities:?}"
15967 );
15968 assert!(capabilities
15969 .iter()
15970 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15971 }
15972
15973 #[tokio::test]
15974 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
15975 let handler = ControlHandler::new(Arc::new(Registry::default()));
15976 let (module, mut rx) = route_ctx(ConnectionId::new(156));
15977 let ack = hello_via_sink(
15978 &handler,
15979 &module,
15980 &mut rx,
15981 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
15982 )
15983 .await;
15984 assert!(parse_ack(&ack)
15985 .subc_capabilities
15986 .iter()
15987 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15988 assert!(parse_ack(&ack)
15989 .subc_capabilities
15990 .iter()
15991 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15992 let (client, _rx) = route_ctx(ConnectionId::new(157));
15993 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15994 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15995 let reply = handler
15996 .handle_control_frame(&client, frame)
15997 .await
15998 .unwrap()
15999 .pop()
16000 .unwrap();
16001 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16002 serde_json::from_slice(&reply.body).unwrap()
16003 else {
16004 panic!("server.describe reply")
16005 };
16006 assert!(capabilities
16007 .iter()
16008 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
16009 assert!(capabilities
16010 .iter()
16011 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
16012 }
16013
16014 #[tokio::test]
16015 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
16016 let handler = ControlHandler::new(Arc::new(Registry::default()));
16017 let (module, mut rx) = route_ctx(ConnectionId::new(158));
16018 let ack = hello_via_sink(
16019 &handler,
16020 &module,
16021 &mut rx,
16022 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
16023 )
16024 .await;
16025 assert!(parse_ack(&ack)
16026 .subc_capabilities
16027 .iter()
16028 .any(|cap| cap == "scope-workdir/v1"));
16029 let (client, _rx) = route_ctx(ConnectionId::new(159));
16030 let frame = Frame::build(
16031 FrameType::Request,
16032 control_flags(),
16033 0,
16034 0,
16035 5,
16036 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16037 )
16038 .unwrap();
16039 let reply = handler
16040 .handle_control_frame(&client, frame)
16041 .await
16042 .unwrap()
16043 .remove(0);
16044 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16045 serde_json::from_slice(&reply.body).unwrap()
16046 else {
16047 panic!("server.describe reply")
16048 };
16049 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
16050 }
16051
16052 #[tokio::test]
16053 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
16054 let handler = ControlHandler::new(Arc::new(Registry::default()));
16055 let (module, mut rx) = route_ctx(ConnectionId::new(160));
16056 let ack = hello_via_sink(
16057 &handler,
16058 &module,
16059 &mut rx,
16060 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
16061 )
16062 .await;
16063 assert!(parse_ack(&ack)
16064 .subc_capabilities
16065 .iter()
16066 .any(|cap| cap == "worker-runs/v1"));
16067 let (client, _rx) = route_ctx(ConnectionId::new(161));
16068 let frame = Frame::build(
16069 FrameType::Request,
16070 control_flags(),
16071 0,
16072 0,
16073 5,
16074 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16075 )
16076 .unwrap();
16077 let reply = handler
16078 .handle_control_frame(&client, frame)
16079 .await
16080 .unwrap()
16081 .remove(0);
16082 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16083 serde_json::from_slice(&reply.body).unwrap()
16084 else {
16085 panic!("server.describe reply")
16086 };
16087 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
16088 }
16089
16090 #[tokio::test]
16091 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
16092 let handler = ControlHandler::new(Arc::new(Registry::default()));
16093 let (module, mut rx) = route_ctx(ConnectionId::new(154));
16094 let ack = hello_via_sink(
16095 &handler,
16096 &module,
16097 &mut rx,
16098 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
16099 )
16100 .await;
16101 assert!(parse_ack(&ack)
16102 .subc_capabilities
16103 .iter()
16104 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16105 let (client, _rx) = route_ctx(ConnectionId::new(155));
16106 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16107 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16108 let reply = handler
16109 .handle_control_frame(&client, frame)
16110 .await
16111 .unwrap()
16112 .pop()
16113 .unwrap();
16114 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16115 serde_json::from_slice(&reply.body).unwrap()
16116 else {
16117 panic!("not a server.describe reply")
16118 };
16119 assert!(capabilities
16120 .iter()
16121 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16122 }
16123
16124 #[tokio::test]
16125 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
16126 let registry = Arc::new(Registry::default());
16127 let forwarding = Arc::new(ForwardingTable::default());
16128 let handler =
16129 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16130 .with_health_probe_timeout(Duration::from_secs(5));
16131 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
16132 hello_via_sink(
16133 &handler,
16134 &module_ctx,
16135 &mut module_rx,
16136 non_routable_hello_frame_with_control_ops(
16137 "mcp",
16138 300,
16139 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16140 ),
16141 )
16142 .await;
16143 assert!(registry
16144 .get_module("mcp")
16145 .unwrap()
16146 .unwrap()
16147 .manifest
16148 .provides
16149 .is_empty());
16150
16151 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
16152 let route_response = handler
16153 .handle_control_frame(
16154 &route_client_ctx,
16155 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
16156 )
16157 .await
16158 .unwrap();
16159 assert_eq!(route_response[0].header.ty, FrameType::Error);
16160 assert_eq!(
16161 parse_error(&route_response[0])["code"],
16162 "target_unavailable"
16163 );
16164 assert!(parse_error(&route_response[0])["message"]
16165 .as_str()
16166 .unwrap()
16167 .contains("does not provide the requested target"));
16168 assert!(module_rx.try_recv().is_err());
16169
16170 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
16171 let health_handler = handler.clone();
16172 let health_task = tokio::spawn(async move {
16173 health_handler
16174 .handle_control_frame(
16175 &health_client_ctx,
16176 supervisor_health_probe_frame(302, "mcp"),
16177 )
16178 .await
16179 .unwrap()
16180 });
16181 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16182 .await
16183 .unwrap()
16184 .unwrap();
16185 assert_eq!(
16186 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
16187 ModuleControlRequest::HealthCheck {}
16188 );
16189 handler
16190 .handle_control_frame(
16191 &module_ctx,
16192 health_response(health_frame.header.corr, HealthStatus::Ok),
16193 )
16194 .await
16195 .unwrap();
16196 let health_response = health_task.await.unwrap();
16197 assert_eq!(health_response[0].header.ty, FrameType::Response);
16198 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
16199 ClientControlResponse::SupervisorHealthProbe {
16200 module_id, status, ..
16201 } => {
16202 assert_eq!(module_id, "mcp");
16203 assert_eq!(status, HealthStatus::Ok);
16204 }
16205 other => panic!("unexpected health response: {other:?}"),
16206 }
16207
16208 forwarding
16213 .cleanup_connection(module_ctx.connection_id)
16214 .unwrap();
16215 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
16216 let cleanup_response = tokio::time::timeout(
16217 Duration::from_millis(200),
16218 handler.handle_control_frame(
16219 &cleanup_probe_ctx,
16220 supervisor_health_probe_frame(303, "mcp"),
16221 ),
16222 )
16223 .await
16224 .expect("probe should fail immediately when the control lane is gone")
16225 .unwrap();
16226 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
16227 assert_eq!(
16228 parse_error(&cleanup_response[0])["code"],
16229 "target_unavailable"
16230 );
16231 assert!(parse_error(&cleanup_response[0])["message"]
16232 .as_str()
16233 .unwrap()
16234 .contains("no module connection"));
16235
16236 handler
16237 .cleanup_connection(module_ctx.connection_id)
16238 .unwrap();
16239 }
16240
16241 #[tokio::test]
16242 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
16243 let registry = Arc::new(Registry::default());
16244 let supervisor_handle = SupervisorHandle::new();
16245 let supervisor =
16246 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
16247 .with_handle(supervisor_handle.clone())
16248 .with_connection_file_path(
16249 std::env::temp_dir()
16250 .join(format!("subc-route-open-warming-{}", std::process::id())),
16251 );
16252 let module = supervisor
16253 .supervise_configured(
16254 ModuleSpec {
16255 script: None,
16256 carry_containment: false,
16257 module_id: "warming".to_string(),
16258 program: fake_aft_stub_path(),
16259 args: Vec::new(),
16260 env: Vec::new(),
16261 reserved: false,
16262 reserved_prefixes: Vec::new(),
16263 protocol: ModuleProtocol::Subc,
16264 overlap: Default::default(),
16265 },
16266 true,
16267 )
16268 .unwrap();
16269 assert_eq!(module.state().unwrap(), ModuleState::Running);
16270
16271 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16272 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
16273 let response = handler
16274 .handle_control_frame(
16275 &ctx,
16276 route_open_frame(304, "warming", unique_project_root("warming")),
16277 )
16278 .await
16279 .unwrap();
16280 module.stop().await.unwrap();
16281
16282 assert_eq!(response[0].header.ty, FrameType::Error);
16283 let error = parse_error(&response[0]);
16284 assert_eq!(error["code"], "module_warming");
16285 assert!(error["message"]
16286 .as_str()
16287 .unwrap()
16288 .contains("state=running, enabled=true, live=false"));
16289 }
16290
16291 #[test]
16292 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
16293 let handler = ControlHandler::new(Arc::new(Registry::default()));
16294 let capture = EventCapture::default();
16295 let _subscriber =
16296 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16297 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
16298 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
16299 let pending = (0..limit).collect::<Vec<_>>();
16300 let response = handler
16301 .route_open_capacity_refusal(
16302 &ctx,
16303 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
16304 "busy",
16305 pending.len(),
16306 limit,
16307 )
16308 .unwrap();
16309 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16310 let event = capture
16311 .events()
16312 .into_iter()
16313 .find(|event| {
16314 event.target == "control"
16315 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
16316 })
16317 .expect("connection admission refusal event");
16318 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16319 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16320 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
16323 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
16324 }
16325
16326 #[test]
16327 fn route_open_target_cap_logs_admission_reason_and_capacity() {
16328 let handler = ControlHandler::new(Arc::new(Registry::default()));
16329 let capture = EventCapture::default();
16330 let _subscriber =
16331 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16332 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
16333 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
16334 let guards = (0..limit)
16335 .map(|_| {
16336 handler
16337 .route_bind_concurrency
16338 .try_admit("busy", limit)
16339 .unwrap()
16340 })
16341 .collect::<Vec<_>>();
16342 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
16343 Err(in_flight) => in_flight,
16344 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
16345 };
16346 let response = handler
16347 .route_open_target_capacity_refusal(
16348 &ctx,
16349 &RouteOpenCaller::unknown(),
16350 &route_open_frame(397, "busy", unique_project_root("target-cap")),
16351 "busy",
16352 in_flight,
16353 )
16354 .unwrap();
16355 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16356 let event = capture
16357 .events()
16358 .into_iter()
16359 .find(|event| {
16360 event.target == "control"
16361 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
16362 })
16363 .expect("target admission refusal event");
16364 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16365 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16366 drop(guards);
16367 }
16368
16369 #[cfg(unix)]
16370 #[tokio::test(start_paused = true)]
16371 async fn supervisor_list_shows_a_failed_modules_next_retry() {
16372 let registry = Arc::new(Registry::default());
16373 let handle = SupervisorHandle::new();
16374 let supervisor =
16375 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
16376 .with_handle(handle.clone());
16377 let module = supervisor
16378 .spawn(crate::ModuleSpec {
16379 script: None,
16380 carry_containment: false,
16381 module_id: "retry-list".into(),
16382 program: PathBuf::from("/bin/sh"),
16383 args: vec!["-c".into(), "exit 17".into()],
16384 env: vec![],
16385 reserved: false,
16386 reserved_prefixes: vec![],
16387 protocol: ModuleProtocol::None,
16388 overlap: Default::default(),
16389 })
16390 .unwrap();
16391 let deadline = std::time::Instant::now() + Duration::from_secs(10);
16392 while module.state().unwrap() != crate::ModuleState::Failed {
16393 assert!(
16394 std::time::Instant::now() < deadline,
16395 "waiting for budget failure"
16396 );
16397 tokio::task::yield_now().await;
16398 }
16399 let handler = ControlHandler::new(registry).with_supervisor(handle);
16400 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16401 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16402 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16403 assert_eq!(body["modules"][0]["state"], "failed");
16404 let retry = body["modules"][0]["next_retry_at_ms"]
16405 .as_u64()
16406 .expect("scheduled retry on the wire");
16407 let now = std::time::SystemTime::now()
16408 .duration_since(std::time::UNIX_EPOCH)
16409 .unwrap()
16410 .as_millis() as u64;
16411 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
16412 module.set_enabled(false).await.unwrap();
16413 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16414 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16415 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16416 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
16417 }
16418
16419 #[tokio::test]
16420 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
16421 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
16422 let supervisor =
16423 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
16424 let module = supervisor
16425 .spawn(crate::ModuleSpec {
16426 script: None,
16427 carry_containment: false,
16428 module_id: "pid-read-budget".into(),
16429 program: std::env::current_exe().unwrap(),
16430 args: vec![
16431 "--exact".into(),
16432 "control::tests::pid_resource_read_fixture".into(),
16433 ],
16434 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16435 .into_iter()
16436 .map(|key| (key.into(), dir.display().to_string()))
16437 .chain(std::iter::once((
16438 "SUBC_SUBTREE_PID_FIXTURE".into(),
16439 "1".into(),
16440 )))
16441 .collect(),
16442 reserved: false,
16443 reserved_prefixes: Vec::new(),
16444 protocol: ModuleProtocol::None,
16445 overlap: Default::default(),
16446 })
16447 .unwrap();
16448 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16449 while !matches!(
16450 module.child_resource_usage(),
16451 subc_control::ChildResourceUsage::Measured(_)
16452 ) {
16453 assert!(
16454 tokio::time::Instant::now() < ready_deadline,
16455 "fixture process was not measurable"
16456 );
16457 tokio::time::sleep(Duration::from_millis(10)).await;
16458 }
16459 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
16460 let reading = {
16461 let _guard = crate::subtree_resources::fault_read_guard().await;
16462 crate::subtree_resources::read_process(module.clone()).await
16463 };
16464 stop_resource_fixture(&module, &dir).await;
16465 assert!(
16466 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
16467 "{reading:?}"
16468 );
16469 }
16470
16471 #[test]
16472 fn pid_resource_read_fixture() {
16473 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
16474 let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
16475 let deadline = std::time::Instant::now() + Duration::from_secs(60);
16476 while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
16477 std::thread::sleep(Duration::from_millis(5));
16478 }
16479 std::fs::write(dir.join("module-stopped"), "").unwrap();
16480 }
16481 }
16482
16483 async fn stop_resource_fixture(
16484 module: &crate::supervise::SupervisedModule,
16485 dir: &std::path::Path,
16486 ) {
16487 let stopping = module.stop();
16488 tokio::pin!(stopping);
16489 tokio::select! {
16490 result = &mut stopping => result.unwrap(),
16491 () = async {
16492 while module.state().unwrap() != crate::supervise::ModuleState::Draining {
16493 tokio::time::sleep(Duration::from_millis(5)).await;
16494 }
16495 std::fs::write(dir.join("stop-module"), "").unwrap();
16496 } => stopping.await.unwrap(),
16497 }
16498 #[cfg(windows)]
16499 assert!(
16500 dir.join("module-stopped").exists(),
16501 "the resource fixture must exit cooperatively, not wait for the drain kill"
16502 );
16503 assert_eq!(
16504 module.state().unwrap(),
16505 crate::supervise::ModuleState::Stopped
16506 );
16507 }
16508
16509 #[tokio::test]
16510 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
16511 let registry = Arc::new(Registry::default());
16512 let handle = SupervisorHandle::new();
16513 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16514 .with_handle(handle.clone());
16515 supervisor
16516 .supervise_configured(
16517 crate::ModuleSpec {
16518 script: None,
16519 module_id: "disabled-resource-reading".into(),
16520 program: std::env::current_exe().unwrap(),
16521 args: Vec::new(),
16522 env: Vec::new(),
16523 reserved: false,
16524 reserved_prefixes: Vec::new(),
16525 protocol: ModuleProtocol::None,
16526 overlap: Default::default(),
16527 carry_containment: false,
16528 },
16529 false,
16530 )
16531 .unwrap();
16532 let handler = ControlHandler::new(registry).with_supervisor(handle);
16533 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16534 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16535 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16536 assert!(body["modules"][0].get("tree").is_none());
16537 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
16538 }
16539
16540 #[tokio::test]
16541 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
16542 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
16543 let registry = Arc::new(Registry::default());
16544 let handle = SupervisorHandle::new();
16545 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16546 .with_handle(handle.clone());
16547 let module = supervisor
16548 .spawn(crate::ModuleSpec {
16549 script: None,
16550 carry_containment: false,
16551 module_id: "list-process-priority".into(),
16552 program: std::env::current_exe().unwrap(),
16553 args: vec![
16554 "--exact".into(),
16555 "control::tests::pid_resource_read_fixture".into(),
16556 ],
16557 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16558 .into_iter()
16559 .map(|key| (key.into(), dir.display().to_string()))
16560 .chain(std::iter::once((
16561 "SUBC_SUBTREE_PID_FIXTURE".into(),
16562 "1".into(),
16563 )))
16564 .collect(),
16565 reserved: false,
16566 reserved_prefixes: Vec::new(),
16567 protocol: ModuleProtocol::None,
16568 overlap: Default::default(),
16569 })
16570 .unwrap();
16571 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16572 loop {
16573 let image = module.running_image_agreement().await;
16574 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
16575 break;
16576 }
16577 assert!(
16578 tokio::time::Instant::now() < ready_deadline,
16579 "the fixture process never became observable: {image:?}"
16580 );
16581 tokio::time::sleep(Duration::from_millis(10)).await;
16582 }
16583 let guard = crate::subtree_resources::fault_read_guard().await;
16584 let (target, release, _) = crate::subtree_resources::held();
16585 module.set_subtree_target_for_test(target);
16586 let handler = ControlHandler::new(registry).with_supervisor(handle);
16587 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16588 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16589 drop(release);
16590 drop(guard);
16591 stop_resource_fixture(&module, &dir).await;
16592 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16593 assert_eq!(
16594 body["modules"][0]["resources"]["status"], "measured",
16595 "the optional tree timeout must not discard the process reading: {body}"
16596 );
16597 assert!(body["modules"][0].get("tree").is_none());
16598 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
16599 assert_eq!(
16600 body["modules"][0]["pending_reload"]["image"]["status"], "match",
16601 "the tree timeout must not hide a completed image observation"
16602 );
16603 }
16604
16605 #[cfg(any(target_os = "linux", target_os = "macos"))]
16606 #[tokio::test]
16607 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
16608 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
16609 .with_daemon_provenance(
16610 std::process::id(),
16611 0,
16612 Some(std::env::current_exe().unwrap()),
16613 None,
16614 None,
16615 );
16616 let (probe, release, began) = ExecutableIdentityProbe::held();
16617 handler.daemon_provenance.probe = probe;
16618 let observe = |handler: ControlHandler, corr| {
16619 tokio::spawn(async move {
16620 let (ctx, _) = route_ctx(ConnectionId::new(corr));
16621 let frame =
16622 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
16623 .unwrap();
16624 handler
16625 .handle_supervisor_provenance(&ctx, frame, None)
16626 .await
16627 .unwrap()
16628 })
16629 };
16630 let explicit = observe(handler.clone(), 114);
16631 began.await.unwrap();
16632 let concurrent = observe(handler.clone(), 115);
16633 tokio::time::sleep(Duration::from_millis(300)).await;
16636 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16637 let list = tokio::time::timeout(
16638 Duration::from_secs(1),
16639 handler.handle_supervisor_list(frame),
16640 )
16641 .await;
16642 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
16643 drop(release);
16644 let replies = explicit.await.unwrap();
16645 let concurrent_replies = concurrent.await.unwrap();
16646 assert!(
16647 explicit_was_pending,
16648 "explicit provenance must await the full observation"
16649 );
16650 let list = list
16651 .expect("a concurrent explicit probe must not stall list")
16652 .unwrap();
16653 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
16654 assert_eq!(list_body["op"], "supervisor.list");
16655 for replies in [replies, concurrent_replies] {
16656 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16657 assert_eq!(
16658 body["daemon"]["daemon_observed"]["running_image"]["status"],
16659 "match"
16660 );
16661 }
16662 }
16663
16664 #[tokio::test]
16665 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
16666 let registry = Arc::new(Registry::default());
16667 let handle = SupervisorHandle::new();
16668 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16669 .with_handle(handle.clone());
16670 let module = supervisor
16671 .supervise_configured(
16672 crate::ModuleSpec {
16673 script: None,
16674 carry_containment: false,
16675 module_id: "memory-read".into(),
16676 program: PathBuf::from("disabled-module"),
16677 args: Vec::new(),
16678 env: Vec::new(),
16679 reserved: false,
16680 reserved_prefixes: Vec::new(),
16681 protocol: ModuleProtocol::Subc,
16682 overlap: Default::default(),
16683 },
16684 true,
16685 )
16686 .unwrap();
16687 let _guard = crate::subtree_resources::fault_read_guard().await;
16688 let (target, release, began) = crate::subtree_resources::held();
16689 let stalled_target = target.clone();
16692 let stalled = tokio::task::spawn_blocking(move || {
16693 stalled_target.tree(&subc_os::ProcessTable::default())
16694 });
16695 began.await.unwrap();
16696 module.set_subtree_target_for_test(target);
16697 let handler = ControlHandler::new(registry).with_supervisor(handle);
16698 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
16699 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16700 let replies = handler
16701 .handle_client_control_request(
16702 &ctx,
16703 frame,
16704 ClientControlRequest::SupervisorList {},
16705 &mut None,
16706 )
16707 .await
16708 .unwrap();
16709 assert!(
16710 replies.is_empty(),
16711 "list reply must be detached from the connection reader"
16712 );
16713 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16714 let replies = handler
16715 .handle_client_control_request(
16716 &ctx,
16717 frame,
16718 ClientControlRequest::SupervisorSpawnSnapshot {},
16719 &mut None,
16720 )
16721 .await
16722 .unwrap();
16723 assert_eq!(
16724 replies.len(),
16725 1,
16726 "another request must complete while the subtree read stalls"
16727 );
16728 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16729 .await
16730 .unwrap()
16731 .unwrap();
16732 release.send(()).unwrap();
16733 drop(release);
16734 stalled.await.unwrap();
16735 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
16736 assert!(
16737 body["modules"][0].get("tree").is_none(),
16738 "an unknown tree reading must be omitted after the bounded read expires"
16739 );
16740 handler
16744 .resource_tracker
16745 .read(handler.supervisor.list(), false, "released_tree_fixture")
16746 .await;
16747 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16748 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16749 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16750 assert_eq!(
16751 body["modules"][0]["tree"]["source"], "process_tree",
16752 "the next request must export the completed refresh"
16753 );
16754 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
16755 }
16756
16757 #[tokio::test]
16758 async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
16759 use subc_control::{
16760 AgentCommandResources, ChildMemoryKind, ChildResourceReading,
16761 ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
16762 TreeReading, TreeSource,
16763 };
16764 let registry = Arc::new(Registry::default());
16765 let handle = SupervisorHandle::new();
16766 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16767 .with_handle(handle.clone());
16768 supervisor
16769 .supervise_configured(
16770 crate::ModuleSpec {
16771 script: None,
16772 carry_containment: false,
16773 module_id: "budget-worker".into(),
16774 program: PathBuf::from("disabled-module"),
16775 args: Vec::new(),
16776 env: Vec::new(),
16777 reserved: false,
16778 reserved_prefixes: Vec::new(),
16779 protocol: ModuleProtocol::Subc,
16780 overlap: Default::default(),
16781 },
16782 true,
16783 )
16784 .unwrap();
16785 let handler = ControlHandler::new(registry).with_supervisor(handle);
16786 let now = u64::try_from(
16787 std::time::SystemTime::now()
16788 .duration_since(std::time::UNIX_EPOCH)
16789 .unwrap()
16790 .as_millis(),
16791 )
16792 .unwrap();
16793 let measured = now - 1_000;
16794 let hour = CpuLastHour::new(100, 60_000);
16795 let daemon = DaemonResources::new(7, 1, "test")
16796 .with_measured_at_ms(Some(measured))
16797 .with_resources(Some(ChildResourceUsage::Measured(
16798 ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
16799 )))
16800 .with_cpu_last_hour(Some(hour.clone()));
16801 let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
16802 .with_measured_at_ms(Some(measured))
16803 .with_cpu_last_hour(Some(hour.clone()));
16804 let tree = TreeReading::new(
16805 150,
16806 50,
16807 ChildMemoryKind::ResidentSet,
16808 1,
16809 TreeSource::ProcessTree,
16810 )
16811 .with_measured_at_ms(Some(measured));
16812 handler.resource_tracker.seed_committed_for_test(
16813 &handler.supervisor.list(),
16814 crate::resource_totals::Readings {
16815 modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
16816 daemon: Some(daemon),
16817 agent_commands: Some(vec![agents]),
16818 ..Default::default()
16819 },
16820 now,
16821 );
16822 list_diagnostics::scope(async {
16823 tokio::time::sleep(Duration::from_millis(110)).await;
16824 assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
16825 let request =
16826 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16827 let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
16828 let ClientControlResponse::SupervisorList {
16829 modules,
16830 daemon,
16831 agent_commands,
16832 ..
16833 } = serde_json::from_slice(&replies[0].body).unwrap()
16834 else {
16835 panic!("expected a list response")
16836 };
16837 assert!(matches!(
16838 modules[0].resources,
16839 Some(ChildResourceUsage::Unavailable {
16840 reason: ChildResourceUnavailableReason::Unreadable
16841 })
16842 ));
16843 assert_eq!(
16844 (
16845 modules[0].tree.as_ref().unwrap().cpu_ms,
16846 modules[0].tree.as_ref().unwrap().measured_at_ms
16847 ),
16848 (150, Some(measured))
16849 );
16850 let daemon = daemon.unwrap();
16851 let agents = agent_commands.unwrap();
16852 assert_eq!(daemon.measured_at_ms, Some(measured));
16853 assert_eq!(
16854 (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
16855 (1, 50, Some(measured))
16856 );
16857 for hour in [
16858 modules[0].cpu_last_hour.as_ref(),
16859 daemon.cpu_last_hour.as_ref(),
16860 agents[0].cpu_last_hour.as_ref(),
16861 ] {
16862 let hour = hour.unwrap();
16863 assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
16864 }
16865 })
16866 .await;
16867 }
16868
16869 #[tokio::test]
16870 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
16871 let registry = Arc::new(Registry::default());
16872 let handle = SupervisorHandle::new();
16873 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16874 .with_handle(handle.clone());
16875 supervisor
16876 .supervise_configured(
16877 crate::ModuleSpec {
16878 script: None,
16879 carry_containment: false,
16880 module_id: "drift".into(),
16881 program: PathBuf::from("disabled-module"),
16882 args: Vec::new(),
16883 env: Vec::new(),
16884 reserved: false,
16885 reserved_prefixes: Vec::new(),
16886 protocol: ModuleProtocol::Subc,
16887 overlap: Default::default(),
16888 },
16889 false,
16890 )
16891 .unwrap();
16892 let handler = ControlHandler::new(registry).with_supervisor(handle);
16893 for configured in [None, Some(false), Some(true)] {
16894 handler.capability_evaluator.configure(
16895 configured.map(|enabled| ("drift".to_string(), enabled)),
16896 BTreeMap::new(),
16897 );
16898 let frame =
16899 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16900 let replies = handler
16901 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
16902 .await
16903 .unwrap();
16904 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16905 assert!(body.get("configured_enabled").is_none());
16906 assert_eq!(body["op"], "supervisor.ack");
16907 let frame =
16908 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16909 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16910 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16911 assert_eq!(
16912 body["modules"][0].get("configured_enabled"),
16913 configured.map(Value::Bool).as_ref()
16914 );
16915 assert!(body["modules"][0].get("tree").is_none());
16916 assert!(body["modules"][0].get("subtree").is_none());
16917 }
16918 }
16919
16920 #[tokio::test(flavor = "current_thread")]
16921 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
16922 let handler = ControlHandler::new(Arc::new(Registry::default()));
16923 let capture = EventCapture::default();
16924 let _subscriber =
16925 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16926 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
16927 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16928
16929 let _ = handler
16930 .handle_client_control_request(
16931 &ctx,
16932 frame,
16933 ClientControlRequest::SupervisorSetEnabled {
16934 module_id: "broca".to_string(),
16935 enabled: false,
16936 },
16937 &mut None,
16938 )
16939 .await
16940 .unwrap();
16941
16942 let events = capture
16943 .events()
16944 .into_iter()
16945 .filter(|event| {
16946 event.target == "control"
16947 && event.fields.get("message").map(String::as_str)
16948 == Some("supervisor request received")
16949 })
16950 .collect::<Vec<_>>();
16951 assert_eq!(events.len(), 1, "one supervisor request log line");
16952 let event = &events[0];
16953 assert_eq!(event.level, tracing::Level::INFO);
16954 assert_eq!(
16955 event.fields.get("op"),
16956 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
16957 );
16958 assert_eq!(
16959 event.fields.get("module_id"),
16960 Some(&"\"broca\"".to_string())
16961 );
16962 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
16963 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
16964 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
16965 }
16966
16967 #[tokio::test(flavor = "current_thread")]
16968 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
16969 let registry = Arc::new(Registry::default());
16970 let module_connection = ConnectionId::new(303);
16971 registry
16972 .register_with_control_ops(
16973 manifest("aft", PROTOCOL_VERSION),
16974 PROTOCOL_VERSION,
16975 module_connection,
16976 module_baseline_control_ops(),
16977 )
16978 .unwrap();
16979 let handler = ControlHandler::new(registry);
16980 let capture = EventCapture::default();
16981 let _subscriber =
16982 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16983 let (ctx, _rx) = route_ctx(module_connection);
16984 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16985
16986 let _ = handler
16987 .handle_client_control_request(
16988 &ctx,
16989 frame,
16990 ClientControlRequest::SupervisorSetEnabled {
16991 module_id: "broca".to_string(),
16992 enabled: false,
16993 },
16994 &mut None,
16995 )
16996 .await
16997 .unwrap();
16998
16999 let callers = capture
17000 .events()
17001 .into_iter()
17002 .filter(|event| {
17003 event.fields.get("message").map(String::as_str)
17004 == Some("supervisor request received")
17005 })
17006 .map(|event| event.fields.get("caller").cloned())
17007 .collect::<Vec<_>>();
17008 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
17009 }
17010
17011 #[tokio::test(flavor = "current_thread")]
17012 async fn supervisor_list_does_not_log_request_received() {
17013 let handler = ControlHandler::new(Arc::new(Registry::default()));
17014 let capture = EventCapture::default();
17015 let _subscriber =
17016 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17017 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
17018 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
17019
17020 handler
17021 .handle_client_control_request(
17022 &ctx,
17023 frame,
17024 ClientControlRequest::SupervisorList {},
17025 &mut None,
17026 )
17027 .await
17028 .unwrap();
17029
17030 assert!(capture.events().into_iter().all(|event| {
17031 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17032 }));
17033 }
17034
17035 #[tokio::test(flavor = "current_thread")]
17036 async fn supervisor_rescan_preview_does_not_log_request_received() {
17037 let handler = ControlHandler::new(Arc::new(Registry::default()));
17038 let capture = EventCapture::default();
17039 let _subscriber =
17040 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17041 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
17042 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
17043
17044 handler
17045 .handle_client_control_request(
17046 &ctx,
17047 frame,
17048 ClientControlRequest::SupervisorRescan { preview: true },
17049 &mut None,
17050 )
17051 .await
17052 .unwrap();
17053
17054 assert!(capture.events().into_iter().all(|event| {
17055 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17056 }));
17057 }
17058
17059 struct ApproveOperator;
17060 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
17061 fn prompt(
17062 &self,
17063 _: &str,
17064 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
17065 ) -> crate::operator_confirm::ProviderResult {
17066 crate::operator_confirm::ProviderResult::Approved
17067 }
17068 }
17069
17070 #[tokio::test]
17071 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
17072 assert_eq!(
17073 ControlHandler::default().privileged_operator_prompt,
17074 PrivilegedOperatorPrompt::Log
17075 );
17076 for (initial, changed, running) in [
17077 ("log", "enforce", PrivilegedOperatorPrompt::Log),
17078 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
17079 ] {
17080 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
17081 let path = dir.join("subc.jsonc");
17082 std::fs::write(
17083 &path,
17084 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
17085 )
17086 .unwrap();
17087 let config = crate::daemon_config::load(&path).unwrap().unwrap();
17088 let registry = Arc::new(Registry::default());
17089 let handle = SupervisorHandle::new();
17090 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
17091 .with_handle(handle.clone());
17092 let handler = ControlHandler::new(registry)
17093 .with_supervisor(handle)
17094 .with_privileged_operator_prompt(config.privileged_operator_prompt)
17095 .with_supervisor_rescan(supervisor, &path, config.port);
17096 handler.forwarding.operator_confirms().configure(
17097 Arc::new(ApproveOperator),
17098 crate::operator_confirm::OperatorLimits::default(),
17099 );
17100 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
17101
17102 for (disk_mode, needs_restart) in [
17103 (Some(initial), false),
17104 (Some(changed), true),
17105 (Some(changed), true),
17106 (None, running == PrivilegedOperatorPrompt::Enforce),
17107 (Some(initial), false),
17108 ] {
17109 let mut doc = json!({"version":1});
17110 if let Some(mode) = disk_mode {
17111 doc["privileged_operator_prompt"] = json!(mode);
17112 }
17113 std::fs::write(&path, doc.to_string()).unwrap();
17114 for preview in [true, false] {
17115 let request = ClientControlRequest::SupervisorRescan { preview };
17116 let frame = Frame::build(
17117 FrameType::Request,
17118 control_flags(),
17119 0,
17120 0,
17121 1,
17122 serde_json::to_vec(&request).unwrap(),
17123 )
17124 .unwrap();
17125 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17126 assert_eq!(frames.len(), 1);
17127 let ClientControlResponse::SupervisorRescan { result } =
17128 serde_json::from_slice(&frames[0].body).unwrap()
17129 else {
17130 panic!("expected a rescan response");
17131 };
17132 let expected = if needs_restart {
17133 vec!["privileged_operator_prompt"]
17134 } else {
17135 Vec::new()
17136 };
17137 assert_eq!(result.restart_required, expected);
17138 assert_eq!(handler.privileged_operator_prompt, running);
17139 assert_eq!(
17140 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
17141 running
17142 );
17143 }
17144 }
17145 }
17146 }
17147
17148 #[tokio::test]
17149 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
17150 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
17151 let path = dir.join("subc.jsonc");
17152 let registry = Arc::new(Registry::default());
17153 let handle = SupervisorHandle::new();
17154 let supervisor =
17155 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
17156 let handler = ControlHandler::new(registry)
17157 .with_supervisor(handle)
17158 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
17159 .with_supervisor_rescan(supervisor, &path, None);
17160 handler.forwarding.operator_confirms().configure(
17161 Arc::new(ApproveOperator),
17162 crate::operator_confirm::OperatorLimits::default(),
17163 );
17164 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
17165 for preview in [true, false] {
17166 std::fs::write(
17167 &path,
17168 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
17169 )
17170 .unwrap();
17171 let request = ClientControlRequest::SupervisorRescan { preview };
17172 let frame = Frame::build(
17173 FrameType::Request,
17174 control_flags(),
17175 0,
17176 0,
17177 1,
17178 serde_json::to_vec(&request).unwrap(),
17179 )
17180 .unwrap();
17181 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17182 assert_eq!(frames.len(), 1);
17183 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
17184 assert_eq!(
17185 handler.privileged_operator_prompt,
17186 PrivilegedOperatorPrompt::Enforce
17187 );
17188 }
17189 }
17190
17191 #[tokio::test]
17196 async fn route_open_refusal_names_the_check_that_refused() {
17197 let handler = ControlHandler::new(Arc::new(Registry::default()));
17198 let capture = EventCapture::default();
17199 let _subscriber =
17200 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17201 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17202 let response = handler
17203 .handle_control_frame(
17204 &ctx,
17205 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
17206 )
17207 .await
17208 .unwrap();
17209
17210 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17211 let event = capture
17212 .events()
17213 .into_iter()
17214 .find(|event| {
17215 event.target == "control"
17216 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17217 })
17218 .expect("route.open refusal event");
17219 assert_eq!(
17220 event.fields.get("reason"),
17221 Some(&"\"not_registered\"".to_string())
17222 );
17223 }
17224
17225 #[tokio::test]
17226 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
17227 let registry = Arc::new(Registry::default());
17228 let supervisor_handle = SupervisorHandle::new();
17229 let supervisor =
17230 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17231 .with_handle(supervisor_handle.clone())
17232 .with_connection_file_path(std::env::temp_dir().join(format!(
17233 "subc-route-open-refusal-info-{}",
17234 std::process::id()
17235 )));
17236 let module = supervisor
17237 .supervise_configured(
17238 ModuleSpec {
17239 script: None,
17240 carry_containment: false,
17241 module_id: "warming".to_string(),
17242 program: fake_aft_stub_path(),
17243 args: Vec::new(),
17244 env: Vec::new(),
17245 reserved: false,
17246 reserved_prefixes: Vec::new(),
17247 protocol: ModuleProtocol::Subc,
17248 overlap: Default::default(),
17249 },
17250 true,
17251 )
17252 .unwrap();
17253 assert_eq!(module.state().unwrap(), ModuleState::Running);
17254
17255 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17256 assert!(handler
17257 .counters()
17258 .snapshot()
17259 .get("route_open_refused_by_code")
17260 .is_none());
17261 let capture = EventCapture::default();
17262 let _subscriber =
17263 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17264 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
17265 let response = handler
17266 .handle_control_frame(
17267 &ctx,
17268 route_open_frame(394, "warming", unique_project_root("refusal-info")),
17269 )
17270 .await
17271 .unwrap();
17272 module.stop().await.unwrap();
17273
17274 assert_eq!(parse_error(&response[0])["code"], "module_warming");
17275 let event = capture
17276 .events()
17277 .into_iter()
17278 .find(|event| {
17279 event.target == "control"
17280 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
17281 })
17282 .expect("route.open refusal event");
17283 assert_eq!(
17284 event.fields.get("module_id"),
17285 Some(&"\"warming\"".to_string())
17286 );
17287 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
17288 assert_eq!(
17289 event.fields.get("reason"),
17290 Some(&"\"supervised_not_registered\"".to_string())
17291 );
17292 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
17293 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
17294 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
17295 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
17296 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
17297 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
17298 assert_eq!(
17299 handler.counters().snapshot()["route_open_refused_by_code"],
17300 json!({ "module_warming": 1 })
17301 );
17302 }
17303
17304 const OUTAGE_START: &str = "route.open refusing module: not serving";
17305 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
17306
17307 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
17308 capture
17309 .events()
17310 .into_iter()
17311 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
17312 .collect()
17313 }
17314
17315 fn supervise_stub(
17316 registry: &Arc<Registry>,
17317 module_id: &str,
17318 enabled: bool,
17319 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
17320 let supervisor_handle = SupervisorHandle::new();
17321 let supervisor =
17322 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
17323 .with_handle(supervisor_handle.clone())
17324 .with_connection_file_path(std::env::temp_dir().join(format!(
17325 "subc-route-outage-{module_id}-{}",
17326 std::process::id()
17327 )));
17328 let module = supervisor
17329 .supervise_configured(
17330 ModuleSpec {
17331 script: None,
17332 carry_containment: false,
17333 module_id: module_id.to_string(),
17334 program: fake_aft_stub_path(),
17335 args: Vec::new(),
17336 env: Vec::new(),
17337 reserved: false,
17338 reserved_prefixes: Vec::new(),
17339 protocol: ModuleProtocol::Subc,
17340 overlap: Default::default(),
17341 },
17342 enabled,
17343 )
17344 .unwrap();
17345 (supervisor_handle, module)
17346 }
17347
17348 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
17349 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
17350 module_id: module_id.to_string(),
17351 drain_timeout_ms: Some(50),
17352 })
17353 .unwrap();
17354 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
17355 }
17356
17357 #[test]
17361 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
17362 let registry = Arc::new(Registry::default());
17363 let forwarding = Arc::new(ForwardingTable::default());
17364 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17365 let second = ControlHandler::with_forwarding(registry, forwarding);
17366 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
17367 }
17368
17369 #[tokio::test(flavor = "current_thread")]
17373 async fn route_open_unknown_module_refusals_add_no_outage_state() {
17374 let handler = ControlHandler::new(Arc::new(Registry::default()));
17375 let capture = EventCapture::default();
17376 let _subscriber =
17377 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17378 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
17379 for corr in 0..8 {
17380 let response = handler
17381 .handle_control_frame(
17382 &ctx,
17383 route_open_frame(
17384 380 + corr,
17385 &format!("nobody-{corr}"),
17386 unique_project_root("outage-unknown"),
17387 ),
17388 )
17389 .await
17390 .unwrap();
17391 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17392 }
17393
17394 assert_eq!(handler.route_outages.tracked_module_count(), 0);
17395 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
17396 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
17397 }
17398
17399 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17404 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
17405 let registry = Arc::new(Registry::default());
17406 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
17407 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17408 let capture = EventCapture::default();
17409 let _subscriber =
17410 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17411 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
17412 handler.route_outages.record_accepted("outage-restart");
17415
17416 let response = handler
17417 .handle_control_frame(
17418 &ctx,
17419 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
17420 )
17421 .await
17422 .unwrap();
17423 assert_eq!(response[0].header.ty, FrameType::Error);
17424 let starts = outage_lines(&capture, OUTAGE_START);
17425 assert_eq!(starts.len(), 1, "{starts:?}");
17426 assert_eq!(starts[0].level, tracing::Level::WARN);
17427 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
17428 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
17429 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
17430 handler.route_outages.record_accepted("outage-restart");
17431 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
17432
17433 let restart = handler
17434 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
17435 .await
17436 .unwrap();
17437 assert_eq!(
17438 restart[0].header.ty,
17439 FrameType::Response,
17440 "{:?}",
17441 parse_error(&restart[0])
17442 );
17443 handler
17444 .handle_control_frame(
17445 &ctx,
17446 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
17447 )
17448 .await
17449 .unwrap();
17450 module.stop().await.unwrap();
17451
17452 let starts = outage_lines(&capture, OUTAGE_START);
17453 assert_eq!(starts.len(), 2, "{starts:?}");
17454 assert_eq!(starts[1].level, tracing::Level::INFO);
17455 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
17456 }
17457
17458 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17462 async fn failed_operator_restart_leaves_no_operator_mark() {
17463 let registry = Arc::new(Registry::default());
17464 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
17465 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17466 let capture = EventCapture::default();
17467 let _subscriber =
17468 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17469 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
17470 handler.route_outages.record_accepted("outage-disabled");
17471
17472 let restart = handler
17473 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
17474 .await
17475 .unwrap();
17476 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
17477 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
17478
17479 handler
17480 .handle_control_frame(
17481 &ctx,
17482 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
17483 )
17484 .await
17485 .unwrap();
17486 let starts = outage_lines(&capture, OUTAGE_START);
17487 assert_eq!(starts.len(), 1, "{starts:?}");
17488 assert_eq!(starts[0].level, tracing::Level::WARN);
17489 }
17490
17491 #[tokio::test(flavor = "current_thread")]
17492 async fn route_open_unknown_module_escapes_target_module_id() {
17493 let handler = ControlHandler::new(Arc::new(Registry::default()));
17494 let capture = EventCapture::default();
17495 let _subscriber =
17496 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17497 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
17498 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17499 let response = handler
17500 .handle_control_frame(
17501 &ctx,
17502 route_open_frame(
17503 395,
17504 hostile_module_id,
17505 unique_project_root("hostile-target-module-id"),
17506 ),
17507 )
17508 .await
17509 .unwrap();
17510
17511 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17512 let event = capture
17513 .events()
17514 .into_iter()
17515 .find(|event| {
17516 event.target == "control"
17517 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17518 })
17519 .expect("route.open unknown-module refusal event");
17520 let logged = event.fields.get("module_id").expect("module_id field");
17521 assert!(!logged.bytes().any(|byte| byte < 0x20));
17522 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17523 }
17524
17525 #[tokio::test(flavor = "current_thread")]
17528 async fn route_open_refusal_carries_daemon_origin() {
17529 let handler = ControlHandler::new(Arc::new(Registry::default()));
17530 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17531 let response = handler
17532 .handle_control_frame(
17533 &ctx,
17534 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
17535 )
17536 .await
17537 .unwrap();
17538
17539 assert_eq!(response.len(), 1);
17540 assert_eq!(response[0].header.ty, FrameType::Error);
17541 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17542 assert!(
17543 response[0].header.flags.is_daemon_origin(),
17544 "route.open refusal must carry daemon origin: {:?}",
17545 response[0].header.flags
17546 );
17547 }
17548
17549 #[tokio::test(flavor = "current_thread")]
17553 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
17554 let registry = Arc::new(Registry::default());
17555 let forwarding = Arc::new(ForwardingTable::default());
17556 let handler =
17557 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17558 let module_connection = ConnectionId::new(95);
17559 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17560 hello_via_sink(
17561 &handler,
17562 &module_ctx,
17563 &mut module_rx,
17564 hello_frame("aft", PROTOCOL_VERSION, 395),
17565 )
17566 .await;
17567 forwarding
17568 .begin_module_drain("aft", RouteCloseReason::Restart)
17569 .unwrap()
17570 .expect("the module endpoint drains");
17571
17572 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
17573 let response = handler
17574 .handle_control_frame(
17575 &client_ctx,
17576 route_open_frame(396, "aft", unique_project_root("drain-origin")),
17577 )
17578 .await
17579 .unwrap();
17580
17581 assert_eq!(response.len(), 1);
17582 assert_eq!(response[0].header.ty, FrameType::Error);
17583 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17584 assert!(
17585 response[0].header.flags.is_daemon_origin(),
17586 "drain module_reloading must carry daemon origin: {:?}",
17587 response[0].header.flags
17588 );
17589 }
17590
17591 #[tokio::test(flavor = "current_thread")]
17595 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
17596 let registry = Arc::new(Registry::default());
17597 let forwarding = Arc::new(ForwardingTable::default());
17598 let handler =
17599 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17600 let module_connection = ConnectionId::new(95);
17601 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17602 hello_via_sink(
17603 &handler,
17604 &module_ctx,
17605 &mut module_rx,
17606 hello_frame("aft", PROTOCOL_VERSION, 395),
17607 )
17608 .await;
17609
17610 let client_connection = ConnectionId::new(96);
17611 let (client_ctx, _client_rx) = route_ctx(client_connection);
17612 let (route_task, bind) = relay_route_open(
17613 &handler,
17614 client_connection,
17615 &client_ctx.egress,
17616 &mut module_rx,
17617 396,
17618 "aft",
17619 "spoofed-reloading",
17620 )
17621 .await;
17622 let rejection = Frame::build(
17625 FrameType::Error,
17626 crate::router::daemon_error_flags(),
17627 0,
17628 0,
17629 bind.header.corr,
17630 serde_json::to_vec(&ErrorBody::new(
17631 "module_reloading",
17632 "module refused route.bind",
17633 ))
17634 .unwrap(),
17635 )
17636 .unwrap();
17637 handler
17638 .handle_control_frame(&module_ctx, rejection)
17639 .await
17640 .unwrap();
17641
17642 let response = route_task.await.unwrap();
17643 assert_eq!(response.len(), 1);
17644 assert_eq!(response[0].header.ty, FrameType::Error);
17645 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17646 assert!(
17647 !response[0].header.flags.is_daemon_origin(),
17648 "a module's own rejection must not carry daemon origin: {:?}",
17649 response[0].header.flags
17650 );
17651 }
17652
17653 #[tokio::test(flavor = "current_thread")]
17654 async fn route_open_module_rejection_uses_daemon_counter_key() {
17655 let registry = Arc::new(Registry::default());
17656 let forwarding = Arc::new(ForwardingTable::default());
17657 let handler =
17658 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17659 let module_connection = ConnectionId::new(95);
17660 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17661 hello_via_sink(
17662 &handler,
17663 &module_ctx,
17664 &mut module_rx,
17665 hello_frame("aft", PROTOCOL_VERSION, 395),
17666 )
17667 .await;
17668
17669 let client_connection = ConnectionId::new(96);
17670 let (client_ctx, _client_rx) = route_ctx(client_connection);
17671 let capture = EventCapture::default();
17672 static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
17679 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
17680 let _subscriber =
17681 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17682 let (route_task, bind) = relay_route_open(
17683 &handler,
17684 client_connection,
17685 &client_ctx.egress,
17686 &mut module_rx,
17687 396,
17688 "aft",
17689 "hostile-module-code",
17690 )
17691 .await;
17692 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
17693 let rejection = Frame::build(
17694 FrameType::Error,
17695 control_flags(),
17696 0,
17697 0,
17698 bind.header.corr,
17699 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
17700 )
17701 .unwrap();
17702 handler
17703 .handle_control_frame(&module_ctx, rejection)
17704 .await
17705 .unwrap();
17706
17707 let response = route_task.await.unwrap();
17708 assert_eq!(parse_error(&response[0])["code"], hostile_code);
17709 let counters = handler.counters().snapshot();
17710 assert_eq!(
17711 counters["route_open_refused_by_code"],
17712 json!({ "module_rejected": 1 })
17713 );
17714 assert!(counters["route_open_refused_by_code"]
17715 .get(hostile_code)
17716 .is_none());
17717
17718 let event = capture
17719 .events()
17720 .into_iter()
17721 .find(|event| {
17722 event.target == "control"
17723 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
17724 })
17725 .expect("route.open module-rejection refusal event");
17726 let logged = event.fields.get("module_code").expect("module_code field");
17727 assert!(!logged.bytes().any(|byte| byte < 0x20));
17728 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17729 }
17730
17731 #[tokio::test]
17732 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
17733 let registry = Arc::new(Registry::default());
17734 let supervisor_handle = SupervisorHandle::new();
17735 let missing_program = std::env::temp_dir().join(format!(
17736 "subc-route-open-missing-program-{}",
17737 std::process::id()
17738 ));
17739 let supervisor =
17740 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17741 .with_handle(supervisor_handle.clone());
17742 let module = supervisor
17743 .supervise_configured(
17744 ModuleSpec {
17745 script: None,
17746 carry_containment: false,
17747 module_id: "failed".to_string(),
17748 program: missing_program,
17749 args: Vec::new(),
17750 env: Vec::new(),
17751 reserved: false,
17752 reserved_prefixes: Vec::new(),
17753 protocol: ModuleProtocol::Subc,
17754 overlap: Default::default(),
17755 },
17756 true,
17757 )
17758 .unwrap();
17759 assert_eq!(module.state().unwrap(), ModuleState::Failed);
17760
17761 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17762 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
17763 let response = handler
17764 .handle_control_frame(
17765 &ctx,
17766 route_open_frame(305, "failed", unique_project_root("failed")),
17767 )
17768 .await
17769 .unwrap();
17770
17771 assert_eq!(response[0].header.ty, FrameType::Error);
17772 let error = parse_error(&response[0]);
17773 assert_eq!(error["code"], "target_unavailable");
17774 assert!(error["message"]
17775 .as_str()
17776 .unwrap()
17777 .contains("state=failed, enabled=true, live=false"));
17778 }
17779
17780 #[tokio::test]
17781 async fn route_open_role_mismatch_remains_target_unavailable() {
17782 let registry = Arc::new(Registry::default());
17783 let handler = ControlHandler::new(Arc::clone(®istry));
17784 handler
17785 .handle_control(
17786 ConnectionId::new(41),
17787 non_routable_hello_frame_with_control_ops("health-only", 306, None),
17788 )
17789 .unwrap();
17790
17791 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
17792 let response = handler
17793 .handle_control_frame(
17794 &ctx,
17795 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
17796 )
17797 .await
17798 .unwrap();
17799
17800 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17801 assert!(parse_error(&response[0])["message"]
17802 .as_str()
17803 .unwrap()
17804 .contains("does not provide the requested target"));
17805 }
17806
17807 #[tokio::test]
17808 async fn route_open_inactive_registration_remains_target_unavailable() {
17809 let registry = Arc::new(Registry::default());
17810 let handler = ControlHandler::new(Arc::clone(®istry));
17811 handler
17812 .handle_control(
17813 ConnectionId::new(43),
17814 hello_frame("inactive", PROTOCOL_VERSION, 308),
17815 )
17816 .unwrap();
17817 assert!(registry
17818 .set_module_state_for_test("inactive", ChannelState::Closed)
17819 .unwrap());
17820
17821 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
17822 let response = handler
17823 .handle_control_frame(
17824 &ctx,
17825 route_open_frame(309, "inactive", unique_project_root("inactive")),
17826 )
17827 .await
17828 .unwrap();
17829
17830 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17831 assert!(parse_error(&response[0])["message"]
17832 .as_str()
17833 .unwrap()
17834 .contains("is not active"));
17835 }
17836
17837 #[tokio::test]
17838 async fn late_health_reply_is_recorded_through_the_module_response_path() {
17839 let registry = Arc::new(Registry::default());
17840 let forwarding = Arc::new(ForwardingTable::default());
17841 let supervisor_handle = SupervisorHandle::new();
17842 let supervisor =
17843 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
17844 .with_forwarding(Arc::clone(&forwarding))
17845 .with_handle(supervisor_handle.clone());
17846 let module = supervisor
17847 .supervise_configured(
17848 crate::ModuleSpec {
17849 script: None,
17850 carry_containment: false,
17851 module_id: "late-health-response".to_string(),
17852 program: PathBuf::from("disabled-module"),
17853 args: Vec::new(),
17854 env: Vec::new(),
17855 reserved: false,
17856 reserved_prefixes: Vec::new(),
17857 protocol: ModuleProtocol::Subc,
17858 overlap: Default::default(),
17859 },
17860 false,
17861 )
17862 .unwrap();
17863 let handler =
17864 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17865 .with_supervisor(supervisor_handle);
17866 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
17867 handler
17868 .handle_control_frame(
17869 &module_ctx,
17870 hello_frame_with_control_ops(
17871 "late-health-response",
17872 PROTOCOL_VERSION,
17873 7,
17874 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17875 ),
17876 )
17877 .await
17878 .unwrap();
17879 let probe_started_at = Instant::now() - Duration::from_millis(80);
17880 let pending = forwarding
17881 .begin_health_probe_rpc_for(
17882 "late-health-response",
17883 MODULE_CONTROL_OP_HEALTH_CHECK,
17884 probe_started_at,
17885 Instant::now() - Duration::from_millis(1),
17886 )
17887 .unwrap();
17888 assert!(forwarding
17889 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
17890 .unwrap());
17891
17892 let responses = handler
17893 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
17894 .await
17895 .unwrap();
17896
17897 assert!(responses.is_empty());
17898 let health = module.status().unwrap().health;
17899 assert_eq!(health.late_answer_count, 1);
17900 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
17901 }
17902
17903 #[tokio::test]
17904 async fn health_probe_timeout_and_module_death_are_typed() {
17905 let registry = Arc::new(Registry::default());
17906 let forwarding = Arc::new(ForwardingTable::default());
17907 let handler =
17908 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17909 .with_health_probe_timeout(Duration::from_millis(50));
17910 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
17911 hello_via_sink(
17912 &handler,
17913 &module_ctx,
17914 &mut module_rx,
17915 hello_frame_with_control_ops(
17916 "aft",
17917 PROTOCOL_VERSION,
17918 7,
17919 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17920 ),
17921 )
17922 .await;
17923
17924 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
17925 let responses = handler
17926 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
17927 .await
17928 .unwrap();
17929 assert_eq!(responses[0].header.ty, FrameType::Error);
17930 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
17931 let _ = module_rx.try_recv();
17932
17933 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
17934 let health_handler = handler.clone();
17935 let death_task = tokio::spawn(async move {
17936 health_handler
17937 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
17938 .await
17939 .unwrap()
17940 });
17941 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
17942 .await
17943 .unwrap()
17944 .unwrap();
17945 handler
17946 .cleanup_connection(module_ctx.connection_id)
17947 .unwrap();
17948 let responses = death_task.await.unwrap();
17949 assert_eq!(responses[0].header.ty, FrameType::Error);
17950 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
17951 }
17952
17953 #[test]
17954 fn hello_requires_exact_protocol_version() {
17955 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
17956 let registry = Arc::new(Registry::default());
17957 let handler = ControlHandler::new(Arc::clone(®istry));
17958 let responses = handler
17959 .handle_control(
17960 ConnectionId::new(connection),
17961 hello_frame("aft", offered, 9),
17962 )
17963 .unwrap();
17964
17965 assert_eq!(responses.len(), 1);
17966 assert_eq!(responses[0].header.ty, FrameType::Error);
17967 let error = parse_error(&responses[0]);
17968 assert_eq!(error["code"], "version_unsupported");
17969 assert!(registry.get_module("aft").unwrap().is_none());
17970 assert_eq!(registry.active_registration_count().unwrap(), 0);
17971 }
17972 }
17973
17974 #[test]
17975 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
17976 let registry = Arc::new(Registry::default());
17977 let forwarding = Arc::new(ForwardingTable::default());
17978 let handler =
17979 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17980 let module_connection = ConnectionId::new(301);
17981 let registration = registry
17982 .register_with_control_ops(
17983 manifest("aft-push", PROTOCOL_VERSION),
17984 PROTOCOL_VERSION,
17985 module_connection,
17986 module_baseline_control_ops(),
17987 )
17988 .unwrap();
17989 let (module_tx, _module_rx) = mpsc::channel(8);
17990 let endpoint = forwarding
17991 .register_module_connection(
17992 module_connection,
17993 "aft-push".to_string(),
17994 PROTOCOL_VERSION,
17995 manifest_concurrency(®istration.manifest),
17996 FrameSink::new(module_tx),
17997 )
17998 .unwrap();
17999
18000 let unknown = Frame::build(
18002 FrameType::Push,
18003 control_flags(),
18004 0,
18005 0,
18006 5,
18007 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
18008 )
18009 .unwrap();
18010 let out = handler.handle_status_update(endpoint, unknown).unwrap();
18011 assert!(
18012 out.is_empty(),
18013 "unknown push op must be ignored, got {out:?}"
18014 );
18015
18016 let malformed = Frame::build(
18018 FrameType::Push,
18019 control_flags(),
18020 0,
18021 0,
18022 6,
18023 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
18024 )
18025 .unwrap();
18026 let out = handler.handle_status_update(endpoint, malformed).unwrap();
18027 assert_eq!(out.len(), 1);
18028 assert_eq!(out[0].header.ty, FrameType::Error);
18029 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
18030 }
18031
18032 #[test]
18033 fn hello_rejected_when_connection_already_owns_client_routes() {
18034 let registry = Arc::new(Registry::default());
18035 let forwarding = Arc::new(ForwardingTable::default());
18036 let handler =
18037 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18038 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
18040 let client_connection = ConnectionId::new(202);
18041
18042 let responses = handler
18045 .handle_control(
18046 client_connection,
18047 hello_frame("aft-second", PROTOCOL_VERSION, 9),
18048 )
18049 .unwrap();
18050 assert_eq!(responses[0].header.ty, FrameType::Error);
18051 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
18052 assert!(registry.get_module("aft-second").unwrap().is_none());
18053 }
18054
18055 #[tokio::test]
18056 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
18057 let registry = Arc::new(Registry::default());
18058 let forwarding = Arc::new(ForwardingTable::default());
18059 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
18060 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
18061 hello_via_sink(
18062 &handler,
18063 &module_ctx,
18064 &mut module_rx,
18065 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
18066 )
18067 .await;
18068 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
18069 let pending = forwarding
18070 .begin_route_bind_relay_for_test(
18071 client_ctx.connection_id,
18072 client_ctx.egress.clone(),
18073 2,
18074 "alpha",
18075 )
18076 .unwrap();
18077 forwarding
18078 .complete_pending_relay(
18079 module_ctx.connection_id,
18080 pending.corr,
18081 RouteBindRelayOutcome::Accepted,
18082 )
18083 .unwrap();
18084 client_rx.try_recv().unwrap();
18085 for module_id in ["beta", "alpha"] {
18086 let replies = handler
18087 .handle_control_frame(
18088 &module_ctx,
18089 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
18090 )
18091 .await
18092 .unwrap();
18093 assert_eq!(replies.len(), 1, "second HELLO must be refused");
18094 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
18095 }
18096 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
18097 assert!(registry.get_module("beta").unwrap().is_none());
18098 assert!(matches!(
18099 forwarding
18100 .lookup_data_route(
18101 client_ctx.connection_id,
18102 pending.client_channel,
18103 pending.client_epoch,
18104 )
18105 .unwrap(),
18106 DataRoute::Client(DataRouteState::Bound(_))
18107 ));
18108 assert!(handler
18109 .hello_launch_nonces
18110 .lock()
18111 .unwrap()
18112 .presented(module_ctx.connection_id, Some("alpha-nonce")));
18113 assert!(module_rx.try_recv().is_err());
18114 }
18115
18116 #[test]
18117 fn reserved_module_hello_requires_matching_launch_nonce() {
18118 let registry = Arc::new(Registry::default());
18119 let supervisor = SupervisorHandle::new();
18120 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
18123 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18124
18125 let no_nonce = handler
18127 .handle_control(
18128 ConnectionId::new(1),
18129 hello_frame("vault", PROTOCOL_VERSION, 1),
18130 )
18131 .unwrap();
18132 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
18133 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
18134 assert!(registry.get_module("vault").unwrap().is_none());
18135
18136 let wrong = handler
18138 .handle_control(
18139 ConnectionId::new(2),
18140 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
18141 )
18142 .unwrap();
18143 assert_eq!(wrong[0].header.ty, FrameType::Error);
18144 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
18145 assert!(registry.get_module("vault").unwrap().is_none());
18146
18147 let ok = handler
18149 .handle_control(
18150 ConnectionId::new(3),
18151 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
18152 )
18153 .unwrap();
18154 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
18155 assert!(registry.get_module("vault").unwrap().is_some());
18156 }
18157
18158 #[test]
18159 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
18160 let registry = Arc::new(Registry::default());
18161 let supervisor = SupervisorHandle::new();
18162 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18163 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18164 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18165
18166 let squat = handler
18167 .handle_control(
18168 ConnectionId::new(1),
18169 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
18170 )
18171 .unwrap();
18172 assert_eq!(squat[0].header.ty, FrameType::Error);
18173 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
18174 assert!(parse_error(&squat[0])["message"]
18175 .as_str()
18176 .unwrap()
18177 .contains("fed:"));
18178
18179 let accepted_peer = handler
18180 .handle_control(
18181 ConnectionId::new(2),
18182 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
18183 )
18184 .unwrap();
18185 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
18186
18187 let accepted_short = handler
18188 .handle_control(
18189 ConnectionId::new(3),
18190 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
18191 )
18192 .unwrap();
18193 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
18194
18195 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
18196 let response = handler
18197 .handle_control(
18198 ConnectionId::new(conn),
18199 hello_frame(module_id, PROTOCOL_VERSION, conn),
18200 )
18201 .unwrap();
18202 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
18203 }
18204 }
18205
18206 #[test]
18207 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
18208 let registry = Arc::new(Registry::default());
18209 let supervisor = SupervisorHandle::new();
18210 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18211 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18212 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
18213 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18214
18215 let owner_nonce = handler
18216 .handle_control(
18217 ConnectionId::new(1),
18218 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
18219 )
18220 .unwrap();
18221 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
18222 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
18223 assert!(registry.get_module("fed:special").unwrap().is_none());
18224
18225 let exact_nonce = handler
18226 .handle_control(
18227 ConnectionId::new(2),
18228 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
18229 )
18230 .unwrap();
18231 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
18232 assert!(registry.get_module("fed:special").unwrap().is_some());
18233 }
18234
18235 #[test]
18236 fn non_reserved_module_ignores_launch_nonce() {
18237 let registry = Arc::new(Registry::default());
18238 let handler = ControlHandler::new(Arc::clone(®istry));
18241 let no_nonce = handler
18242 .handle_control(
18243 ConnectionId::new(1),
18244 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
18245 )
18246 .unwrap();
18247 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
18248 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
18249
18250 let echoed_nonce = handler
18251 .handle_control(
18252 ConnectionId::new(2),
18253 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
18254 )
18255 .unwrap();
18256 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
18257 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
18258 }
18259
18260 #[test]
18261 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
18262 let handler = ControlHandler::default();
18263 let conn = ConnectionId::new(1);
18264 let malformed = Frame::build(
18265 FrameType::Hello,
18266 control_flags(),
18267 0,
18268 0,
18269 3,
18270 b"{not json".to_vec(),
18271 )
18272 .unwrap();
18273
18274 let error = handler.handle_control(conn, malformed).unwrap();
18275 assert_eq!(error[0].header.ty, FrameType::Error);
18276 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
18277
18278 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18279 let pong = handler.handle_control(conn, ping).unwrap();
18280 assert_eq!(pong[0].header.ty, FrameType::Pong);
18281 assert_eq!(pong[0].header.corr, 4);
18282 }
18283
18284 #[test]
18285 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
18286 let registry = Arc::new(Registry::default());
18287 let handler = ControlHandler::new(Arc::clone(®istry));
18288
18289 handler
18290 .handle_control(
18291 ConnectionId::new(1),
18292 hello_frame("aft", PROTOCOL_VERSION, 1),
18293 )
18294 .unwrap();
18295 let duplicate = handler
18296 .handle_control(
18297 ConnectionId::new(2),
18298 hello_frame("aft", PROTOCOL_VERSION, 2),
18299 )
18300 .unwrap();
18301
18302 assert_eq!(duplicate[0].header.ty, FrameType::Error);
18303 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18304 let registration = registry.get_module("aft").unwrap().unwrap();
18305 assert_eq!(registration.connection_id, ConnectionId::new(1));
18306 }
18307
18308 #[test]
18309 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
18310 let registry = Arc::new(Registry::default());
18311 let forwarding = Arc::new(ForwardingTable::default());
18312 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
18313 let handler =
18314 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18315 .with_process_liveness(process_liveness);
18316 let (ctx, route_channel, route_epoch) =
18317 bind_liveness_route(®istry, &forwarding, "aft-dead");
18318 let responses = handler
18319 .handle_route_poll(
18320 &ctx,
18321 route_poll_frame(41, PollKind::Liveness, route_channel),
18322 route_channel,
18323 route_epoch,
18324 PollKind::Liveness,
18325 )
18326 .unwrap();
18327
18328 assert_eq!(responses.len(), 1);
18329 assert_eq!(responses[0].header.ty, FrameType::Response);
18330 assert_route_poll_liveness(&responses[0], false);
18331 }
18332
18333 #[test]
18334 fn liveness_poll_without_process_source_uses_bound_route() {
18335 let registry = Arc::new(Registry::default());
18336 let forwarding = Arc::new(ForwardingTable::default());
18337 let handler =
18338 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18339 let (ctx, route_channel, route_epoch) =
18340 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
18341 let responses = handler
18342 .handle_route_poll(
18343 &ctx,
18344 route_poll_frame(42, PollKind::Liveness, route_channel),
18345 route_channel,
18346 route_epoch,
18347 PollKind::Liveness,
18348 )
18349 .unwrap();
18350
18351 assert_route_poll_liveness(&responses[0], true);
18352 }
18353
18354 #[test]
18355 fn liveness_poll_untracked_process_source_uses_bound_route() {
18356 let registry = Arc::new(Registry::default());
18357 let forwarding = Arc::new(ForwardingTable::default());
18358 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
18359 let handler =
18360 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18361 .with_process_liveness(process_liveness);
18362 let (ctx, route_channel, route_epoch) =
18363 bind_liveness_route(®istry, &forwarding, "aft-untracked");
18364 let responses = handler
18365 .handle_route_poll(
18366 &ctx,
18367 route_poll_frame(43, PollKind::Liveness, route_channel),
18368 route_channel,
18369 route_epoch,
18370 PollKind::Liveness,
18371 )
18372 .unwrap();
18373
18374 assert_route_poll_liveness(&responses[0], true);
18375 }
18376
18377 #[tokio::test]
18378 async fn unknown_op_returns_unknown_control_op() {
18379 let handler = ControlHandler::default();
18380 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
18381 let request = Frame::build(
18382 FrameType::Request,
18383 control_flags(),
18384 0,
18385 0,
18386 55,
18387 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
18388 )
18389 .unwrap();
18390
18391 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18392
18393 assert_eq!(response.len(), 1);
18394 assert_eq!(response[0].header.ty, FrameType::Error);
18395 assert_eq!(response[0].header.corr, 55);
18396 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
18397 }
18398
18399 #[tokio::test]
18400 async fn supervisor_provenance_rejects_unknown_exact_module() {
18401 let handler = ControlHandler::default();
18402 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
18403 let request = Frame::build(
18404 FrameType::Request,
18405 control_flags(),
18406 0,
18407 0,
18408 57,
18409 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
18410 )
18411 .unwrap();
18412
18413 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18414
18415 assert_eq!(response.len(), 1);
18416 assert_eq!(response[0].header.ty, FrameType::Error);
18417 assert_eq!(response[0].header.corr, 57);
18418 let error = parse_error(&response[0]);
18419 assert_eq!(error["code"], "unknown_module");
18420 assert_eq!(error["message"], "module_id 'missing' is not supervised");
18421 }
18422
18423 #[test]
18424 fn provenance_probe_override_keeps_handler_tests_deterministic() {
18425 let expected = subc_control::RunningImageAgreement::Unavailable {
18426 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18427 };
18428 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
18429 assert_eq!(handler.provenance_probe_override, Some(expected));
18430 }
18431
18432 #[test]
18433 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
18434 let verdict = reload_verdict(
18435 std::path::Path::new("/bin/new"),
18436 Some(std::path::Path::new("/bin/old")),
18437 subc_control::RunningImageAgreement::Unavailable {
18438 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18439 },
18440 );
18441 assert!(matches!(
18442 verdict.path,
18443 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
18444 if configured == std::path::Path::new("/bin/new")
18445 && spawned_from == std::path::Path::new("/bin/old")
18446 ));
18447 }
18448
18449 #[test]
18450 fn launch_spec_without_running_digest_is_unavailable_not_match() {
18451 use subc_control::{
18452 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
18453 };
18454 let configured = crate::supervise::ModuleSpec {
18455 script: None,
18456 carry_containment: false,
18457 module_id: "legacy".into(),
18458 program: "/module".into(),
18459 args: Vec::new(),
18460 env: Vec::new(),
18461 reserved: false,
18462 reserved_prefixes: Vec::new(),
18463 protocol: subc_control::ModuleProtocol::Subc,
18464 overlap: Default::default(),
18465 };
18466 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
18467 assert_eq!(
18468 launch_spec_agreement(&configured, false, Some(&digest)),
18469 Agreement::Unavailable {
18470 reason: Reason::NotRunning
18471 }
18472 );
18473 assert_eq!(
18474 launch_spec_agreement(&configured, true, None),
18475 Agreement::Unavailable {
18476 reason: Reason::SpawnedSpecUnavailable
18477 }
18478 );
18479 assert_eq!(
18480 launch_spec_agreement(&configured, true, Some(&digest)),
18481 Agreement::Match
18482 );
18483 }
18484
18485 #[cfg(windows)]
18486 #[test]
18487 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
18488 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
18489 let configured = dir.join("module.exe");
18490 std::fs::write(&configured, b"executable path fixture").unwrap();
18491 let spawned = std::fs::canonicalize(&configured).unwrap();
18492 assert_ne!(
18493 configured, spawned,
18494 "fixture must exercise different path spellings"
18495 );
18496 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
18497 let lower_case = configured.to_string_lossy().to_lowercase();
18498 for spelling in [
18499 &configured,
18500 Path::new(&forward_slashes),
18501 Path::new(&lower_case),
18502 ] {
18503 let verdict = reload_verdict(
18504 spelling,
18505 Some(&spawned),
18506 subc_control::RunningImageAgreement::Unavailable {
18507 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18508 },
18509 );
18510 assert_eq!(
18511 verdict.path,
18512 ReloadPathAgreement::Match,
18513 "configured {spelling:?}, spawned {spawned:?}"
18514 );
18515 }
18516 }
18517
18518 #[cfg(windows)]
18519 #[test]
18520 fn windows_reload_path_keeps_different_existing_program_pending() {
18521 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
18522 let old = dir.join("old.exe");
18523 let new = dir.join("new.exe");
18524 std::fs::write(&old, b"same executable content").unwrap();
18525 std::fs::copy(&old, &new).unwrap();
18526 let spawned = std::fs::canonicalize(&old).unwrap();
18527 let image = subc_control::RunningImageAgreement::Unavailable {
18528 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18529 };
18530 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
18531 assert!(
18532 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
18533 if configured == new && spawned_from == spawned)
18534 );
18535 let missing = dir.join("missing.exe");
18536 assert!(matches!(
18537 reload_verdict(&missing, Some(&spawned), image).path,
18538 ReloadPathAgreement::Mismatch { .. }
18539 ));
18540 }
18541
18542 #[test]
18543 fn reload_verdict_detects_replaced_image_at_same_path() {
18544 let image = subc_control::RunningImageAgreement::Mismatch {
18545 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
18546 digest: "old".into(),
18547 },
18548 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
18549 digest: "new".into(),
18550 },
18551 };
18552 let verdict = reload_verdict(
18553 std::path::Path::new("/bin/same"),
18554 Some(std::path::Path::new("/bin/same")),
18555 image.clone(),
18556 );
18557 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18558 assert_eq!(verdict.image, image);
18559 }
18560
18561 #[test]
18562 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
18563 let image = subc_control::RunningImageAgreement::Unavailable {
18564 reason: subc_control::RunningImageUnavailableReason::NotRunning,
18565 };
18566 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
18567 assert_eq!(
18568 verdict.path,
18569 subc_control::ReloadPathAgreement::Unavailable {
18570 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
18571 }
18572 );
18573 assert_eq!(verdict.image, image);
18574
18575 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
18576 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18577 };
18578 let verdict = reload_verdict(
18579 std::path::Path::new("/bin/same"),
18580 Some(std::path::Path::new("/bin/same")),
18581 unconfirmed.clone(),
18582 );
18583 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18584 assert_eq!(verdict.image, unconfirmed);
18585 }
18586
18587 #[test]
18588 fn reload_verdict_preserves_each_image_unavailability_reason() {
18589 use subc_control::RunningImageUnavailableReason as Reason;
18590
18591 for reason in [
18592 Reason::NotRunning,
18593 Reason::UnsupportedPlatform,
18594 Reason::RunningExecutableUnreadable,
18595 Reason::SpawnedPathUnreadable,
18596 Reason::HashFailed,
18597 Reason::ProcessIdentityUnconfirmed,
18598 Reason::Unknown("future_probe_reason".to_string()),
18599 ] {
18600 let image = subc_control::RunningImageAgreement::Unavailable {
18601 reason: reason.clone(),
18602 };
18603 let verdict = reload_verdict(
18604 std::path::Path::new("/bin/same"),
18605 Some(std::path::Path::new("/bin/same")),
18606 image.clone(),
18607 );
18608 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18609 assert_eq!(verdict.image, image, "{reason:?}");
18610 }
18611 }
18612
18613 #[tokio::test]
18614 async fn malformed_control_bodies_return_invalid_control_body() {
18615 let handler = ControlHandler::default();
18616 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
18617
18618 for (corr, body) in [
18619 (56, br#"{"route_channel":1}"#.as_slice()),
18620 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
18621 (
18622 58,
18623 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
18624 ),
18625 ] {
18626 let request = Frame::build(
18627 FrameType::Request,
18628 control_flags(),
18629 0,
18630 0,
18631 corr,
18632 body.to_vec(),
18633 )
18634 .unwrap();
18635 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18636
18637 assert_eq!(response.len(), 1);
18638 assert_eq!(response[0].header.ty, FrameType::Error);
18639 assert_eq!(response[0].header.corr, corr);
18640 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
18641 }
18642 }
18643
18644 #[tokio::test]
18645 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
18646 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18647 let registry = Arc::new(Registry::default());
18648 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18649 let router = Router::with_control_handler(Arc::clone(&control));
18650 let connection = router.begin_connection();
18651 let (ctx, mut rx) = route_ctx(connection.id());
18652
18653 router
18654 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
18655 .await
18656 .unwrap();
18657 let response = rx.recv().await.unwrap();
18658 let ack = parse_ack(&response);
18659 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18660 let channel = 1;
18661
18662 let goodbye =
18663 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
18664 router.route_for_connection(&ctx, goodbye).await.unwrap();
18665 assert!(rx.try_recv().is_err());
18666 assert!(registry.get_module("aft").unwrap().is_none());
18667
18668 router
18669 .route_for_connection(&ctx, channel_request(channel, 13))
18670 .await
18671 .unwrap();
18672 let error_frame = rx.recv().await.unwrap();
18673 assert_eq!(error_frame.header.ty, FrameType::Error);
18674 assert_eq!(error_frame.header.channel, channel);
18675 let captured = crate::router::test_log::captured_logs(&logs);
18676 assert_eq!(
18677 captured
18678 .lines()
18679 .filter(|line| {
18680 line.contains("module_id=aft")
18681 && line.contains("reason=explicit_goodbye")
18682 && line.contains("module registration ended")
18683 })
18684 .count(),
18685 1,
18686 "unexpected GOODBYE registry log: {captured}"
18687 );
18688 }
18689
18690 #[tokio::test]
18691 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
18692 let registry = Arc::new(Registry::default());
18693 let handler = ControlHandler::new(registry).with_capability_config(
18694 [("prov".to_string(), true), ("cons".to_string(), true)],
18695 BTreeMap::new(),
18696 );
18697 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
18698 register_capability_manifest(
18699 &handler,
18700 &provider_ctx,
18701 &mut provider_rx,
18702 capability_manifest("prov", &["thing/v1"], &[]),
18703 1,
18704 )
18705 .await;
18706 let mut consumer = capability_manifest("cons", &[], &[]);
18707 consumer.capabilities.as_mut().unwrap().requires.push(
18708 subc_protocol::manifest::CapabilityRequirement {
18709 capability: "thing/v1".to_string(),
18710 need: subc_protocol::manifest::CapabilityNeed::Required,
18711 },
18712 );
18713 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
18714 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
18715 assert_eq!(
18716 handler.capability_evaluator.verdict("cons", "thing/v1"),
18717 Some(CapabilityVerdict::Provided)
18718 );
18719 let (mut client_rx, _) = open_route_for_capability_test(
18720 &handler,
18721 &provider_ctx,
18722 &mut provider_rx,
18723 703,
18724 3,
18725 "prov",
18726 None,
18727 )
18728 .await;
18729 let goodbye =
18730 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18731 handler
18732 .handle_control_frame(&provider_ctx, goodbye)
18733 .await
18734 .unwrap();
18735 assert_eq!(
18736 handler.capability_evaluator.verdict("cons", "thing/v1"),
18737 Some(CapabilityVerdict::NeverProvided)
18738 );
18739 let closed = client_rx
18740 .try_recv()
18741 .expect("GOODBYE pushes route.closed before route GOODBYE");
18742 assert!(
18743 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
18744 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
18745 );
18746 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
18747 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
18748 }
18749
18750 struct RouteClosedFixture {
18751 handler: ControlHandler,
18752 #[cfg(unix)]
18753 supervisor: Supervisor,
18754 module_ctx: RouteCtx,
18755 #[cfg(unix)]
18756 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18757 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18758 route: Arc<crate::forwarding::RouteBinding>,
18759 }
18760
18761 async fn route_closed_fixture() -> RouteClosedFixture {
18762 let registry = Arc::new(Registry::default());
18763 let handler = ControlHandler::new(Arc::clone(®istry));
18764 #[cfg(unix)]
18765 let supervisor = Supervisor::new(registry, RestartPolicy::default())
18766 .with_forwarding(Arc::clone(&handler.forwarding));
18767 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
18768 register_capability_manifest(
18769 &handler,
18770 &module_ctx,
18771 &mut module_rx,
18772 capability_manifest("prov", &[], &[]),
18773 1,
18774 )
18775 .await;
18776 let (client_rx, bind) = open_route_for_capability_test(
18777 &handler,
18778 &module_ctx,
18779 &mut module_rx,
18780 702,
18781 2,
18782 "prov",
18783 None,
18784 )
18785 .await;
18786 let ModuleControlRequest::RouteBind {
18787 route_channel: channel,
18788 epoch: route_epoch,
18789 ..
18790 } = bind
18791 else {
18792 panic!("expected route.bind");
18793 };
18794 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
18795 handler
18796 .forwarding
18797 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
18798 .unwrap()
18799 else {
18800 panic!("the acknowledged route must be bound");
18801 };
18802 RouteClosedFixture {
18803 handler,
18804 #[cfg(unix)]
18805 supervisor,
18806 module_ctx,
18807 #[cfg(unix)]
18808 module_rx,
18809 client_rx,
18810 route,
18811 }
18812 }
18813
18814 impl RouteClosedFixture {
18815 #[cfg(unix)]
18816 async fn cut(&mut self) {
18817 let notices = async {
18818 let mut draining = self.module_rx.recv().await.unwrap();
18819 assert!(matches!(
18820 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
18821 &draining.body
18822 )
18823 .unwrap(),
18824 subc_protocol::session::ModuleControlCommand::Draining {
18825 reason: RouteCloseReason::Disable,
18826 ..
18827 }
18828 ));
18829 draining.flushed.take().unwrap().send(()).unwrap();
18830 let mut closing = self.client_rx.recv().await.unwrap();
18831 assert!(matches!(
18832 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
18833 ClientControlPush::RouteClosing {
18834 reason: RouteCloseReason::Restart,
18835 ..
18836 }
18837 ));
18838 closing.flushed.take().unwrap().send(()).unwrap();
18839 };
18840 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
18841 result.unwrap();
18842 }
18843
18844 fn disconnect(&mut self) -> ClientControlPush {
18845 self.handler
18846 .cleanup_connection(self.module_ctx.connection_id)
18847 .unwrap();
18848 let closed = self.client_rx.try_recv().unwrap();
18849 assert_eq!(closed.header.ty, FrameType::Push);
18850 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
18851 assert!(matches!(
18852 &push,
18853 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
18854 if channels == &[self.route.client_channel]
18855 ));
18856 assert_eq!(
18857 self.client_rx.try_recv().unwrap().header.ty,
18858 FrameType::Goodbye
18859 );
18860 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
18861 push
18862 }
18863 }
18864
18865 #[cfg(unix)]
18866 #[tokio::test(start_paused = true)]
18867 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
18868 let mut fixture = route_closed_fixture().await;
18869 fixture.cut().await;
18870 assert!(matches!(
18871 fixture.disconnect(),
18872 ClientControlPush::RouteClosed {
18873 reason: RouteCloseReason::Restart,
18874 drained: true,
18875 ..
18876 }
18877 ));
18878 }
18879
18880 #[cfg(unix)]
18881 #[tokio::test(start_paused = true)]
18882 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
18883 let mut fixture = route_closed_fixture().await;
18884 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
18885 let started = tokio::time::Instant::now();
18886 fixture.cut().await;
18887 assert!(started.elapsed() >= Duration::from_secs(2));
18888 assert_eq!(fixture.route.flow.in_flight(), 1);
18889 assert!(matches!(
18890 fixture.disconnect(),
18891 ClientControlPush::RouteClosed {
18892 reason: RouteCloseReason::Restart,
18893 drained: false,
18894 ..
18895 }
18896 ));
18897 }
18898
18899 #[tokio::test]
18900 async fn module_connection_crash_reports_crash_and_not_drained() {
18901 let mut fixture = route_closed_fixture().await;
18902 assert!(matches!(
18906 fixture.disconnect(),
18907 ClientControlPush::RouteClosed {
18908 reason: RouteCloseReason::Crash,
18909 drained: false,
18910 ..
18911 }
18912 ));
18913 }
18914
18915 #[tokio::test]
18916 async fn dropping_router_connection_releases_registration() {
18917 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18918 let registry = Arc::new(Registry::default());
18919 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18920 let router = Router::with_control_handler(Arc::clone(&control));
18921 let connection = router.begin_connection();
18922 let connection_id = connection.id();
18923 let (ctx, mut rx) = route_ctx(connection_id);
18924
18925 router
18926 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
18927 .await
18928 .unwrap();
18929 let response = rx.recv().await.unwrap();
18930 let ack = parse_ack(&response);
18931 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18932 assert!(registry.get_module("aft").unwrap().is_some());
18933
18934 drop(connection);
18935
18936 assert!(registry.get_module("aft").unwrap().is_none());
18937 assert_eq!(registry.active_registration_count().unwrap(), 0);
18938
18939 control
18940 .cleanup_connection(ConnectionId::new(u64::MAX))
18941 .unwrap();
18942 let captured = crate::router::test_log::captured_logs(&logs);
18943 println!("captured registry lifecycle logs:\n{captured}");
18944 let events: Vec<_> = captured
18945 .lines()
18946 .filter(|line| {
18947 line.contains("module registered module_id=aft ")
18948 || line.contains("module registration ended")
18949 })
18950 .collect();
18951 assert_eq!(
18952 events.len(),
18953 2,
18954 "unexpected registry lifecycle logs: {captured}"
18955 );
18956 assert!(events[0].contains("module registered module_id=aft "));
18957 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
18958 assert!(events[1].contains(&format!(
18959 "module_id=aft connection_id={}",
18960 connection_id.get()
18961 )));
18962 assert!(events[1].contains("reason=connection_closed"));
18963 assert!(events[1].contains("module registration ended"));
18964 assert_eq!(
18965 captured
18966 .lines()
18967 .filter(|line| line.contains("module registered module_id=aft "))
18968 .count(),
18969 1,
18970 "legacy registration admission line must appear once: {captured}"
18971 );
18972 }
18973
18974 #[tokio::test]
18975 async fn hello_registration_keeps_legacy_module_registered_line_once() {
18976 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18977 let registry = Arc::new(Registry::default());
18978 let control = ControlHandler::new(Arc::clone(®istry));
18979 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
18980 hello_via_sink(
18981 &control,
18982 &ctx,
18983 &mut rx,
18984 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
18985 )
18986 .await;
18987
18988 let captured = crate::router::test_log::captured_logs(&logs);
18989 let admissions: Vec<_> = captured
18990 .lines()
18991 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
18992 .collect();
18993 assert_eq!(
18994 admissions.len(),
18995 1,
18996 "legacy admission line must remain exactly once: {captured}"
18997 );
18998 assert!(admissions[0].contains("routable_provider=true"));
18999 assert!(admissions[0].contains("connection_id=777"));
19000 }
19001
19002 fn capability_manifest(
19003 module_id: &str,
19004 provides: &[&str],
19005 must_never_reach: &[&str],
19006 ) -> ModuleManifest {
19007 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
19008 manifest.capabilities = Some(CapabilityDeclarations {
19009 provides: provides
19010 .iter()
19011 .map(|capability| (*capability).to_string())
19012 .collect(),
19013 requires: Vec::new(),
19014 must_never_reach: must_never_reach
19015 .iter()
19016 .map(|capability| (*capability).to_string())
19017 .collect(),
19018 });
19019 manifest
19020 }
19021
19022 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
19023 Frame::build(
19024 FrameType::Hello,
19025 control_flags(),
19026 0,
19027 0,
19028 corr,
19029 serde_json::to_vec(&ModuleHelloBody {
19030 protocol_ver: manifest.protocol_ver,
19031 manifest,
19032 control_ops: None,
19033 launch_nonce: None,
19034 })
19035 .expect("capability test HELLO serializes"),
19036 )
19037 .expect("capability test HELLO frame builds")
19038 }
19039
19040 fn catalog_update_with_capabilities_frame(
19041 corr: u64,
19042 capabilities: CapabilityDeclarations,
19043 ) -> Frame {
19044 Frame::build(
19045 FrameType::Request,
19046 control_flags(),
19047 0,
19048 0,
19049 corr,
19050 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
19051 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
19052 capabilities: Some(capabilities),
19053 ready: None,
19054 })
19055 .expect("capability catalog.update serializes"),
19056 )
19057 .expect("capability catalog.update frame builds")
19058 }
19059
19060 async fn register_capability_manifest(
19061 handler: &ControlHandler,
19062 ctx: &RouteCtx,
19063 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19064 manifest: ModuleManifest,
19065 corr: u64,
19066 ) {
19067 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
19068 }
19069
19070 async fn open_route_for_capability_test(
19071 handler: &ControlHandler,
19072 target_ctx: &RouteCtx,
19073 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19074 client_connection_id: u64,
19075 corr: u64,
19076 target_module_id: &str,
19077 consumer_identity: Option<ConsumerIdentity>,
19078 ) -> (
19079 mpsc::Receiver<crate::router::OutboundFrame>,
19080 ModuleControlRequest,
19081 ) {
19082 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
19083 let route_handler = handler.clone();
19084 let target_module_id = target_module_id.to_string();
19085 let route_task = tokio::spawn(async move {
19086 route_handler
19087 .handle_control_frame(
19088 &client_ctx,
19089 route_open_frame_with_admission_facts(
19090 corr,
19091 &target_module_id,
19092 unique_project_root("admission-facts"),
19093 consumer_identity,
19094 None,
19095 ),
19096 )
19097 .await
19098 .expect("capability test route.open succeeds")
19099 });
19100 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
19101 .await
19102 .expect("capability test route.open must reach route.bind")
19103 .expect("target control receiver stays open");
19104 let bind_request: ModuleControlRequest =
19105 serde_json::from_slice(&bind.body).expect("route.bind decodes");
19106 handler
19107 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
19108 .await
19109 .expect("capability test route.bind ACK succeeds");
19110 assert!(route_task.await.expect("route.open task joins").is_empty());
19111 let opened = client_rx
19112 .recv()
19113 .await
19114 .expect("successful route.open publishes a response");
19115 assert!(matches!(
19116 serde_json::from_slice::<ClientControlResponse>(&opened.body),
19117 Ok(ClientControlResponse::RouteOpen { .. })
19118 ));
19119 (client_rx, bind_request)
19120 }
19121
19122 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
19123 assert_eq!(frame.header.ty, FrameType::Push);
19124 assert_eq!(frame.header.channel, 0);
19125 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
19126 .expect("route.closed control push decodes");
19127 let ClientControlPush::RouteClosed { channels, .. } = &push else {
19128 panic!("expected route.closed");
19129 };
19130 assert_eq!(channels.len(), 1, "exactly one violating route closed");
19131 let closed_channel = channels[0];
19132 let channels = channels.clone();
19133 assert_eq!(
19134 push,
19135 ClientControlPush::RouteClosed {
19136 module_id: target_module_id.to_string(),
19137 channels,
19138 reason: RouteCloseReason::CapabilityDenied,
19139 drained: false,
19140 abandoned: 0,
19141 excluded_subscriptions: 0,
19142 terminal: Some(false),
19143 }
19144 );
19145 closed_channel
19146 }
19147
19148 #[tokio::test]
19149 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
19150 let registry = Arc::new(Registry::default());
19151 let forwarding = Arc::new(ForwardingTable::default());
19152 let supervisor = SupervisorHandle::new();
19153 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19154 let handler =
19155 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19156 .with_supervisor(supervisor);
19157 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
19158 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
19159 register_capability_manifest(
19160 &handler,
19161 &target_ctx,
19162 &mut target_rx,
19163 capability_manifest("target", &["credentials-provider/v1"], &[]),
19164 1,
19165 )
19166 .await;
19167 register_capability_manifest(
19168 &handler,
19169 &opener_ctx,
19170 &mut opener_rx,
19171 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19172 2,
19173 )
19174 .await;
19175
19176 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
19177 let replies = handler
19178 .handle_control_frame(
19179 &client_ctx,
19180 route_open_frame_with_admission_facts(
19181 3,
19182 "target",
19183 unique_project_root("admission-facts"),
19184 Some(ConsumerIdentity {
19185 module_id: "opener".to_string(),
19186 launch_nonce: "opener-nonce".to_string(),
19187 }),
19188 None,
19189 ),
19190 )
19191 .await
19192 .expect("denied route.open returns a typed frame");
19193 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19194 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19195 assert!(
19196 target_rx.try_recv().is_err(),
19197 "forbidden route.open must not relay route.bind"
19198 );
19199 }
19200
19201 #[tokio::test]
19202 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
19203 let registry = Arc::new(Registry::default());
19204 let forwarding = Arc::new(ForwardingTable::default());
19205 let supervisor = SupervisorHandle::new();
19206 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19207 let handler =
19208 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19209 .with_supervisor(supervisor);
19210 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
19211 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
19212 register_capability_manifest(
19213 &handler,
19214 &target_ctx,
19215 &mut target_rx,
19216 capability_manifest("target", &["credentials-provider/v1"], &[]),
19217 1,
19218 )
19219 .await;
19220 register_capability_manifest(
19221 &handler,
19222 &old_opener_ctx,
19223 &mut old_opener_rx,
19224 capability_manifest("opener", &[], &[]),
19225 2,
19226 )
19227 .await;
19228 let (mut client_rx, _) = open_route_for_capability_test(
19229 &handler,
19230 &target_ctx,
19231 &mut target_rx,
19232 712,
19233 3,
19234 "target",
19235 Some(ConsumerIdentity {
19236 module_id: "opener".to_string(),
19237 launch_nonce: "opener-nonce".to_string(),
19238 }),
19239 )
19240 .await;
19241 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19242
19243 handler
19244 .cleanup_connection(old_opener_ctx.connection_id)
19245 .expect("old opener registration cleans up");
19246 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
19247 register_capability_manifest(
19248 &handler,
19249 &new_opener_ctx,
19250 &mut new_opener_rx,
19251 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19252 4,
19253 )
19254 .await;
19255
19256 let closed_channel = assert_capability_denied_push(
19257 client_rx
19258 .try_recv()
19259 .expect("HELLO deny addition must emit route.closed")
19260 .frame,
19261 "target",
19262 );
19263 let goodbye = client_rx
19264 .try_recv()
19265 .expect("HELLO deny addition must send the client a route GOODBYE")
19266 .frame;
19267 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19268 assert_eq!(goodbye.header.channel, closed_channel);
19269 assert_ne!(goodbye.header.channel, 0);
19270 assert_ne!(goodbye.header.epoch, 0);
19271 assert!(client_rx.try_recv().is_err());
19272 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19273 assert!(matches!(
19274 target_rx.try_recv(),
19275 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19276 ));
19277 }
19278
19279 #[tokio::test]
19280 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
19281 let registry = Arc::new(Registry::default());
19282 let forwarding = Arc::new(ForwardingTable::default());
19283 let supervisor = SupervisorHandle::new();
19284 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19285 let handler =
19286 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19287 .with_supervisor(supervisor);
19288 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
19289 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
19290 register_capability_manifest(
19291 &handler,
19292 &target_ctx,
19293 &mut target_rx,
19294 capability_manifest("target", &[], &[]),
19295 1,
19296 )
19297 .await;
19298 register_capability_manifest(
19299 &handler,
19300 &opener_ctx,
19301 &mut opener_rx,
19302 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19303 2,
19304 )
19305 .await;
19306 let (mut client_rx, _) = open_route_for_capability_test(
19307 &handler,
19308 &target_ctx,
19309 &mut target_rx,
19310 722,
19311 3,
19312 "target",
19313 Some(ConsumerIdentity {
19314 module_id: "opener".to_string(),
19315 launch_nonce: "opener-nonce".to_string(),
19316 }),
19317 )
19318 .await;
19319 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19320
19321 let replies = handler
19322 .handle_control_frame(
19323 &target_ctx,
19324 catalog_update_with_capabilities_frame(
19325 4,
19326 CapabilityDeclarations {
19327 provides: vec!["credentials-provider/v1".to_string()],
19328 requires: Vec::new(),
19329 must_never_reach: Vec::new(),
19330 },
19331 ),
19332 )
19333 .await
19334 .expect("claim catalog.update succeeds");
19335 assert!(matches!(
19336 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
19337 Ok(ModuleControlResponseToModule::CatalogUpdate {})
19338 ));
19339 let closed_channel = assert_capability_denied_push(
19340 client_rx
19341 .try_recv()
19342 .expect("claim addition must emit route.closed")
19343 .frame,
19344 "target",
19345 );
19346 let goodbye = client_rx
19347 .try_recv()
19348 .expect("claim addition must send the client a route GOODBYE")
19349 .frame;
19350 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19351 assert_eq!(goodbye.header.channel, closed_channel);
19352 assert_ne!(goodbye.header.channel, 0);
19353 assert_ne!(goodbye.header.epoch, 0);
19354 assert!(client_rx.try_recv().is_err());
19355 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19356 assert!(matches!(
19357 target_rx.try_recv(),
19358 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19359 ));
19360 }
19361
19362 #[tokio::test]
19363 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
19364 let registry = Arc::new(Registry::default());
19365 let forwarding = Arc::new(ForwardingTable::default());
19366 let supervisor = SupervisorHandle::new();
19367 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19368 let handler =
19369 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19370 .with_supervisor(supervisor);
19371 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
19372 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
19373 register_capability_manifest(
19374 &handler,
19375 &target_ctx,
19376 &mut target_rx,
19377 capability_manifest("target", &["credentials-provider/v1"], &[]),
19378 1,
19379 )
19380 .await;
19381 register_capability_manifest(
19382 &handler,
19383 &opener_ctx,
19384 &mut opener_rx,
19385 capability_manifest("opener", &[], &[]),
19386 2,
19387 )
19388 .await;
19389 let (mut client_rx, _) = open_route_for_capability_test(
19390 &handler,
19391 &target_ctx,
19392 &mut target_rx,
19393 732,
19394 3,
19395 "target",
19396 Some(ConsumerIdentity {
19397 module_id: "opener".to_string(),
19398 launch_nonce: "opener-nonce".to_string(),
19399 }),
19400 )
19401 .await;
19402 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19403
19404 handler
19405 .handle_control_frame(
19406 &target_ctx,
19407 catalog_update_with_capabilities_frame(
19408 4,
19409 CapabilityDeclarations {
19410 provides: Vec::new(),
19411 requires: Vec::new(),
19412 must_never_reach: Vec::new(),
19413 },
19414 ),
19415 )
19416 .await
19417 .expect("claim removal catalog.update succeeds");
19418 assert_eq!(
19419 forwarding.active_binding_count().unwrap(),
19420 1,
19421 "removing an attested target claim must leave the route census unchanged"
19422 );
19423 assert!(
19424 client_rx.try_recv().is_err(),
19425 "claim removal must not emit route.closed or a client route GOODBYE"
19426 );
19427 assert!(
19428 target_rx.try_recv().is_err(),
19429 "claim removal must not send the target a route GOODBYE"
19430 );
19431 }
19432
19433 #[tokio::test]
19436 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
19437 let registry = Arc::new(Registry::default());
19438 let forwarding = Arc::new(ForwardingTable::default());
19439 let supervisor = SupervisorHandle::new();
19440 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19441 let handler =
19442 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19443 .with_supervisor(supervisor);
19444 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
19445 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
19446 register_capability_manifest(
19447 &handler,
19448 &target_ctx,
19449 &mut target_rx,
19450 capability_manifest("target", &["credentials-provider/v1"], &[]),
19451 1,
19452 )
19453 .await;
19454 register_capability_manifest(
19455 &handler,
19456 &opener_ctx,
19457 &mut opener_rx,
19458 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19459 2,
19460 )
19461 .await;
19462
19463 let (_client_rx, bind) = open_route_for_capability_test(
19464 &handler,
19465 &target_ctx,
19466 &mut target_rx,
19467 742,
19468 3,
19469 "target",
19470 None,
19471 )
19472 .await;
19473 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
19474 panic!("direct scope-honesty route must bind");
19475 };
19476 assert_eq!(principal, Some(Principal::Direct));
19477 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19478 }
19479
19480 #[tokio::test]
19483 async fn must_never_reach_self_route_is_capability_forbidden() {
19484 let registry = Arc::new(Registry::default());
19485 let forwarding = Arc::new(ForwardingTable::default());
19486 let supervisor = SupervisorHandle::new();
19487 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
19488 let handler =
19489 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19490 .with_supervisor(supervisor);
19491 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
19492 register_capability_manifest(
19493 &handler,
19494 &self_ctx,
19495 &mut self_rx,
19496 capability_manifest(
19497 "self-provider",
19498 &["credentials-provider/v1"],
19499 &["credentials-provider/v1"],
19500 ),
19501 1,
19502 )
19503 .await;
19504
19505 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
19506 let replies = handler
19507 .handle_control_frame(
19508 &client_ctx,
19509 route_open_frame_with_admission_facts(
19510 2,
19511 "self-provider",
19512 unique_project_root("admission-facts"),
19513 Some(ConsumerIdentity {
19514 module_id: "self-provider".to_string(),
19515 launch_nonce: "self-nonce".to_string(),
19516 }),
19517 None,
19518 ),
19519 )
19520 .await
19521 .expect("self-route refusal returns a typed frame");
19522 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19523 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19524 assert!(
19525 self_rx.try_recv().is_err(),
19526 "self denial must not relay route.bind"
19527 );
19528 }
19529
19530 #[test]
19531 fn unsupported_channel_zero_frame_returns_error() {
19532 let handler = ControlHandler::default();
19533 let request = Frame::build(
19534 FrameType::Request,
19535 control_flags(),
19536 0,
19537 0,
19538 21,
19539 b"opaque".to_vec(),
19540 )
19541 .unwrap();
19542
19543 let response = handler
19544 .handle_control(ConnectionId::new(1), request)
19545 .unwrap();
19546
19547 assert_eq!(response[0].header.ty, FrameType::Error);
19548 assert_eq!(
19549 parse_error(&response[0])["code"],
19550 "unsupported_control_frame"
19551 );
19552 }
19553
19554 mod swap {
19559 use super::*;
19560
19561 const INCUMBENT: ConnectionId = ConnectionId::new(30);
19562 const CANDIDATE: ConnectionId = ConnectionId::new(40);
19563
19564 struct Swap {
19565 registry: Arc<Registry>,
19566 forwarding: Arc<ForwardingTable>,
19567 handler: ControlHandler,
19568 incumbent_ctx: RouteCtx,
19569 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19570 candidate_ctx: RouteCtx,
19571 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19572 }
19573
19574 async fn swap_with_incumbent() -> Swap {
19575 let registry = Arc::new(Registry::default());
19576 let forwarding = Arc::new(ForwardingTable::default());
19577 let handler =
19578 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
19579 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
19580 hello_via_sink(
19581 &handler,
19582 &incumbent_ctx,
19583 &mut incumbent_rx,
19584 hello_frame("aft", PROTOCOL_VERSION, 7),
19585 )
19586 .await;
19587 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
19588 Swap {
19589 registry,
19590 forwarding,
19591 handler,
19592 incumbent_ctx,
19593 incumbent_rx,
19594 candidate_ctx,
19595 candidate_rx,
19596 }
19597 }
19598
19599 fn register_candidate(swap: &Swap, ready: Option<bool>) {
19600 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
19601 candidate_manifest.ready = ready;
19602 let registration = swap
19603 .registry
19604 .register_candidate_with_control_ops(
19605 candidate_manifest,
19606 PROTOCOL_VERSION,
19607 CANDIDATE,
19608 module_baseline_control_ops(),
19609 )
19610 .unwrap();
19611 swap.forwarding
19612 .register_candidate_module_connection(
19613 CANDIDATE,
19614 "aft".to_string(),
19615 PROTOCOL_VERSION,
19616 manifest_concurrency(®istration.manifest),
19617 swap.candidate_ctx.egress.clone(),
19618 )
19619 .unwrap();
19620 }
19621
19622 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
19623 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
19624 swap.registry.promote_candidate("aft").unwrap().unwrap();
19625 cutover.incumbent.unwrap()
19626 }
19627
19628 #[tokio::test]
19637 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
19638 let mut swap = swap_with_incumbent().await;
19639 let hold = Arc::new(std::sync::Barrier::new(2));
19640 let handler = swap
19641 .handler
19642 .clone()
19643 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
19644 handler
19645 .supervisor
19646 .open_swap("aft", "candidate-nonce".to_string());
19647 let hello = {
19648 let handler = handler.clone();
19649 let candidate_ctx = swap.candidate_ctx.clone();
19650 std::thread::spawn(move || {
19651 handler
19652 .handle_hello(
19653 candidate_ctx.connection_id,
19654 Some(candidate_ctx.egress.clone()),
19655 hello_frame_with_nonce(
19656 "aft",
19657 PROTOCOL_VERSION,
19658 8,
19659 Some("candidate-nonce"),
19660 ),
19661 )
19662 .unwrap()
19663 })
19664 };
19665
19666 hold.wait();
19667 assert!(swap
19670 .registry
19671 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
19672 .unwrap()
19673 .is_some());
19674 assert!(swap
19675 .registry
19676 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19677 .unwrap()
19678 .is_none());
19679 assert!(swap
19680 .forwarding
19681 .module_endpoint_for_connection(CANDIDATE)
19682 .unwrap()
19683 .is_none());
19684
19685 hold.wait();
19686 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
19687 assert_eq!(
19688 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19689 FrameType::HelloAck
19690 );
19691 let published = swap
19692 .registry
19693 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19694 .unwrap()
19695 .expect("candidate is published once its HELLO completes");
19696 assert_eq!(published.connection_id, CANDIDATE);
19697 assert!(swap
19698 .forwarding
19699 .module_endpoint_for_connection(CANDIDATE)
19700 .unwrap()
19701 .is_some());
19702 }
19703
19704 #[tokio::test]
19705 async fn candidate_lookup_after_hello_ack_waits_for_publication() {
19706 let mut swap = swap_with_incumbent().await;
19707 let hold = Arc::new(std::sync::Barrier::new(2));
19708 swap.handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
19709 swap.handler
19710 .supervisor
19711 .open_swap("aft", "candidate-nonce".to_string());
19712 let hello = {
19713 let handler = swap.handler.clone();
19714 let candidate_ctx = swap.candidate_ctx.clone();
19715 std::thread::spawn(move || {
19716 handler
19717 .handle_hello(
19718 CANDIDATE,
19719 Some(candidate_ctx.egress),
19720 hello_frame_with_nonce(
19721 "aft",
19722 PROTOCOL_VERSION,
19723 8,
19724 Some("candidate-nonce"),
19725 ),
19726 )
19727 .unwrap()
19728 })
19729 };
19730 hold.wait();
19731 assert_eq!(
19732 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19733 FrameType::HelloAck
19734 );
19735 let reader_registry = Arc::clone(&swap.registry);
19736 let published = lookup_during_hello_publication(&swap.registry, &hold, move || {
19737 reader_registry
19738 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19739 .unwrap()
19740 });
19741 assert!(hello.join().unwrap().is_empty());
19742 assert_eq!(
19743 published
19744 .expect("ACK makes candidate publication visible")
19745 .connection_id,
19746 CANDIDATE
19747 );
19748 assert_eq!(
19749 swap.registry.generation().unwrap(),
19750 1,
19751 "candidate is not a second catalog entry"
19752 );
19753 }
19754
19755 #[tokio::test]
19756 async fn audit4_ungated_removed_candidate_publication_rolls_back_admission() {
19757 let mut swap = swap_with_incumbent().await;
19758 let hold = Arc::new(std::sync::Barrier::new(2));
19759 let handler = swap
19760 .handler
19761 .clone()
19762 .with_hello_registry_to_forwarding_hold(hold.clone());
19763 handler
19764 .supervisor
19765 .open_swap("aft", "candidate-nonce".into());
19766 let hello = {
19767 let handler = handler.clone();
19768 let ctx = swap.candidate_ctx.clone();
19769 std::thread::spawn(move || {
19770 handler
19771 .handle_hello(
19772 CANDIDATE,
19773 Some(ctx.egress),
19774 hello_frame_with_nonce(
19775 "aft",
19776 PROTOCOL_VERSION,
19777 8,
19778 Some("candidate-nonce"),
19779 ),
19780 )
19781 .unwrap()
19782 })
19783 };
19784 hold.wait();
19785 assert_eq!(
19786 swap.registry
19787 .deregister_connection(CANDIDATE)
19788 .unwrap()
19789 .len(),
19790 1
19791 );
19792 hold.wait();
19793 assert!(hello.join().unwrap().is_empty());
19794 assert_eq!(
19795 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19796 FrameType::HelloAck
19797 );
19798 assert!(swap.registry.get_candidate("aft").unwrap().is_none());
19799 assert_eq!(
19800 handler
19801 .supervisor
19802 .swap_hello_admission("aft", Some("candidate-nonce")),
19803 SwapHelloAdmission::Candidate,
19804 "failed publication must roll back candidate_admitted for an ungated nonce swap"
19805 );
19806 handler.supervisor.close_swap("aft");
19807 }
19808
19809 fn keyed_total(counters: &Value, key: &str) -> u64 {
19810 counters[key]
19811 .as_object()
19812 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
19813 .unwrap_or(0)
19814 }
19815
19816 #[tokio::test]
19817 async fn replacement_logs_old_end_and_new_admission_once() {
19818 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19819 let mut swap = swap_with_incumbent().await;
19820 swap.handler
19821 .supervisor
19822 .open_swap("aft", "candidate-nonce".to_string());
19823 hello_via_sink(
19824 &swap.handler,
19825 &swap.candidate_ctx,
19826 &mut swap.candidate_rx,
19827 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
19828 )
19829 .await;
19830 cutover(&swap);
19831 swap.handler.cleanup_connection(INCUMBENT).unwrap();
19832
19833 let captured = crate::router::test_log::captured_logs(&logs);
19834 println!("captured replacement registry logs:\n{captured}");
19835 let new_admissions: Vec<_> = captured
19836 .lines()
19837 .filter(|line| {
19838 line.contains("connection_id=40")
19839 && line.contains("swap candidate registered; not routable until cutover")
19840 })
19841 .collect();
19842 assert_eq!(
19843 new_admissions.len(),
19844 1,
19845 "unexpected admission logs: {captured}"
19846 );
19847 assert!(new_admissions[0].contains("module_id=aft"));
19848 assert!(new_admissions[0].contains("connection_id=40"));
19849 assert!(
19850 new_admissions[0].contains("swap candidate registered; not routable until cutover")
19851 );
19852 assert!(new_admissions[0].contains("ready=true"));
19853
19854 let old_admissions: Vec<_> = captured
19855 .lines()
19856 .filter(|line| {
19857 line.contains("module registered module_id=aft ")
19858 && line.contains("connection_id=30")
19859 })
19860 .collect();
19861 assert_eq!(
19862 old_admissions.len(),
19863 1,
19864 "unexpected incumbent admission: {captured}"
19865 );
19866
19867 let old_ends: Vec<_> = captured
19868 .lines()
19869 .filter(|line| {
19870 line.contains("connection_id=30") && line.contains("module registration ended")
19871 })
19872 .collect();
19873 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
19874 assert!(old_ends[0].contains("module_id=aft"));
19875 assert!(old_ends[0].contains("connection_id=30"));
19876 assert!(old_ends[0].contains("reason=replaced"));
19877 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
19878 assert_eq!(
19879 captured
19880 .lines()
19881 .filter(|line| line.contains("module registration promoted"))
19882 .count(),
19883 1,
19884 "unexpected promotion logs: {captured}"
19885 );
19886 }
19887
19888 #[tokio::test]
19894 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
19895 let mut swap = swap_with_incumbent().await;
19896 let handler = swap.handler.clone();
19897
19898 let cotenant = ConnectionId::new(31);
19900 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
19901 let (cotenant_task, cotenant_bind) = relay_route_open(
19902 &handler,
19903 cotenant,
19904 &cotenant_ctx.egress,
19905 &mut swap.incumbent_rx,
19906 100,
19907 "aft",
19908 "swap-cotenant",
19909 )
19910 .await;
19911 handler
19912 .handle_control_frame(
19913 &swap.incumbent_ctx,
19914 route_bind_ack(cotenant_bind.header.corr),
19915 )
19916 .await
19917 .unwrap();
19918 assert!(cotenant_task.await.unwrap().is_empty());
19919 let (cotenant_channel, cotenant_epoch) =
19920 published_route(&cotenant_rx.recv().await.unwrap());
19921
19922 let caller = ConnectionId::new(32);
19924 let (caller_ctx, mut caller_rx) = route_ctx(caller);
19925 let (caller_task, caller_bind) = relay_route_open(
19926 &handler,
19927 caller,
19928 &caller_ctx.egress,
19929 &mut swap.incumbent_rx,
19930 101,
19931 "aft",
19932 "swap-caller",
19933 )
19934 .await;
19935 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
19936
19937 register_candidate(&swap, None);
19938 cutover(&swap);
19939
19940 let ack = handler
19942 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
19943 .await;
19944 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
19945 if module_loop_error.is_some() {
19946 handler.cleanup_connection(INCUMBENT).unwrap();
19949 }
19950
19951 assert!(
19953 cotenant_rx.try_recv().is_err(),
19954 "the co-tenant route on the incumbent was torn down by one late ack: \
19955 {module_loop_error:?}"
19956 );
19957 assert!(matches!(
19958 swap.forwarding
19959 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
19960 .unwrap(),
19961 DataRoute::Client(DataRouteState::Bound(_))
19962 ));
19963 assert_eq!(module_loop_error, None);
19964 assert!(swap
19965 .registry
19966 .get_module_by_connection(INCUMBENT)
19967 .unwrap()
19968 .is_some());
19969
19970 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
19972 .await
19973 .expect("the incumbent is told to drop the abandoned binding")
19974 .unwrap()
19975 .frame;
19976 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19977 assert_eq!(goodbye.header.channel, abandoned_channel);
19978 assert_eq!(goodbye.header.epoch, abandoned_epoch);
19979 assert!(swap.incumbent_rx.try_recv().is_err());
19980
19981 let response = caller_task.await.unwrap();
19983 assert_eq!(response.len(), 1);
19984 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
19985 assert!(caller_rx.try_recv().is_err());
19986
19987 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
19991 let counters = handler.counters().snapshot();
19992 assert_eq!(
19993 keyed_total(&counters, "route_open_accepted_by_principal"),
19994 1
19995 );
19996 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
19997 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
19998 }
19999
20000 #[tokio::test]
20004 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
20005 let mut swap = swap_with_incumbent().await;
20006 register_candidate(&swap, None);
20007 let incumbent = cutover(&swap);
20008 swap.forwarding
20009 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
20010 .unwrap()
20011 .expect("the incumbent is still registered");
20012
20013 let client = ConnectionId::new(33);
20014 let (client_ctx, mut client_rx) = route_ctx(client);
20015 let route_handler = swap.handler.clone();
20016 let open_ctx = RouteCtx {
20017 connection_id: client,
20018 egress: client_ctx.egress.clone(),
20019 };
20020 let mut route_task = tokio::spawn(async move {
20021 route_handler
20022 .handle_control_frame(
20023 &open_ctx,
20024 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
20025 )
20026 .await
20027 .unwrap()
20028 });
20029 let bind = tokio::select! {
20030 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
20031 response = &mut route_task => {
20032 let response = response.unwrap();
20033 panic!(
20034 "post-cutover route.open was refused instead of relayed to the candidate: {}",
20035 parse_error(&response[0])["code"]
20036 );
20037 }
20038 };
20039 swap.handler
20040 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
20041 .await
20042 .unwrap();
20043 assert!(route_task.await.unwrap().is_empty());
20044 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
20045 match swap
20046 .forwarding
20047 .lookup_data_route(client, channel, epoch)
20048 .unwrap()
20049 {
20050 DataRoute::Client(DataRouteState::Bound(route)) => {
20051 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
20052 }
20053 other => panic!("expected a bound route on the candidate, got {other:?}"),
20054 }
20055 assert!(swap.incumbent_rx.try_recv().is_err());
20056 }
20057
20058 #[tokio::test]
20063 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
20064 let swap = swap_with_incumbent().await;
20065 register_candidate(&swap, Some(false));
20066 let update = Frame::build(
20067 FrameType::Request,
20068 control_flags(),
20069 0,
20070 0,
20071 55,
20072 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
20073 provides: manifest("aft", PROTOCOL_VERSION).provides,
20074 capabilities: None,
20075 ready: Some(true),
20076 })
20077 .unwrap(),
20078 )
20079 .unwrap();
20080
20081 let replies = swap
20082 .handler
20083 .handle_control_frame(&swap.candidate_ctx, update)
20084 .await
20085 .unwrap();
20086
20087 assert_eq!(replies.len(), 1);
20088 assert_eq!(
20089 replies[0].header.ty,
20090 FrameType::Response,
20091 "candidate catalog.update was refused: {:?}",
20092 serde_json::from_slice::<Value>(&replies[0].body).ok()
20093 );
20094 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
20095 assert_eq!(
20096 swap.registry
20097 .get_module("aft")
20098 .unwrap()
20099 .unwrap()
20100 .connection_id,
20101 INCUMBENT
20102 );
20103 }
20104 }
20105
20106 mod swap_admission {
20110 use super::*;
20111
20112 const INCUMBENT_NONCE: &str = "incumbent-nonce";
20113 const CANDIDATE_NONCE: &str = "candidate-nonce";
20114
20115 fn handler_with_incumbent(
20116 module_id: &str,
20117 reserved: bool,
20118 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
20119 let registry = Arc::new(Registry::default());
20120 let supervisor = SupervisorHandle::new();
20121 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
20122 if reserved {
20123 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
20124 }
20125 let handler =
20126 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
20127 let incumbent = handler
20128 .handle_control(
20129 ConnectionId::new(1),
20130 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
20131 )
20132 .unwrap();
20133 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
20134 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
20135 (registry, supervisor, handler)
20136 }
20137
20138 #[test]
20144 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
20145 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
20146
20147 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
20148 let replies = handler
20149 .handle_control(
20150 ConnectionId::new(connection),
20151 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
20152 )
20153 .unwrap();
20154 assert_eq!(replies[0].header.ty, FrameType::Error);
20155 assert_eq!(
20156 parse_error(&replies[0])["code"],
20157 "swap_token_invalid",
20158 "nonce {nonce:?}"
20159 );
20160 }
20161 assert!(registry.get_candidate("aft").unwrap().is_none());
20162 assert_eq!(
20163 registry.get_module("aft").unwrap().unwrap().connection_id,
20164 ConnectionId::new(1)
20165 );
20166
20167 let admitted = handler
20170 .handle_control(
20171 ConnectionId::new(4),
20172 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
20173 )
20174 .unwrap();
20175 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
20176 assert_eq!(
20177 registry
20178 .get_candidate("aft")
20179 .unwrap()
20180 .unwrap()
20181 .connection_id,
20182 ConnectionId::new(4)
20183 );
20184 assert_eq!(
20185 registry.get_module("aft").unwrap().unwrap().connection_id,
20186 ConnectionId::new(1),
20187 "the candidate must not take the active slot"
20188 );
20189 let replayed = handler
20190 .handle_control(
20191 ConnectionId::new(5),
20192 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
20193 )
20194 .unwrap();
20195 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
20196
20197 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
20201 let squatter = handler
20202 .handle_control(
20203 ConnectionId::new(6),
20204 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
20205 )
20206 .unwrap();
20207 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
20208 assert!(
20209 registry.get_module("aft").unwrap().is_none(),
20210 "a squatter took the active slot of an id being swapped"
20211 );
20212 }
20213
20214 #[test]
20219 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
20220 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
20221
20222 let replies = handler
20223 .handle_control(
20224 ConnectionId::new(2),
20225 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20226 )
20227 .unwrap();
20228
20229 assert_eq!(
20230 replies[0].header.ty,
20231 FrameType::HelloAck,
20232 "reserved candidate refused: {:?}",
20233 serde_json::from_slice::<Value>(&replies[0].body).ok()
20234 );
20235 assert_eq!(
20236 registry
20237 .get_candidate("vault")
20238 .unwrap()
20239 .unwrap()
20240 .connection_id,
20241 ConnectionId::new(2)
20242 );
20243 }
20244
20245 #[test]
20248 fn without_an_open_swap_the_ordinary_gates_decide() {
20249 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
20250 supervisor.close_swap("vault");
20251
20252 let candidate = handler
20253 .handle_control(
20254 ConnectionId::new(2),
20255 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20256 )
20257 .unwrap();
20258 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
20259 let duplicate = handler
20260 .handle_control(
20261 ConnectionId::new(3),
20262 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
20263 )
20264 .unwrap();
20265 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
20266 assert!(registry.get_candidate("vault").unwrap().is_none());
20267 }
20268 }
20269
20270 mod scopes {
20274 mod close_reasons {
20275 include!("route_close_reason_tests.rs");
20276 }
20277 mod idle_tests {
20278 use super::*;
20279 include!("scope_idle_tests.rs");
20280 }
20281 mod route_open_timings {
20282 use super::*;
20283 include!("route_open_timing_tests.rs");
20284 }
20285 mod route_open_refusal_log {
20286 use super::*;
20287 include!("route_open_refusal_log_tests.rs");
20288 }
20289 mod activity {
20290 include!("activity_scope_tests.rs");
20291 }
20292 mod delta {
20293 include!("scope_handler_delta_tests.rs");
20294 }
20295 mod call_modes {
20296 include!("scope_call_mode_tests.rs");
20297 }
20298 mod federated {
20299 include!("scope_federated_tests.rs");
20300 }
20301 mod authorize {
20302 include!("route_authorize_tests.rs");
20303 }
20304 #[tokio::test]
20305 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
20306 let rig = rig().await;
20307 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
20308 "s".into(),
20309 1,
20310 "tool".into(),
20311 ))
20312 .unwrap();
20313 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
20314 let (direct, _rx) = wide_ctx(999);
20315 for (ctx, expected) in [
20316 (&rig.owner, FrameType::Response),
20317 (&direct, FrameType::Error),
20318 ] {
20319 let frame =
20320 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
20321 .unwrap();
20322 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
20323 assert_eq!(replies[0].header.ty, expected);
20324 if expected == FrameType::Response {
20325 let reply: subc_control::ScopeServedByReply =
20326 serde_json::from_slice(&replies[0].body).unwrap();
20327 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
20328 } else {
20329 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
20330 }
20331 }
20332 let reply = call(
20333 &rig.handler,
20334 &rig.owner,
20335 &ModuleControlRequestFromModule::ScopeDescribe {
20336 owner: Principal::Reserved {
20337 module_id: OWNER.into(),
20338 },
20339 scope_ref: "s".into(),
20340 },
20341 )
20342 .await;
20343 assert!(matches!(
20344 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
20345 ModuleControlResponseToModule::ScopeDescribe { .. }
20346 ));
20347 }
20348
20349 use subc_protocol::scope::{
20350 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
20351 ScopeStatus,
20352 };
20353
20354 use super::*;
20355
20356 const OWNER: &str = "prefrontal-core";
20357
20358 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
20359 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
20360 }
20361
20362 async fn call(
20363 handler: &ControlHandler,
20364 ctx: &RouteCtx,
20365 request: &ModuleControlRequestFromModule,
20366 ) -> Frame {
20367 let body = serde_json::to_vec(request).unwrap();
20368 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
20369 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
20370 assert_eq!(replies.len(), 1, "{replies:?}");
20371 replies.pop().unwrap()
20372 }
20373
20374 async fn sync(
20375 handler: &ControlHandler,
20376 ctx: &RouteCtx,
20377 generation: u64,
20378 scopes: Vec<ScopeRecord>,
20379 ) -> Result<ModuleControlResponseToModule, String> {
20380 let reply = call(
20381 handler,
20382 ctx,
20383 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
20384 )
20385 .await;
20386 match reply.header.ty {
20387 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
20388 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
20389 }
20390 }
20391
20392 async fn describe(
20393 handler: &ControlHandler,
20394 ctx: &RouteCtx,
20395 owner: &str,
20396 scope_ref: &str,
20397 ) -> ModuleControlResponseToModule {
20398 let reply = call(
20399 handler,
20400 ctx,
20401 &ModuleControlRequestFromModule::ScopeDescribe {
20402 owner: Principal::Reserved {
20403 module_id: owner.to_string(),
20404 },
20405 scope_ref: scope_ref.to_string(),
20406 },
20407 )
20408 .await;
20409 assert_eq!(
20410 reply.header.ty,
20411 FrameType::Response,
20412 "{:?}",
20413 parse_error(&reply)
20414 );
20415 serde_json::from_slice(&reply.body).unwrap()
20416 }
20417
20418 #[tokio::test]
20419 async fn scope_describe_never_echoes_bind_proof_or_hash() {
20420 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
20421 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
20422 let mut rig = rig().await;
20423 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
20424 .await;
20425 let _bound = rig
20426 .bound(
20427 Some(AFT),
20428 PLEXUS,
20429 Some(
20430 rig.selector("s", Some(1))
20431 .with_bind_proof(Some(PROOF.into())),
20432 ),
20433 )
20434 .await;
20435 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
20436 let output = serde_json::to_string(&reply).unwrap();
20437 assert!(output.contains("\"status\":\"live\""));
20438 assert!(!output.contains(PROOF));
20439 assert!(!output.contains(HASH));
20440 }
20441
20442 async fn module(
20444 handler: &ControlHandler,
20445 connection: u64,
20446 module_id: &str,
20447 nonce: Option<&str>,
20448 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20449 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
20450 hello_via_sink(
20451 handler,
20452 &ctx,
20453 &mut rx,
20454 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
20455 )
20456 .await;
20457 (ctx, rx)
20458 }
20459
20460 #[tokio::test]
20463 async fn a_client_connection_cannot_sync_or_describe() {
20464 let handler = ControlHandler::new(Arc::new(Registry::default()));
20465 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
20466 for request in [
20467 ModuleControlRequestFromModule::ScopeSync {
20468 generation: 1,
20469 scopes: vec![head("s", 1)],
20470 },
20471 ModuleControlRequestFromModule::ScopeDescribe {
20472 owner: Principal::Direct,
20473 scope_ref: "s".to_string(),
20474 },
20475 ] {
20476 let reply = call(&handler, &ctx, &request).await;
20477 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
20478 }
20479 assert!(
20480 !handler
20481 .scopes
20482 .read()
20483 .unwrap()
20484 .describe(
20485 &Principal::Reserved {
20486 module_id: OWNER.to_string()
20487 },
20488 "s"
20489 )
20490 .owner_synced
20491 );
20492 }
20493
20494 #[tokio::test]
20497 async fn a_module_without_a_supervised_launch_cannot_sync() {
20498 let handler = ControlHandler::new(Arc::new(Registry::default()));
20499 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
20500 assert_eq!(
20501 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
20502 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20503 );
20504 }
20505
20506 #[tokio::test]
20507 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
20508 let supervisor = SupervisorHandle::new();
20509 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20510 let handler = ControlHandler::new(Arc::new(Registry::default()))
20511 .with_supervisor(supervisor.clone());
20512 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20513 sync(&handler, &incumbent, 1, vec![head("s", 1)])
20514 .await
20515 .expect("the current launch syncs");
20516
20517 supervisor.open_swap(OWNER, "n2".to_string());
20520 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
20521 assert_eq!(
20522 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20523 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20524 );
20525 sync(&handler, &incumbent, 2, vec![head("s", 1)])
20526 .await
20527 .expect("the serving owner syncs during the swap");
20528
20529 supervisor.close_swap(OWNER);
20532 assert_eq!(
20533 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20534 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20535 );
20536 sync(&handler, &incumbent, 3, vec![head("s", 1)])
20537 .await
20538 .expect("the serving owner syncs after the rollback");
20539 handler.cleanup_connection(candidate.connection_id).unwrap();
20540
20541 supervisor.open_swap(OWNER, "n3".to_string());
20546 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
20547 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
20548 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
20549 .await
20550 .expect("the promoted launch takes authority");
20551 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20552 panic!("unexpected reply {reply:?}");
20553 };
20554 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
20555 assert_eq!(
20556 sync(&handler, &incumbent, 4, Vec::new()).await,
20557 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20558 );
20559 }
20560
20561 #[tokio::test]
20564 async fn closing_the_authority_connection_frees_sync_authority() {
20565 let supervisor = SupervisorHandle::new();
20566 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20567 let handler = ControlHandler::new(Arc::new(Registry::default()))
20568 .with_supervisor(supervisor.clone());
20569 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20570 sync(&handler, &first, 10, vec![head("s", 1)])
20571 .await
20572 .unwrap();
20573 handler.cleanup_connection(first.connection_id).unwrap();
20574
20575 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
20576 sync(&handler, &second, 1, vec![head("s", 1)])
20577 .await
20578 .expect("the next connection takes the released authority");
20579 }
20580
20581 #[tokio::test]
20582 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
20583 let supervisor = SupervisorHandle::new();
20584 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20585 let handler =
20586 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
20587 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
20588 sync(&handler, &first, 10, vec![head("s", 1)])
20589 .await
20590 .unwrap();
20591 handler
20592 .handle_control_frame(
20593 &first,
20594 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
20595 )
20596 .await
20597 .unwrap();
20598 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
20599 sync(&handler, &second, 1, vec![head("s", 1)])
20600 .await
20601 .expect("GOODBYE releases authority even if the old socket remains open");
20602 }
20603
20604 #[tokio::test]
20605 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
20606 let registry = Arc::new(Registry::default());
20607 let supervisor_handle = SupervisorHandle::new();
20608 let supervisor =
20609 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20610 .with_handle(supervisor_handle.clone())
20611 .with_daemon_incarnation("incarnation-7".to_string());
20612 supervisor
20615 .supervise_configured(
20616 ModuleSpec {
20617 script: None,
20618 carry_containment: false,
20619 module_id: OWNER.to_string(),
20620 program: PathBuf::from("/nonexistent/prefrontal-core"),
20621 args: Vec::new(),
20622 env: Vec::new(),
20623 reserved: false,
20624 reserved_prefixes: Vec::new(),
20625 protocol: ModuleProtocol::Subc,
20626 overlap: Default::default(),
20627 },
20628 false,
20629 )
20630 .unwrap();
20631 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
20632 let handler =
20633 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
20634 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
20635
20636 let ModuleControlResponseToModule::ScopeDescribe {
20638 status,
20639 daemon_incarnation,
20640 owner_synced,
20641 owner_configured,
20642 scope,
20643 ..
20644 } = describe(&handler, &reader, OWNER, "s").await
20645 else {
20646 panic!("not a describe reply");
20647 };
20648 assert_eq!(status, ScopeStatus::NotLive);
20649 assert_eq!(daemon_incarnation, "incarnation-7");
20650 assert!(!owner_synced);
20651 assert!(owner_configured);
20652 assert!(scope.is_none());
20653
20654 let ModuleControlResponseToModule::ScopeDescribe {
20657 status,
20658 owner_configured,
20659 ..
20660 } = describe(&handler, &reader, "ghost", "s").await
20661 else {
20662 panic!("not a describe reply");
20663 };
20664 assert_eq!(status, ScopeStatus::NotLive);
20665 assert!(!owner_configured);
20666
20667 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
20669 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
20670 let ModuleControlResponseToModule::ScopeDescribe {
20671 status,
20672 scope_epoch,
20673 owner_synced,
20674 scope,
20675 ..
20676 } = describe(&handler, &reader, OWNER, "s").await
20677 else {
20678 panic!("not a describe reply");
20679 };
20680 assert_eq!(status, ScopeStatus::Live);
20681 assert_eq!(scope_epoch, Some(4));
20682 assert!(owner_synced);
20683 let stamp = scope.expect("a live scope carries its stamp");
20684 assert!(
20685 stamp.owner_authorized,
20686 "prefrontal-core is the default authority"
20687 );
20688 assert_eq!(stamp.kind, ScopeKind::Head);
20689 }
20690
20691 #[tokio::test]
20692 async fn scope_authority_owners_decides_owner_authorized() {
20693 let supervisor = SupervisorHandle::new();
20694 supervisor.set_spawn_nonce("broca", "b1".to_string());
20695 let handler = ControlHandler::new(Arc::new(Registry::default()))
20696 .with_supervisor(supervisor)
20697 .with_scope_authority_owners(vec!["broca".to_string()]);
20698 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
20699 let mut gated = head("s", 1);
20700 gated.attributes.agent_id = Some("agent".to_string());
20701 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
20702 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
20703 describe(&handler, &broca, "broca", "s").await
20704 else {
20705 panic!("not a describe reply");
20706 };
20707 assert!(scope.unwrap().owner_authorized);
20708 }
20709
20710 #[tokio::test]
20714 async fn scope_ops_and_the_scopes_capability_are_advertised() {
20715 let handler = ControlHandler::new(Arc::new(Registry::default()));
20716 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
20717 let ack = hello_via_sink(
20718 &handler,
20719 &ctx,
20720 &mut rx,
20721 hello_frame("m", PROTOCOL_VERSION, 1),
20722 )
20723 .await;
20724 let ack = parse_ack(&ack);
20725 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
20726 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
20727 }
20728 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
20729
20730 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
20731 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
20732 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
20733 let reply = handler
20734 .handle_control_frame(&client, frame)
20735 .await
20736 .unwrap()
20737 .pop()
20738 .unwrap();
20739 let ClientControlResponse::ServerDescribe { capabilities, .. } =
20740 serde_json::from_slice(&reply.body).unwrap()
20741 else {
20742 panic!("not a server.describe reply");
20743 };
20744 assert!(
20745 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
20746 "{capabilities:?}"
20747 );
20748 }
20749
20750 const PLEXUS: &str = "plexus";
20753 const OTHER: &str = "other";
20754 const AFT: &str = "aft";
20755 const BROCA: &str = "broca";
20756 const MAGIC: &str = "magic-context";
20757
20758 fn nonce(module_id: &str) -> String {
20759 format!("nonce-{module_id}")
20760 }
20761
20762 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20763 let (tx, rx) = mpsc::channel(64);
20764 (
20765 RouteCtx {
20766 connection_id: ConnectionId::new(connection),
20767 egress: FrameSink::new(tx),
20768 },
20769 rx,
20770 )
20771 }
20772
20773 struct Rig {
20777 handler: ControlHandler,
20778 forwarding: Arc<ForwardingTable>,
20779 owner: RouteCtx,
20780 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20781 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
20782 generation: u64,
20783 next_connection: u64,
20784 _supervisor: Supervisor,
20785 }
20786
20787 async fn rig() -> Rig {
20788 rig_with_flow_support(true).await
20789 }
20790
20791 async fn rig_with_flow_support(flow_support: bool) -> Rig {
20792 rig_with_scope_capabilities(flow_support, false).await
20793 }
20794
20795 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
20796 let mut provides = Vec::new();
20797 if flow_support {
20798 provides.push("flow-scopes/v1");
20799 }
20800 if run_support {
20801 provides.push("agent-run-scopes/v1");
20802 }
20803 rig_with_target_capabilities(&provides).await
20804 }
20805
20806 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
20807 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
20808 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20809 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20812 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
20813 Frame::build(
20814 FrameType::Hello,
20815 control_flags(),
20816 0,
20817 0,
20818 connection,
20819 serde_json::to_vec(&body).unwrap(),
20820 )
20821 .unwrap()
20822 }
20823
20824 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
20825 let registry = Arc::new(Registry::default());
20826 let forwarding = Arc::new(ForwardingTable::default());
20827 let supervisor_handle = SupervisorHandle::new();
20828 let supervisor =
20829 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20830 .with_handle(supervisor_handle.clone());
20831 supervisor
20832 .supervise_configured(
20833 ModuleSpec {
20834 script: None,
20835 carry_containment: false,
20836 module_id: OWNER.to_string(),
20837 program: PathBuf::from("/nonexistent/prefrontal-core"),
20838 args: Vec::new(),
20839 env: Vec::new(),
20840 reserved: false,
20841 reserved_prefixes: Vec::new(),
20842 protocol: ModuleProtocol::Subc,
20843 overlap: Default::default(),
20844 },
20845 false,
20846 )
20847 .unwrap();
20848 for module_id in [OWNER, AFT, BROCA, MAGIC] {
20849 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
20850 }
20851 let handler =
20852 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
20853 .with_supervisor(supervisor_handle);
20854 let (owner, mut owner_rx) = wide_ctx(1);
20855 hello_via_sink(
20856 &handler,
20857 &owner,
20858 &mut owner_rx,
20859 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
20860 )
20861 .await;
20862 let mut modules = BTreeMap::new();
20863 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
20864 let (ctx, mut rx) = wide_ctx(connection);
20865 let hello = hello_with_capabilities(module_id, connection, provides);
20866 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
20867 modules.insert(module_id.to_string(), (ctx, rx));
20868 }
20869 Rig {
20870 handler,
20871 forwarding,
20872 owner,
20873 _owner_rx: owner_rx,
20874 modules,
20875 generation: 0,
20876 next_connection: 100,
20877 _supervisor: supervisor,
20878 }
20879 }
20880
20881 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
20882 ScopeCarrier::new(Principal::Reserved {
20883 module_id: module_id.to_string(),
20884 })
20885 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
20886 }
20887
20888 fn session(scope_epoch: u64) -> ScopeRecord {
20891 let mut record = head("s", scope_epoch);
20892 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
20893 record.attributes.agent_id = Some("agent-1".to_string());
20894 record.attributes.delegates = true;
20895 record
20896 }
20897
20898 impl Rig {
20899 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
20900 self.generation += 1;
20901 sync(&self.handler, &self.owner, self.generation, scopes)
20902 .await
20903 .expect("the owner's sync is accepted");
20904 }
20905
20906 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
20907 ScopeSelector::new(
20908 Principal::Reserved {
20909 module_id: OWNER.to_string(),
20910 },
20911 scope_ref.to_string(),
20912 scope_epoch,
20913 )
20914 }
20915
20916 fn open_frame(
20917 &mut self,
20918 opener: Option<&str>,
20919 target: &str,
20920 scope: Option<ScopeSelector>,
20921 ) -> (
20922 RouteCtx,
20923 mpsc::Receiver<crate::router::OutboundFrame>,
20924 Frame,
20925 ) {
20926 self.next_connection += 1;
20927 let (ctx, rx) = wide_ctx(self.next_connection);
20928 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
20929 busy_refusals: None,
20930 target: RouteTarget::ToolProvider {
20931 module_id: target.to_string(),
20932 },
20933 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
20934 consumer_identity: opener.map(|module_id| ConsumerIdentity {
20935 module_id: module_id.to_string(),
20936 launch_nonce: nonce(module_id),
20937 }),
20938 consumer_capabilities: None,
20939 role_versions: None,
20940 admission_facts: None,
20941 scope,
20942 })
20943 .unwrap();
20944 let frame = Frame::build(
20945 FrameType::Request,
20946 control_flags(),
20947 0,
20948 0,
20949 self.next_connection,
20950 body,
20951 )
20952 .unwrap();
20953 (ctx, rx, frame)
20954 }
20955
20956 async fn refused(
20958 &mut self,
20959 opener: Option<&str>,
20960 target: &str,
20961 scope: Option<ScopeSelector>,
20962 ) -> String {
20963 self.refusal_body(opener, target, scope).await["code"]
20964 .as_str()
20965 .unwrap()
20966 .to_string()
20967 }
20968
20969 async fn refusal_body(
20970 &mut self,
20971 opener: Option<&str>,
20972 target: &str,
20973 scope: Option<ScopeSelector>,
20974 ) -> Value {
20975 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
20976 let replies = tokio::time::timeout(
20977 Duration::from_secs(2),
20978 self.handler.handle_control_frame(&ctx, frame),
20979 )
20980 .await
20981 .expect("the open must be refused before waiting for a bind ack")
20982 .unwrap();
20983 assert_eq!(replies.len(), 1, "{replies:?}");
20984 assert_eq!(replies[0].header.ty, FrameType::Error);
20985 let (_, module_rx) = self.modules.get_mut(target).unwrap();
20986 assert!(
20987 module_rx.try_recv().is_err(),
20988 "a refused open relays nothing"
20989 );
20990 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
20991 parse_error(&replies[0])
20992 }
20993
20994 async fn relayed(
20996 &mut self,
20997 opener: Option<&str>,
20998 target: &str,
20999 scope: Option<ScopeSelector>,
21000 ) -> Relayed {
21001 self.relayed_with_legacy_root(opener, target, scope, None)
21002 .await
21003 }
21004
21005 async fn relayed_with_legacy_root(
21006 &mut self,
21007 opener: Option<&str>,
21008 target: &str,
21009 scope: Option<ScopeSelector>,
21010 legacy_root: Option<&std::path::Path>,
21011 ) -> Relayed {
21012 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
21013 if let Some(root) = legacy_root {
21014 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
21015 body["identity"]["project_root"] = serde_json::json!(root);
21016 frame.body = serde_json::to_vec(&body).unwrap();
21017 }
21018 let handler = self.handler.clone();
21019 let task_ctx = ctx.clone();
21020 let task = tokio::spawn(async move {
21021 handler
21022 .handle_control_frame(&task_ctx, frame)
21023 .await
21024 .unwrap()
21025 });
21026 let (_, module_rx) = self.modules.get_mut(target).unwrap();
21027 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
21028 .await
21029 .expect("the target receives the relayed route.bind")
21030 .unwrap()
21031 .frame;
21032 Relayed {
21033 target: target.to_string(),
21034 client: ctx,
21035 client_rx: rx,
21036 task,
21037 bind,
21038 }
21039 }
21040
21041 async fn ack(&self, relayed: &Relayed) {
21042 let (module, _) = &self.modules[&relayed.target];
21043 self.handler
21044 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
21045 .await
21046 .unwrap();
21047 }
21048
21049 async fn bound(
21051 &mut self,
21052 opener: Option<&str>,
21053 target: &str,
21054 scope: Option<ScopeSelector>,
21055 ) -> Bound {
21056 let relayed = self.relayed(opener, target, scope).await;
21057 self.commit_relayed(relayed).await
21058 }
21059
21060 async fn bound_with_legacy_root(
21061 &mut self,
21062 opener: Option<&str>,
21063 target: &str,
21064 scope: Option<ScopeSelector>,
21065 root: &std::path::Path,
21066 ) -> Bound {
21067 let relayed = self
21068 .relayed_with_legacy_root(opener, target, scope, Some(root))
21069 .await;
21070 self.commit_relayed(relayed).await
21071 }
21072
21073 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
21074 self.ack(&relayed).await;
21075 let Relayed {
21076 target,
21077 client,
21078 mut client_rx,
21079 task,
21080 bind,
21081 } = relayed;
21082 assert!(
21083 task.await.unwrap().is_empty(),
21084 "the open is answered by commit"
21085 );
21086 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
21087 Bound {
21088 target,
21089 client,
21090 client_rx,
21091 channel,
21092 epoch,
21093 bind,
21094 }
21095 }
21096
21097 fn live(&self, route: &Bound) -> bool {
21098 matches!(
21099 self.forwarding
21100 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21101 .unwrap(),
21102 DataRoute::Client(DataRouteState::Bound(_))
21103 )
21104 }
21105 }
21106
21107 struct Relayed {
21108 target: String,
21109 client: RouteCtx,
21110 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21111 task: tokio::task::JoinHandle<Vec<Frame>>,
21112 bind: Frame,
21113 }
21114
21115 struct Bound {
21116 target: String,
21117 client: RouteCtx,
21118 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21119 channel: u16,
21120 epoch: u32,
21121 bind: Frame,
21122 }
21123
21124 impl Bound {
21125 fn closed_reason(&mut self) -> RouteCloseReason {
21128 let mut reason = None;
21129 let mut goodbye = false;
21130 while let Ok(outbound) = self.client_rx.try_recv() {
21131 let frame = outbound.frame;
21132 match frame.header.ty {
21133 FrameType::Goodbye => {
21134 assert_eq!(
21135 (frame.header.channel, frame.header.epoch),
21136 (self.channel, self.epoch)
21137 );
21138 goodbye = true;
21139 }
21140 FrameType::Push => {
21141 let ClientControlPush::RouteClosed {
21142 reason: r,
21143 module_id,
21144 ..
21145 } = serde_json::from_slice(&frame.body).unwrap()
21146 else {
21147 panic!("unexpected push");
21148 };
21149 assert_eq!(module_id, self.target);
21150 reason = Some(r);
21151 }
21152 other => panic!("unexpected frame {other:?}"),
21153 }
21154 }
21155 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
21156 reason.expect("the client is told why the route closed")
21157 }
21158
21159 fn untouched(&mut self) -> bool {
21160 self.client_rx.try_recv().is_err()
21161 }
21162
21163 fn stamp(&self) -> Option<ScopeStamp> {
21164 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
21165 ModuleControlRequest::RouteBind { scope, .. } => scope,
21166 other => panic!("expected a route.bind, got {other:?}"),
21167 }
21168 }
21169 }
21170
21171 #[tokio::test]
21172 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
21173 ) {
21174 let mut rig = rig().await;
21175 let mut record = session(1);
21176 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
21177 record.child_owners = vec![Principal::Reserved {
21178 module_id: MAGIC.to_string(),
21179 }];
21180 rig.sync(vec![record]).await;
21181 let scope = || Some(rig_selector("s", Some(1)));
21182
21183 rig.bound(Some(OWNER), PLEXUS, scope()).await;
21186 rig.bound(Some(AFT), OTHER, scope()).await;
21187 rig.bound(Some(BROCA), PLEXUS, scope()).await;
21188
21189 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
21193 assert_eq!(
21194 rig.refused(opener, target, scope()).await,
21195 error_codes::SCOPE_NOT_CARRIER,
21196 "{opener:?} -> {target}"
21197 );
21198 }
21199 }
21200
21201 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
21202 ScopeSelector::new(
21203 Principal::Reserved {
21204 module_id: OWNER.to_string(),
21205 },
21206 scope_ref.to_string(),
21207 scope_epoch,
21208 )
21209 }
21210
21211 #[tokio::test]
21212 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
21213 let mut rig = rig().await;
21214 rig.sync(vec![session(1)]).await;
21215 for opener in [OWNER, AFT] {
21216 assert_eq!(
21217 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
21218 .await,
21219 error_codes::SCOPE_EPOCH_REQUIRED,
21220 "{opener}"
21221 );
21222 }
21223 }
21224
21225 #[tokio::test]
21226 async fn admission_separates_not_synced_not_live_and_ended() {
21227 let mut rig = rig().await;
21228 let code = rig
21230 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21231 .await;
21232 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
21233 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
21234
21235 let ghost = ScopeSelector::new(
21237 Principal::Reserved {
21238 module_id: "ghost".to_string(),
21239 },
21240 "s".to_string(),
21241 Some(1),
21242 );
21243 assert_eq!(
21244 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
21245 error_codes::SCOPE_NOT_LIVE
21246 );
21247
21248 rig.sync(vec![session(2)]).await;
21249 assert_eq!(
21250 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
21251 .await,
21252 error_codes::SCOPE_NOT_LIVE
21253 );
21254 for epoch in [1, 3] {
21255 assert_eq!(
21256 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
21257 .await,
21258 error_codes::SCOPE_ENDED,
21259 "epoch {epoch}"
21260 );
21261 }
21262 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
21264 .await;
21265 }
21266
21267 #[tokio::test]
21268 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
21269 let mut rig = rig().await;
21270 rig.sync(vec![session(1)]).await;
21271 let route = rig
21272 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21273 .await;
21274 let stamp = route.stamp().expect("a scoped bind carries the stamp");
21275 assert_eq!(stamp.scope_ref, "s");
21276 assert_eq!(stamp.scope_epoch, 1);
21277 assert_eq!(stamp.kind, ScopeKind::Head);
21278 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
21279 assert!(stamp.attributes.delegates);
21280 assert!(stamp.owner_authorized);
21281 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21282 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
21283
21284 let (broca, mut broca_rx) = wide_ctx(50);
21287 hello_via_sink(
21288 &rig.handler,
21289 &broca,
21290 &mut broca_rx,
21291 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
21292 )
21293 .await;
21294 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
21295 .await
21296 .unwrap();
21297 let own = ScopeSelector::new(
21298 Principal::Reserved {
21299 module_id: BROCA.to_string(),
21300 },
21301 "b".to_string(),
21302 Some(1),
21303 );
21304 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
21305 assert!(!route.stamp().unwrap().owner_authorized);
21306 }
21307
21308 #[tokio::test]
21309 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
21310 let mut rig = rig().await;
21311 let mut record = head("s", 1);
21312 record.carriers = vec![carrier(AFT, None)];
21313 let flow_id = "Flow:run-7/step_2!~";
21314 record.attributes.flow_id = Some(flow_id.to_string());
21315 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
21316 .await
21317 .unwrap();
21318 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
21319 panic!("not a sync reply");
21320 };
21321 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
21322 let route = rig
21323 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21324 .await;
21325 assert!(rig.live(&route), "the stamped bind committed");
21326 let stamp = route.stamp().expect("a flow scope carries a stamp");
21327 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
21328 assert_eq!(stamp.attributes.agent_id, None);
21329 assert!(!stamp.attributes.delegates);
21330 assert!(stamp.owner_authorized);
21331 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21332 assert_eq!(unscoped.stamp(), None);
21333 }
21334
21335 #[tokio::test]
21336 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
21337 let mut rig = rig_with_flow_support(false).await;
21338 let mut record = session(1);
21339 record.attributes.flow_id = Some("flow:7".to_string());
21340 rig.sync(vec![record]).await;
21341 for opener in [OWNER, AFT] {
21342 let body = rig
21343 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21344 .await;
21345 assert_eq!(body["code"], "target_flow_unsupported");
21346 let message = body["message"].as_str().unwrap();
21347 for required in [PLEXUS, "flow-scopes/v1"] {
21348 assert!(message.contains(required), "{message}");
21349 }
21350 }
21351 }
21352
21353 #[tokio::test]
21354 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
21355 let mut rig = rig_with_flow_support(true).await;
21356 let mut record = session(1);
21357 record.attributes.flow_id = Some("flow:7".to_string());
21358 rig.sync(vec![record]).await;
21359 for opener in [OWNER, AFT] {
21360 let route = rig
21361 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21362 .await;
21363 assert!(rig.live(&route));
21364 assert_eq!(
21365 route.stamp().unwrap().attributes.flow_id.as_deref(),
21366 Some("flow:7")
21367 );
21368 }
21369 }
21370
21371 #[tokio::test]
21372 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
21373 use std::future::Future;
21374
21375 let mut rig = rig_with_flow_support(true).await;
21376 let mut record = session(1);
21377 record.attributes.flow_id = Some("flow:7".to_string());
21378 rig.sync(vec![record]).await;
21379 let (client, mut client_rx, frame) =
21380 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
21381 for _ in 0..64 {
21384 client.egress.try_send(route_bind_ack(1)).unwrap();
21385 }
21386 let handler = rig.handler.clone();
21387 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21388 std::future::poll_fn(|cx| {
21389 assert!(open.as_mut().poll(cx).is_pending());
21390 std::task::Poll::Ready(())
21391 })
21392 .await;
21393 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21394
21395 let old_connection = rig.modules[PLEXUS].0.connection_id;
21396 rig.handler.cleanup_connection(old_connection).unwrap();
21397 let (replacement, mut replacement_rx) = wide_ctx(200);
21398 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
21399 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
21400 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
21401 let hello = Frame::build(
21402 FrameType::Hello,
21403 control_flags(),
21404 0,
21405 0,
21406 200,
21407 serde_json::to_vec(&body).unwrap(),
21408 )
21409 .unwrap();
21410 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
21411
21412 client_rx.try_recv().unwrap();
21413 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21414 .await
21415 .expect("the replacement is refused without waiting for a bind ack")
21416 .unwrap();
21417 assert_eq!(replies.len(), 1);
21418 let body = parse_error(&replies[0]);
21419 assert_eq!(body["code"], "target_flow_unsupported");
21420 for required in [PLEXUS, "flow-scopes/v1"] {
21421 assert!(body["message"].as_str().unwrap().contains(required));
21422 }
21423 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
21424 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21425 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21426 assert!(rig
21427 .handler
21428 .registry
21429 .get_module_by_connection(replacement.connection_id)
21430 .unwrap()
21431 .is_some());
21432 }
21433
21434 #[tokio::test]
21435 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
21436 use std::future::Future;
21437 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
21438 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21439 const NEXT_HASH: &str =
21440 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21441 let mut rig = rig().await;
21442 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
21443 record.carriers.push(ScopeCarrier::new(Principal::Direct));
21444 rig.sync(vec![record.clone()]).await;
21445 let (client, mut rx, frame) = rig.open_frame(
21446 None,
21447 PLEXUS,
21448 Some(
21449 rig.selector("s", Some(1))
21450 .with_bind_proof(Some(PROOF.into())),
21451 ),
21452 );
21453 for _ in 0..64 {
21456 client.egress.try_send(route_bind_ack(1)).unwrap();
21457 }
21458 let handler = rig.handler.clone();
21459 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21460 std::future::poll_fn(|cx| {
21461 assert!(open.as_mut().poll(cx).is_pending());
21462 std::task::Poll::Ready(())
21463 })
21464 .await;
21465 record.bind_proof_sha256 = Some(NEXT_HASH.into());
21466 rig.sync(vec![record]).await;
21467 rx.try_recv().unwrap();
21468 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21469 .await
21470 .unwrap()
21471 .unwrap();
21472 assert_eq!(
21473 parse_error(&replies[0])["code"],
21474 error_codes::SCOPE_BIND_PROOF_MISMATCH
21475 );
21476 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
21477 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21478 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21479 }
21480
21481 #[tokio::test]
21482 async fn scope_bind_proof_reserved_openers_ignore_proof() {
21483 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21484 let mut rig = rig().await;
21485 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21486 .await;
21487 for opener in [OWNER, AFT] {
21488 for proof in [None, Some("malformed+proof=")] {
21489 let route = rig
21490 .bound(
21491 Some(opener),
21492 PLEXUS,
21493 Some(
21494 rig.selector("s", Some(1))
21495 .with_bind_proof(proof.map(str::to_string)),
21496 ),
21497 )
21498 .await;
21499 assert!(rig.live(&route));
21500 }
21501 }
21502 }
21503
21504 #[tokio::test]
21505 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
21506 use std::future::Future;
21507 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21508 const NEXT_HASH: &str =
21509 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21510 let mut rig = rig().await;
21511 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21512 .await;
21513 let (client, mut rx, frame) =
21514 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
21515 for _ in 0..64 {
21516 client.egress.try_send(route_bind_ack(1)).unwrap();
21517 }
21518 let handler = rig.handler.clone();
21519 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21520 std::future::poll_fn(|cx| {
21521 assert!(open.as_mut().poll(cx).is_pending());
21522 std::task::Poll::Ready(())
21523 })
21524 .await;
21525 rig.sync(vec![
21526 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
21527 ])
21528 .await;
21529 rx.try_recv().unwrap();
21530 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21531 let bind = std::future::poll_fn(|cx| {
21532 assert!(open.as_mut().poll(cx).is_pending());
21533 module_rx.poll_recv(cx)
21534 })
21535 .await
21536 .unwrap()
21537 .frame;
21538 handler
21539 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
21540 .await
21541 .unwrap();
21542 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21543 .await
21544 .unwrap()
21545 .unwrap();
21546 assert_eq!(replies.len(), 1);
21547 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
21550 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21551 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
21552 let route = rig
21553 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
21554 .await;
21555 assert!(rig.live(&route));
21556 }
21557
21558 #[tokio::test]
21559 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
21560 {
21561 let mut rig = rig_with_flow_support(false).await;
21562 rig.sync(vec![session(1)]).await;
21563 let route = rig
21564 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21565 .await;
21566 assert!(rig.live(&route));
21567 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
21568 let mut record = session(1);
21569 record.attributes.flow_id = Some("flow:7".to_string());
21570 rig.sync(vec![record]).await;
21571 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
21572 assert!(rig.live(&unscoped));
21573 assert_eq!(unscoped.stamp(), None);
21574 }
21575
21576 #[tokio::test]
21577 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
21578 let mut rig = rig().await;
21579 let mut record = session(1);
21580 record.attributes.flow_id = Some("flow:7".to_string());
21581 rig.sync(vec![record.clone()]).await;
21582 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21583 let mut owner_route = rig
21584 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21585 .await;
21586 let mut carrier_route = rig
21587 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
21588 .await;
21589 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21590 rig.sync(vec![record.clone()]).await;
21591 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21592 assert!(rig.live(&owner_route) && owner_route.untouched());
21593 assert!(rig.live(&carrier_route) && carrier_route.untouched());
21594
21595 record.attributes.flow_id = Some("flow:8".to_string());
21596 rig.sync(vec![record]).await;
21597 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
21598 let before = before.unwrap();
21599 assert_eq!(after.scope_epoch, before.scope_epoch);
21600 assert!(after.version > before.version);
21601 for route in [&mut owner_route, &mut carrier_route] {
21602 assert!(!rig.live(route));
21603 assert_eq!(
21604 route.closed_reason(),
21605 RouteCloseReason::ScopeDelegationChanged
21606 );
21607 }
21608 assert!(rig.live(&unscoped) && unscoped.untouched());
21609 for target in [PLEXUS, OTHER] {
21612 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
21613 let goodbye = module_rx
21614 .try_recv()
21615 .expect("the provider sees the drain")
21616 .frame;
21617 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21618 assert!(module_rx.try_recv().is_err());
21619 }
21620 let rebound = rig
21621 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21622 .await;
21623 assert_eq!(
21624 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
21625 Some("flow:8")
21626 );
21627 }
21628
21629 #[tokio::test]
21634 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
21635 let mut rig = rig().await;
21636 rig.sync(vec![session(1)]).await;
21637 let mut cotenant = rig
21638 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21639 .await;
21640
21641 let mut changed = session(1);
21642 changed.child_owners.push(Principal::Reserved {
21643 module_id: MAGIC.to_string(),
21644 });
21645 let mut ended = None;
21646 for (code, next) in [
21647 (error_codes::SCOPE_CHANGED, vec![changed]),
21648 (error_codes::SCOPE_ENDED, Vec::new()),
21649 ] {
21650 let relayed = rig
21651 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21652 .await;
21653 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
21654 ended = Some(next.is_empty());
21655 rig.sync(next).await;
21656 rig.ack(&relayed).await;
21657 let replies = relayed.task.await.unwrap();
21658 assert_eq!(replies.len(), 1, "{replies:?}");
21659 assert_eq!(parse_error(&replies[0])["code"], code);
21660 assert_eq!(
21661 subc_protocol::error_codes::is_retryable_route_open(code),
21662 code == error_codes::SCOPE_CHANGED
21663 );
21664 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21665 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21667 let mut goodbyes = Vec::new();
21670 while let Ok(outbound) = plexus_rx.try_recv() {
21671 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
21672 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
21673 }
21674 assert!(
21675 goodbyes.contains(&(bind_channel, bind_epoch)),
21676 "{goodbyes:?}"
21677 );
21678 assert!(rig
21679 .handler
21680 .registry
21681 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
21682 .unwrap()
21683 .is_some());
21684 }
21685 assert_eq!(ended, Some(true));
21686 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
21689 }
21690
21691 #[tokio::test]
21694 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
21695 struct Case {
21696 name: &'static str,
21697 change: fn(&mut ScopeRecord),
21698 closed: [Option<RouteCloseReason>; 4],
21701 }
21702 use RouteCloseReason::*;
21703 let cases = [
21704 Case {
21705 name: "a carrier entry removed",
21706 change: |r| {
21707 r.carriers.retain(|c| {
21708 c.principal
21709 != Principal::Reserved {
21710 module_id: AFT.to_string(),
21711 }
21712 })
21713 },
21714 closed: [None, Some(ScopeCarrierRemoved), None, None],
21715 },
21716 Case {
21717 name: "a target removed from a carrier",
21718 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
21719 closed: [None, None, None, Some(ScopeCarrierRemoved)],
21720 },
21721 Case {
21722 name: "a bare carrier narrowed to targets",
21723 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
21724 closed: [None, Some(ScopeCarrierRemoved), None, None],
21725 },
21726 Case {
21727 name: "delegates turned off",
21728 change: |r| r.attributes.delegates = false,
21729 closed: [Some(ScopeDelegationChanged); 4],
21730 },
21731 Case {
21732 name: "agent_id changed",
21733 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
21734 closed: [Some(ScopeDelegationChanged); 4],
21735 },
21736 Case {
21737 name: "a carrier added, child owners changed, the record re-sent",
21738 change: |r| {
21739 r.carriers.push(carrier(MAGIC, None));
21740 r.child_owners.push(Principal::Reserved {
21741 module_id: MAGIC.to_string(),
21742 });
21743 },
21744 closed: [None; 4],
21745 },
21746 Case {
21747 name: "a target added",
21748 change: |r| {
21749 r.carriers[1]
21750 .targets
21751 .as_mut()
21752 .unwrap()
21753 .push("third".to_string())
21754 },
21755 closed: [None; 4],
21756 },
21757 Case {
21758 name: "delegates turned on",
21759 change: |r| r.attributes.delegates = true,
21760 closed: [None; 4],
21761 },
21762 ];
21763 for case in cases {
21764 let mut rig = rig().await;
21765 rig.sync(vec![session(1)]).await;
21766 let scope = || Some(rig_selector("s", Some(1)));
21767 let mut routes = [
21768 rig.bound(Some(OWNER), PLEXUS, scope()).await,
21769 rig.bound(Some(AFT), PLEXUS, scope()).await,
21770 rig.bound(Some(BROCA), PLEXUS, scope()).await,
21771 rig.bound(Some(BROCA), OTHER, scope()).await,
21772 ];
21773 let mut record = session(1);
21774 (case.change)(&mut record);
21775 rig.sync(vec![record]).await;
21776 for (index, expected) in case.closed.iter().enumerate() {
21777 let route = &mut routes[index];
21778 match expected {
21779 Some(reason) => {
21780 assert!(!rig.live(route), "{}: route {index} still live", case.name);
21781 assert_eq!(
21782 route.closed_reason(),
21783 *reason,
21784 "{}: route {index}",
21785 case.name
21786 );
21787 }
21788 None => {
21789 assert!(rig.live(route), "{}: route {index} closed", case.name);
21790 assert!(
21791 route.untouched(),
21792 "{}: route {index} was told something",
21793 case.name
21794 );
21795 }
21796 }
21797 }
21798 }
21799 }
21800
21801 #[tokio::test]
21802 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
21803 for next in [Vec::new(), vec![session(2)]] {
21805 let mut rig = rig().await;
21806 rig.sync(vec![session(1)]).await;
21807 let mut route = rig
21808 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21809 .await;
21810 rig.sync(next).await;
21811 assert!(!rig.live(&route));
21812 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
21813 }
21814
21815 let mut rig = rig().await;
21818 let mut child = session(1);
21819 child.scope_ref = "child".to_string();
21820 child.kind = ScopeKind::Worker;
21821 child.parent = Some(ScopeParent::new(
21822 Principal::Reserved {
21823 module_id: OWNER.to_string(),
21824 },
21825 "s".to_string(),
21826 1,
21827 ));
21828 rig.sync(vec![session(1), child.clone()]).await;
21829 let mut child_route = rig
21830 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
21831 .await;
21832 assert_eq!(
21833 child_route.stamp().unwrap().parent_state,
21834 Some(ParentState::Linked)
21835 );
21836 rig.sync(vec![child]).await;
21837 assert!(!rig.live(&child_route));
21838 assert_eq!(
21839 child_route.closed_reason(),
21840 RouteCloseReason::ScopeParentEnded
21841 );
21842 }
21843
21844 #[tokio::test]
21845 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
21846 ) {
21847 let mut rig = rig().await;
21848 rig.sync(vec![session(1)]).await;
21849 let mut route = rig
21850 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21851 .await;
21852 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21853 rig.sync(vec![session(1)]).await;
21854 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21855 assert!(rig.live(&route) && route.untouched());
21856
21857 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
21861 .forwarding
21862 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21863 .unwrap()
21864 else {
21865 panic!("the route is bound");
21866 };
21867 binding.flow.acquire_tagged(9, false).await.unwrap();
21868 let mut widened = session(1);
21869 widened.carriers.push(carrier(MAGIC, None));
21870 rig.sync(vec![widened]).await;
21871 assert!(rig.live(&route) && route.untouched());
21872 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21873 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
21874 assert_eq!(binding.flow.in_flight(), 1);
21877 binding
21878 .flow
21879 .acquire_tagged(10, false)
21880 .await
21881 .expect("the flow is still open");
21882 }
21883
21884 #[tokio::test]
21887 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
21888 let mut rig = rig().await;
21889 rig.sync(vec![session(1)]).await;
21890 let mut on_incumbent = rig
21891 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21892 .await;
21893
21894 let (candidate, _candidate_rx) = wide_ctx(9);
21897 let registration = rig
21898 .handler
21899 .registry
21900 .register_candidate_with_control_ops(
21901 manifest(PLEXUS, PROTOCOL_VERSION),
21902 PROTOCOL_VERSION,
21903 candidate.connection_id,
21904 module_baseline_control_ops(),
21905 )
21906 .unwrap();
21907 rig.forwarding
21908 .register_candidate_module_connection(
21909 candidate.connection_id,
21910 PLEXUS.to_string(),
21911 PROTOCOL_VERSION,
21912 manifest_concurrency(®istration.manifest),
21913 candidate.egress.clone(),
21914 )
21915 .unwrap();
21916 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
21917 rig.handler
21918 .registry
21919 .promote_candidate(PLEXUS)
21920 .unwrap()
21921 .unwrap();
21922 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
21923
21924 rig.sync(Vec::new()).await;
21925 assert!(!rig.live(&on_incumbent));
21926 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
21927 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21928 let goodbye = incumbent_rx
21929 .try_recv()
21930 .expect("the superseded endpoint is told")
21931 .frame;
21932 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21933 }
21934 }
21935}
21936
21937#[cfg(test)]
21938mod concurrency_default_exposure_tests {
21939 use super::*;
21940
21941 fn hello_body(role_json: &str) -> Vec<u8> {
21942 format!(
21943 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
21944 )
21945 .into_bytes()
21946 }
21947
21948 fn manifest_from(body: &[u8]) -> ModuleManifest {
21949 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
21950 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
21951 .expect("manifest parses")
21952 }
21953
21954 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
21955
21956 #[test]
21957 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
21958 let body = hello_body(&format!(
21959 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
21960 ));
21961 let manifest = manifest_from(&body);
21962 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21965 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
21966 }
21967
21968 #[test]
21969 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
21970 let body = hello_body(&format!(
21971 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
21972 ));
21973 let manifest = manifest_from(&body);
21974 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21975 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21976 }
21977
21978 #[test]
21979 fn non_management_roles_are_never_reported() {
21980 let body = hello_body(
21981 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
21982 );
21983 let manifest = manifest_from(&body);
21984 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21985 }
21986}