1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26 error_codes,
27 manifest::{
28 validate_hello_capability_grammar, validate_hello_confirm_operations,
29 validate_hello_display_name, validate_hello_event_declarations,
30 validate_hello_flow_operations, validate_hello_operation_deferral,
31 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32 validate_hello_self_signal_declarations, validate_role_declarations_value,
33 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34 ProviderRole,
35 },
36 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
37 provider_declarations::ProviderDeclarations,
38 scope::{
39 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
40 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
41 SCOPE_SYNC_OP,
42 },
43 session::{
44 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
45 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
46 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
47 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
48 },
49 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
50 Priority, RouteTarget, PROTOCOL_VERSION,
51};
52use tokio::time::{timeout_at, Instant};
53use tracing::{debug, info, instrument::WithSubscriber, warn};
54
55use crate::{
56 caller_identity::CallerIdentity,
57 capability_requirements::{
58 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
59 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
60 RequirementStatus, RuntimeModule,
61 },
62 child_roster::DaemonShutdownFlag,
63 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
64 forwarding::{
65 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
66 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
67 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
68 },
69 observability::{
70 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
71 },
72 privileged_prompt::{prompt_text, PrivilegedOperation},
73 provenance::{
74 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
75 },
76 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
77 router::{daemon_error_flags, RouteCtx, RouterError},
78 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
79 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
80 stderr_tail::{CaptureState, TailEntry},
81 supervise::{
82 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
83 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
84 },
85 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
86};
87
88pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
94
95const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
96const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
97const CAP_PING_PONG: &str = "ping_pong_v1";
98const CAP_SESSION_ATTACH: &str = "session_attach_v1";
99const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
100
101const SUBC_CONTROL_OPS: &[&str] = &[
102 "claim.mint",
103 "claim.present",
104 ops::DAEMON_SHUTDOWN_INTENT,
105 ops::ACTIVITY_RECORD,
106 ops::ACTIVITY_QUERY,
107 ops::ACTIVITY_SUBSCRIBE,
108 ops::SERVER_DESCRIBE,
109 ops::CATALOG_LIST,
110 ops::CATALOG_WATCH,
111 ops::ROUTE_OPEN,
112 ops::ROUTE_POLL,
113 ops::ROUTE_CLOSING,
114 ops::ROUTE_CLOSED,
115 ops::SUPERVISOR_LIST,
116 ops::SUPERVISOR_RESTART,
117 ops::SUPERVISOR_SWAP,
118 ops::SUPERVISOR_RELOAD,
119 ops::SUPERVISOR_RESCAN,
120 ops::SUPERVISOR_RELEASE_RESERVED,
121 ops::SUPERVISOR_SET_ENABLED,
122 ops::SUPERVISOR_HEALTH_PROBE,
123 ops::SUPERVISOR_HEALTH,
124 ops::SUPERVISOR_STDERR_TAIL,
125 ops::SUPERVISOR_TERMINALS,
126 ops::SUPERVISOR_ROUTES,
127 ops::SUPERVISOR_PROVENANCE,
128 ops::SUPERVISOR_SPAWN_SNAPSHOT,
129 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
130];
131
132const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[
139 ops::SUPERVISOR_RESCAN_CHECKED,
140 "config.status",
141 "config.plan",
142 "config.apply",
143 "config.stamp",
144 "config.unstamp",
145 "config.history",
146 "config.rollback",
147];
148
149mod caller_attestation;
150mod channel_operations;
151#[cfg(all(test, unix, feature = "test-support"))]
152#[path = "control/channel_operations_tests.rs"]
153mod channel_operations_tests;
154pub(crate) mod claims;
155#[cfg(test)]
156#[path = "control/config_advertisement_tests.rs"]
157mod config_advertisement_tests;
158mod config_gate;
159use caller_attestation::{caller_for_target, scope_acts_for_another};
160#[cfg(all(test, unix, feature = "test-support"))]
161#[path = "control/caller_attestation_tests.rs"]
162mod caller_attestation_tests;
163mod scope_expiry;
164
165mod agent_commands;
166pub(crate) mod catalog;
167mod rescan_checked;
168use rescan_checked::{RescanMode, RescanPlan};
169
170const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
171 "claim.mint",
172 "claim.present",
173 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
174 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
175 "supervisor.live_roots",
176 "supervisor.agent_commands",
177 SCOPE_SYNC_OP,
178 SCOPE_APPLY_OP,
179 SCOPE_DESCRIBE_OP,
180 subc_protocol::scope::ROUTE_AUTHORIZE_OP,
181 "operator.confirm",
182];
183
184const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
188
189const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
190
191pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
199
200pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
215
216pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
234
235const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
236const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
237
238fn reload_verdict(
239 configured: &Path,
240 spawned_from: Option<&Path>,
241 image: subc_control::RunningImageAgreement,
242) -> PendingReloadVerdict {
243 let path = match spawned_from {
244 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
245 ReloadPathAgreement::Match
246 }
247 Some(spawned_from) => ReloadPathAgreement::Mismatch {
248 configured: configured.to_path_buf(),
249 spawned_from: spawned_from.to_path_buf(),
250 },
251 None => ReloadPathAgreement::Unavailable {
252 reason: if matches!(
253 image,
254 subc_control::RunningImageAgreement::Unavailable {
255 reason: subc_control::RunningImageUnavailableReason::NotRunning
256 }
257 ) {
258 ReloadPathUnavailableReason::NotRunning
259 } else {
260 ReloadPathUnavailableReason::SpawnedPathUnavailable
261 },
262 },
263 };
264 PendingReloadVerdict::new(path, image)
265}
266
267fn launch_spec_agreement(
268 configured: &crate::supervise::ModuleSpec,
269 running: bool,
270 spawned: Option<&crate::supervise::LaunchSpecDigest>,
271) -> subc_control::LaunchSpecAgreement {
272 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
273 if !running {
274 return Agreement::Unavailable {
275 reason: Reason::NotRunning,
276 };
277 }
278 match spawned {
279 None => Agreement::Unavailable {
280 reason: Reason::SpawnedSpecUnavailable,
281 },
282 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
283 Agreement::Match
284 }
285 Some(_) => Agreement::Mismatch,
286 }
287}
288
289fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
290 if configured == spawned_from {
291 return true;
292 }
293 #[cfg(windows)]
294 {
295 match (
301 std::fs::canonicalize(configured),
302 std::fs::canonicalize(spawned_from),
303 ) {
304 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
305 _ => false,
306 }
307 }
308 #[cfg(not(windows))]
309 {
310 false
311 }
312}
313
314#[derive(Clone)]
315struct DaemonProvenanceFacts {
316 build: DaemonBuildProvenance,
317 pid: Option<u32>,
318 started_at_ms: Option<u64>,
319 start_clock: Option<crate::clock::StartClock>,
320 executable_path: Option<PathBuf>,
321 executable_identity: Option<SpawnedFileIdentity>,
322 process_start_time: Option<u64>,
323 probe: ExecutableIdentityProbe,
324}
325
326impl Default for DaemonProvenanceFacts {
327 fn default() -> Self {
328 Self {
329 build: DaemonBuildProvenance {
330 build_git_sha: None,
331 build_lock_digest: None,
332 },
333 pid: None,
334 started_at_ms: None,
335 start_clock: None,
336 executable_path: None,
337 executable_identity: None,
338 process_start_time: None,
339 probe: ExecutableIdentityProbe::default(),
340 }
341 }
342}
343
344#[derive(Debug, Clone)]
345struct SupervisorRescanContext {
346 supervisor: Supervisor,
347 config_path: PathBuf,
348 configured_port: Option<u16>,
349 storage_config: Option<crate::daemon_config::StorageConfig>,
350 admission_facts_carrier_module_id: Option<String>,
351 admission_facts_targets: Option<Vec<String>>,
352 scope_authority_owners: Vec<String>,
353 federated_scope_owner: String,
354 privileged_operator_prompt: PrivilegedOperatorPrompt,
355 generation: Arc<std::sync::atomic::AtomicU64>,
361 module_entries: Arc<Mutex<rescan_checked::ModuleEntries>>,
362}
363
364const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
385 "reloading",
386 "supervisor_not_live",
387 "registration_not_active",
388 "no_forwarding_connection",
389 "relay_send_failed",
390];
391
392#[cfg(test)]
394struct ActivityRequestPause {
395 connection: ConnectionId,
396 started: tokio::sync::Notify,
397 cancelled: tokio::sync::Notify,
398 cleanup_started: std::sync::atomic::AtomicBool,
399 cancelled_after_cleanup: std::sync::atomic::AtomicBool,
400}
401
402#[cfg(test)]
403struct ActivityRequestCancellation(Arc<ActivityRequestPause>);
404
405#[cfg(test)]
406impl Drop for ActivityRequestCancellation {
407 fn drop(&mut self) {
408 self.0.cancelled_after_cleanup.store(
409 self.0
410 .cleanup_started
411 .load(std::sync::atomic::Ordering::Acquire),
412 std::sync::atomic::Ordering::Release,
413 );
414 self.0.cancelled.notify_one();
415 }
416}
417
418#[cfg(test)]
420type TestPause = (
421 tokio::sync::oneshot::Sender<()>,
422 std::sync::mpsc::Receiver<()>,
423);
424
425#[cfg(test)]
427type TestHook = Box<dyn FnOnce() + Send>;
428
429#[cfg(test)]
431type AsyncTestPause = (
432 tokio::sync::oneshot::Sender<()>,
433 tokio::sync::oneshot::Receiver<()>,
434);
435
436#[derive(Clone)]
438pub struct ControlHandler {
439 #[cfg(test)]
440 activity_request_pause: Arc<Mutex<Option<Arc<ActivityRequestPause>>>>,
441 #[cfg(test)]
445 cleanup_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
446 #[cfg(test)]
449 after_nonce_attestation: Arc<Mutex<Option<TestHook>>>,
450 #[cfg(test)]
453 hello_ingress_pause: Arc<Mutex<Option<AsyncTestPause>>>,
454 #[cfg(test)]
456 agent_commands_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
457 claims: Arc<claims::Claims>,
458 registry: Arc<Registry>,
459 config_runtime: Option<Arc<crate::config_gate::runtime::RuntimeOwner>>,
460 config_review_admission: Arc<dyn crate::config_gate::ReviewAdmission>,
461 config_review_admission_is_default: bool,
462 activity: Option<crate::activity::ActivityLedger>,
463 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
464 catalog_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
465 forwarding: Arc<ForwardingTable>,
466 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
467 supervisor: SupervisorHandle,
468 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
469 agent_declarations: agent_commands::Queues,
470 list_admission: Arc<tokio::sync::Semaphore>,
471 subc_capabilities: Arc<[String]>,
472 route_bind_relay_timeout: Duration,
476 route_bind_relay_timeouts: BTreeMap<String, Duration>,
480 route_bind_breakers: RouteBindBreakers,
483 route_bind_concurrency: RouteBindConcurrency,
486 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
491 route_bind_breaker_threshold: u32,
493 route_bind_breaker_cooldown: Duration,
495 health_probe_timeout: Duration,
496 storage_config: Option<crate::daemon_config::StorageConfig>,
499 machine_id: Option<crate::machine_id::MachineId>,
503 admission_facts_carrier_module_id: Option<String>,
504 admission_facts_targets: Option<Vec<String>>,
505 scopes: Arc<RwLock<ScopeTable>>,
509 scope_deadline_added: Arc<tokio::sync::Notify>,
513 scope_authority_owners: Vec<String>,
516 federated_scope_owner: String,
517 privileged_operator_prompt: PrivilegedOperatorPrompt,
519 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
522 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
523 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
524 captured_peers: Arc<Mutex<HashMap<ConnectionId, subc_os::local_peer::CapturedPeer>>>,
525 config_desktop_stamps:
526 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
527 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
528 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
529 pub(crate) operator_apps:
530 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
531 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
536 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
542 #[cfg(test)]
544 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
545 #[cfg(test)]
546 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
547 #[cfg(test)]
548 recovery_exit_test_after_step: Option<fn(usize) -> Result<(), crate::config_gate::GateError>>,
549 #[cfg(test)]
550 recovery_exit_test_completed: Option<fn(u64)>,
551 #[cfg(test)]
552 apply_reconcile_pause: Arc<Mutex<Option<AsyncTestPause>>>,
553 #[cfg(test)]
554 recovery_exit_test_quiet_retry: bool,
555 #[cfg(test)]
556 recovery_exit_test_retry_pause: Option<fn(u64) -> Option<PathBuf>>,
557 #[cfg(test)]
558 config_delivery_test_wakes:
559 Option<tokio::sync::mpsc::UnboundedSender<(&'static str, tokio::time::Instant)>>,
560 #[cfg(all(test, unix))]
561 config_delivery_test_idle:
562 Option<tokio::sync::mpsc::UnboundedSender<tokio::sync::oneshot::Sender<()>>>,
563 #[allow(dead_code)]
567 #[cfg(windows)]
568 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
569 #[allow(dead_code)]
570 #[cfg(windows)]
571 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
572 #[allow(dead_code)]
576 #[cfg(windows)]
577 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
578 #[cfg(all(test, windows))]
581 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
582 #[cfg(test)]
583 caller_lookup: Option<TestCallerLookup>,
584 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
587 rescan: Option<SupervisorRescanContext>,
588 connected_clients: ConnectedClients,
589 counters: DaemonCounters,
590 local_endpoint_unavailable: Option<String>,
593 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
594 daemon_shutdown: DaemonShutdownFlag,
599 daemon_provenance: DaemonProvenanceFacts,
600 #[cfg(test)]
601 control_dispatch_delay: Option<Duration>,
602 #[cfg(test)]
603 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
604 #[cfg(test)]
608 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
609 #[cfg(test)]
615 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
616 #[cfg(test)]
617 hello_ack_to_publication_hold: Option<Arc<std::sync::Barrier>>,
618 #[cfg(test)]
619 hello_before_publication_hold: Option<Arc<std::sync::Barrier>>,
620 #[cfg(test)]
621 hello_after_launch_check_hold: Option<Arc<std::sync::Barrier>>,
622 #[cfg(test)]
623 hello_inside_publication_hold: Option<Arc<std::sync::Barrier>>,
624}
625
626impl fmt::Debug for ControlHandler {
627 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
628 f.debug_struct("ControlHandler")
629 .field("registry", &self.registry)
630 .field("forwarding", &self.forwarding)
631 .field("process_liveness", &self.process_liveness.is_some())
632 .field("supervisor", &self.supervisor)
633 .field("subc_capabilities", &self.subc_capabilities)
634 .finish()
635 }
636}
637
638struct RouteOpenRequest {
639 target: RouteTarget,
640 identity: BindIdentity,
641 consumer_identity: Option<ConsumerIdentity>,
642 consumer_capabilities: Option<Vec<String>>,
643 role_versions: Option<BTreeMap<String, String>>,
644 busy_refusals: Option<bool>,
645 admission_facts: Option<serde_json::Value>,
646 scope: Option<ScopeSelector>,
647 accepts_close_reasons: Option<Vec<String>>,
648}
649
650#[derive(Clone, Copy)]
651enum RouteOpenPhase {
652 Admission,
653 ProjectRoot,
654 EgressReservation,
655 BindSend,
656 BindAckWait,
657}
658
659struct RouteOpenTimings {
663 module_id: String,
664 principal: String,
665 admission: Duration,
666 project_root: Duration,
667 egress_reservation: Duration,
668 bind_send: Duration,
669 bind_ack_wait: Duration,
670 phase: Option<RouteOpenPhase>,
671 phase_started_at: Instant,
672}
673
674impl RouteOpenTimings {
675 fn new() -> Self {
676 Self {
677 module_id: String::new(),
678 principal: "unresolved".to_string(),
680 admission: Duration::ZERO,
681 project_root: Duration::ZERO,
682 egress_reservation: Duration::ZERO,
683 bind_send: Duration::ZERO,
684 bind_ack_wait: Duration::ZERO,
685 phase: Some(RouteOpenPhase::Admission),
686 phase_started_at: Instant::now(),
687 }
688 }
689
690 fn finish(&mut self) {
691 let elapsed = self.phase_started_at.elapsed();
692 match self.phase.take() {
693 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
694 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
695 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
696 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
697 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
698 None => {}
699 }
700 }
701
702 fn enter(&mut self, phase: RouteOpenPhase) {
703 self.finish();
704 self.phase = Some(phase);
705 self.phase_started_at = Instant::now();
706 }
707}
708
709struct RouteBindReservationGuard {
710 forwarding: Arc<ForwardingTable>,
711 endpoint: ModuleEndpointId,
712 relay_corr: u64,
713 armed: bool,
714}
715
716struct ModuleControlRpcGuard {
717 forwarding: Arc<ForwardingTable>,
718 endpoint: ModuleEndpointId,
719 corr: u64,
720 armed: bool,
721}
722
723impl ModuleControlRpcGuard {
724 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
725 Self {
726 forwarding,
727 endpoint,
728 corr,
729 armed: true,
730 }
731 }
732
733 fn disarm(&mut self) {
734 self.armed = false;
735 }
736}
737
738impl Drop for ModuleControlRpcGuard {
739 fn drop(&mut self) {
740 if self.armed {
741 let _ = self
742 .forwarding
743 .cancel_module_control_rpc(self.endpoint, self.corr);
744 }
745 }
746}
747
748impl RouteBindReservationGuard {
749 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
750 Self {
751 forwarding,
752 endpoint,
753 relay_corr,
754 armed: true,
755 }
756 }
757
758 fn release_and_disarm(&mut self) {
759 if !self.armed {
760 return;
761 }
762 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
763 self.endpoint,
764 self.relay_corr,
765 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
766 ) {
767 send_goodbye_target_best_effort(
768 &self.forwarding.counters(),
769 &target,
770 "canceled route.bind",
771 );
772 }
773 self.armed = false;
774 }
775
776 fn disarm(&mut self) {
777 self.armed = false;
778 }
779}
780
781impl Drop for RouteBindReservationGuard {
782 fn drop(&mut self) {
783 self.release_and_disarm();
784 }
785}
786
787#[derive(Debug, Clone, Default)]
817pub(crate) struct RouteBindBreakers {
818 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
819}
820
821#[derive(Debug, Clone, Default)]
822pub(crate) struct RouteBindConcurrency {
823 modules: Arc<Mutex<HashMap<String, usize>>>,
824}
825
826struct RouteBindConcurrencyGuard {
827 concurrency: RouteBindConcurrency,
828 module_id: String,
829}
830
831impl RouteBindConcurrency {
832 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
835 let mut modules = self
836 .modules
837 .lock()
838 .expect("route.bind concurrency mutex poisoned");
839 let in_flight = modules.entry(module_id.to_string()).or_default();
840 if *in_flight >= limit {
841 return Err(*in_flight);
842 }
843 *in_flight += 1;
844 Ok(RouteBindConcurrencyGuard {
845 concurrency: self.clone(),
846 module_id: module_id.to_string(),
847 })
848 }
849}
850
851impl Drop for RouteBindConcurrencyGuard {
852 fn drop(&mut self) {
853 let mut modules = self
854 .concurrency
855 .modules
856 .lock()
857 .expect("route.bind concurrency mutex poisoned");
858 let remove = {
859 let in_flight = modules
860 .get_mut(&self.module_id)
861 .expect("admitted route.bind has a concurrency entry");
862 *in_flight -= 1;
863 *in_flight == 0
864 };
865 if remove {
866 modules.remove(&self.module_id);
867 }
868 }
869}
870
871#[derive(Debug, Default)]
872struct ModuleBreakerState {
873 consecutive_timeouts: u32,
875 cooldown_until: Option<Instant>,
878 probe_in_flight: Option<Arc<()>>,
883}
884
885enum RouteBindAdmission<'a> {
887 Admitted {
888 guard: RouteBindBreakerGuard<'a>,
889 probe: bool,
892 },
893 Refused {
894 consecutive_timeouts: u32,
895 retry_in: Duration,
899 probe_in_flight: bool,
900 },
901}
902
903struct RouteBindBreakerGuard<'a> {
910 breakers: RouteBindBreakers,
911 module_id: &'a str,
912 probe_token: Option<Arc<()>>,
913 settled: bool,
914}
915
916impl RouteBindBreakerGuard<'_> {
917 fn record_accepted(&mut self) -> bool {
921 self.settled = true;
922 self.breakers.record_accepted(self.module_id)
923 }
924
925 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
928 self.settled = true;
929 self.breakers.record_timeout(
930 self.module_id,
931 self.probe_token.as_ref(),
932 threshold,
933 cooldown,
934 )
935 }
936
937 fn record_inconclusive(&mut self) {
946 self.settled = true;
947 self.breakers
948 .record_inconclusive(self.module_id, self.probe_token.as_ref());
949 }
950}
951
952impl Drop for RouteBindBreakerGuard<'_> {
953 fn drop(&mut self) {
954 if !self.settled {
955 self.breakers
956 .record_inconclusive(self.module_id, self.probe_token.as_ref());
957 }
958 }
959}
960
961struct BreakerOpened {
965 consecutive_timeouts: u32,
966 reopened_after_probe: bool,
969}
970
971impl RouteBindBreakers {
972 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
973 self.modules
974 .lock()
975 .expect("route.bind breaker mutex poisoned")
976 }
977
978 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
981 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
982 probe: probe_token.is_some(),
983 guard: RouteBindBreakerGuard {
984 breakers: self.clone(),
985 module_id,
986 probe_token,
987 settled: false,
988 },
989 };
990
991 let mut modules = self.lock();
992 let Some(state) = modules.get_mut(module_id) else {
993 return admitted(None);
994 };
995 let Some(cooldown_until) = state.cooldown_until else {
996 return admitted(None);
997 };
998 if state.probe_in_flight.is_some() {
999 return RouteBindAdmission::Refused {
1000 consecutive_timeouts: state.consecutive_timeouts,
1001 retry_in: Duration::ZERO,
1002 probe_in_flight: true,
1003 };
1004 }
1005 let now = Instant::now();
1006 if now < cooldown_until {
1007 return RouteBindAdmission::Refused {
1008 consecutive_timeouts: state.consecutive_timeouts,
1009 retry_in: cooldown_until - now,
1010 probe_in_flight: false,
1011 };
1012 }
1013 let token = Arc::new(());
1014 state.probe_in_flight = Some(Arc::clone(&token));
1015 admitted(Some(token))
1016 }
1017
1018 fn record_accepted(&self, module_id: &str) -> bool {
1019 self.lock()
1020 .remove(module_id)
1021 .is_some_and(|state| state.cooldown_until.is_some())
1022 }
1023
1024 fn record_timeout(
1025 &self,
1026 module_id: &str,
1027 probe_token: Option<&Arc<()>>,
1028 threshold: u32,
1029 cooldown: Duration,
1030 ) -> Option<BreakerOpened> {
1031 let mut modules = self.lock();
1032 let state = modules.entry(module_id.to_string()).or_default();
1033 let was_open = state.cooldown_until.is_some();
1034 let was_probe = Self::owns_probe(state, probe_token);
1035 if was_probe {
1036 state.probe_in_flight = None;
1037 }
1038 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
1039 if state.consecutive_timeouts < threshold {
1040 return None;
1041 }
1042 state.cooldown_until = Some(Instant::now() + cooldown);
1043 Some(BreakerOpened {
1044 consecutive_timeouts: state.consecutive_timeouts,
1045 reopened_after_probe: was_open && was_probe,
1046 })
1047 }
1048
1049 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
1050 state
1054 .probe_in_flight
1055 .as_ref()
1056 .zip(token)
1057 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
1058 }
1059
1060 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
1061 if let Some(state) = self.lock().get_mut(module_id) {
1062 if Self::owns_probe(state, probe_token) {
1063 state.probe_in_flight = None;
1064 }
1065 }
1066 }
1067
1068 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
1083 self.lock()
1084 .remove(module_id)
1085 .map(|state| state.consecutive_timeouts)
1086 .filter(|discarded| *discarded > 0)
1087 }
1088
1089 fn open_snapshot(&self) -> Option<serde_json::Value> {
1096 let now = Instant::now();
1097 let modules = self.lock();
1098 let open = modules
1099 .iter()
1100 .filter_map(|(module_id, state)| {
1101 let cooldown_until = state.cooldown_until?;
1102 Some((
1103 module_id.clone(),
1104 serde_json::json!({
1105 "consecutive_timeouts": state.consecutive_timeouts,
1106 "cooldown_remaining_ms":
1107 cooldown_until.saturating_duration_since(now).as_millis() as u64,
1108 "probe_in_flight": state.probe_in_flight.is_some(),
1109 }),
1110 ))
1111 })
1112 .collect::<serde_json::Map<String, serde_json::Value>>();
1113 (!open.is_empty()).then_some(serde_json::Value::Object(open))
1114 }
1115}
1116
1117impl ControlHandler {
1118 pub(crate) fn install_registration_service(
1119 &self,
1120 router: std::sync::Weak<crate::Router>,
1121 auth: crate::server::ServerAuth,
1122 ) {
1123 self.supervisor.install_registration_service(router, auth);
1124 }
1125 pub fn new(registry: Arc<Registry>) -> Self {
1126 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
1127 }
1128
1129 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
1130 let counters = forwarding.counters();
1131 let route_bind_breakers = forwarding.route_bind_breakers();
1135 let route_bind_concurrency = forwarding.route_bind_concurrency();
1136 let route_outages = forwarding.route_outages();
1137 Self {
1138 registry,
1139 #[cfg(test)]
1140 activity_request_pause: Arc::new(Mutex::new(None)),
1141 #[cfg(test)]
1142 cleanup_pause: Arc::new(Mutex::new(None)),
1143 #[cfg(test)]
1144 after_nonce_attestation: Arc::new(Mutex::new(None)),
1145 #[cfg(test)]
1146 hello_ingress_pause: Arc::new(Mutex::new(None)),
1147 #[cfg(test)]
1148 agent_commands_pause: Arc::new(Mutex::new(None)),
1149 claims: Arc::new(claims::Claims::default()),
1150 config_runtime: None,
1151 config_review_admission: Arc::new(crate::config_gate::MissingReviewAdmission),
1152 config_review_admission_is_default: true,
1153 activity: None,
1154 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1155 catalog_streams: Arc::new(Mutex::new(HashMap::new())),
1156 forwarding,
1157 process_liveness: None,
1158 supervisor: SupervisorHandle::new(),
1159 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1160 env!("CARGO_PKG_VERSION").to_owned(),
1161 )),
1162 agent_declarations: agent_commands::Queues::default(),
1163 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1164 subc_capabilities: Arc::from([
1165 CAP_MANIFEST_REGISTRATION.to_string(),
1166 CAP_CHANNEL_LIFECYCLE.to_string(),
1167 CAP_PING_PONG.to_string(),
1168 CAP_SESSION_ATTACH.to_string(),
1169 CAP_ADMISSION_FACTS_RELAY.to_string(),
1170 CAP_SCOPES_V1.to_string(),
1171 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1172 subc_protocol::peer::CALLER_ATTESTATION_CAPABILITY.to_string(),
1173 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1174 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1175 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1176 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1177 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1178 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1179 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1180 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1181 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1182 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1183 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1184 subc_protocol::ROUTE_CLOSE_REASONS_CAPABILITY.to_string(),
1185 ]),
1186 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1187 route_bind_relay_timeouts: BTreeMap::new(),
1188 route_bind_breakers,
1189 route_bind_concurrency,
1190 route_outages,
1191 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1192 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1193 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1194 storage_config: None,
1195 machine_id: None,
1196 admission_facts_carrier_module_id: None,
1197 admission_facts_targets: None,
1198 scopes: Arc::new(RwLock::new(ScopeTable::new(
1199 crate::daemon_config::default_scope_authority_owners(),
1200 ))),
1201 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1202 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1203 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1204 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1205 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1206 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1207 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1208 captured_peers: Arc::new(Mutex::new(HashMap::new())),
1209 config_desktop_stamps: Arc::new(Mutex::new(HashMap::new())),
1210 local_connections: Arc::new(Mutex::new(HashSet::new())),
1211 operator_app_policy: Default::default(),
1212 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1213 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1214 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1215 #[cfg(test)]
1216 app_open_test_platform: None,
1217 #[cfg(test)]
1218 app_open_test_launch: None,
1219 #[cfg(test)]
1220 recovery_exit_test_after_step: None,
1221 #[cfg(test)]
1222 recovery_exit_test_completed: None,
1223 #[cfg(test)]
1224 apply_reconcile_pause: Arc::new(Mutex::new(None)),
1225 #[cfg(test)]
1226 recovery_exit_test_quiet_retry: false,
1227 #[cfg(test)]
1228 recovery_exit_test_retry_pause: None,
1229 #[cfg(test)]
1230 config_delivery_test_wakes: None,
1231 #[cfg(all(test, unix))]
1232 config_delivery_test_idle: None,
1233 #[cfg(windows)]
1234 windows_verify: None,
1235 #[cfg(windows)]
1236 windows_flash: None,
1237 #[cfg(windows)]
1238 windows_track: None,
1239 #[cfg(all(test, windows))]
1240 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1241 #[cfg(test)]
1242 caller_lookup: None,
1243 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1244 rescan: None,
1245 connected_clients: ConnectedClients::new(),
1246 local_endpoint_unavailable: None,
1247 counters,
1248 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1249 daemon_shutdown: DaemonShutdownFlag::default(),
1250 daemon_provenance: DaemonProvenanceFacts::default(),
1251 #[cfg(test)]
1252 control_dispatch_delay: None,
1253 #[cfg(test)]
1254 provenance_probe_override: None,
1255 #[cfg(test)]
1256 route_bind_relay_sent_hold: None,
1257 #[cfg(test)]
1258 hello_registry_to_forwarding_hold: None,
1259 #[cfg(test)]
1260 hello_ack_to_publication_hold: None,
1261 #[cfg(test)]
1262 hello_before_publication_hold: None,
1263 #[cfg(test)]
1264 hello_after_launch_check_hold: None,
1265 #[cfg(test)]
1266 hello_inside_publication_hold: None,
1267 }
1268 }
1269
1270 pub fn with_storage_config(
1273 mut self,
1274 storage_config: Option<crate::daemon_config::StorageConfig>,
1275 ) -> Self {
1276 self.storage_config = storage_config;
1277 self
1278 }
1279
1280 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1283 self.activity = Some(crate::activity::ActivityLedger::open(
1284 run_dir.join("activity.jsonl"),
1285 incarnation,
1286 ));
1287 self.supervisor.set_activity(self.activity.clone());
1288 self
1289 }
1290
1291 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1294 self.machine_id = machine_id;
1295 self
1296 }
1297
1298 pub fn with_admission_facts_config(
1302 mut self,
1303 carrier_module_id: Option<String>,
1304 targets: Option<Vec<String>>,
1305 ) -> Self {
1306 self.admission_facts_carrier_module_id = carrier_module_id;
1307 self.admission_facts_targets = targets;
1308 self
1309 }
1310
1311 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1315 self.scopes = Arc::new(RwLock::new(
1316 ScopeTable::new(owners.iter().cloned())
1317 .with_federated_owner(self.federated_scope_owner.clone()),
1318 ));
1319 self.scope_authority_owners = owners;
1320 self
1321 }
1322
1323 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1325 self.scopes = Arc::new(RwLock::new(
1326 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1327 .with_federated_owner(owner.clone()),
1328 ));
1329 self.federated_scope_owner = owner;
1330 self
1331 }
1332
1333 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1335 self.privileged_operator_prompt = mode;
1336 self
1337 }
1338
1339 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1342 self.route_bind_relay_timeout = timeout;
1343 self
1344 }
1345
1346 pub fn with_route_bind_relay_timeouts(
1352 mut self,
1353 timeouts: impl IntoIterator<Item = (String, Duration)>,
1354 ) -> Self {
1355 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1356 self
1357 }
1358
1359 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1365 self.route_bind_relay_timeouts
1366 .get(module_id)
1367 .copied()
1368 .unwrap_or(self.route_bind_relay_timeout)
1369 }
1370
1371 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1379 self.route_bind_breaker_threshold = threshold.max(1);
1380 self.route_bind_breaker_cooldown = cooldown;
1381 self
1382 }
1383
1384 #[cfg(test)]
1385 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1386 self.health_probe_timeout = timeout;
1387 self
1388 }
1389
1390 #[cfg(test)]
1391 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1392 self.control_dispatch_delay = Some(delay);
1393 self
1394 }
1395
1396 #[cfg(test)]
1400 pub(crate) fn with_route_bind_relay_sent_hold(
1401 mut self,
1402 hold: Arc<tokio::sync::Barrier>,
1403 ) -> Self {
1404 self.route_bind_relay_sent_hold = Some(hold);
1405 self
1406 }
1407
1408 #[cfg(test)]
1413 pub(crate) fn with_hello_registry_to_forwarding_hold(
1414 mut self,
1415 hold: Arc<std::sync::Barrier>,
1416 ) -> Self {
1417 self.hello_registry_to_forwarding_hold = Some(hold);
1418 self
1419 }
1420
1421 #[cfg(test)]
1422 fn hold_between_registry_and_forwarding(&self) {
1423 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1424 hold.wait();
1425 hold.wait();
1426 }
1427 }
1428
1429 #[cfg(test)]
1432 fn hold_between_ack_and_publication(&self) {
1433 if let Some(hold) = &self.hello_ack_to_publication_hold {
1434 hold.wait();
1435 hold.wait();
1436 }
1437 }
1438
1439 #[cfg(all(test, unix))]
1440 pub(crate) fn with_hello_before_publication_hold(
1441 mut self,
1442 hold: Arc<std::sync::Barrier>,
1443 ) -> Self {
1444 self.hello_before_publication_hold = Some(hold);
1445 self
1446 }
1447
1448 #[cfg(all(test, unix))]
1449 pub(crate) fn with_hello_after_launch_check_hold(
1450 mut self,
1451 hold: Arc<std::sync::Barrier>,
1452 ) -> Self {
1453 self.hello_after_launch_check_hold = Some(hold);
1454 self
1455 }
1456
1457 #[cfg(all(test, unix))]
1458 pub(crate) fn with_hello_inside_publication_hold(
1459 mut self,
1460 hold: Arc<std::sync::Barrier>,
1461 ) -> Self {
1462 self.hello_inside_publication_hold = Some(hold);
1463 self
1464 }
1465
1466 pub fn with_process_liveness(
1467 mut self,
1468 process_liveness: Arc<dyn ModuleProcessLiveness>,
1469 ) -> Self {
1470 self.process_liveness = Some(process_liveness);
1471 self
1472 }
1473
1474 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1475 self.supervisor = supervisor;
1476 if let Some(owner) = &self.config_runtime {
1477 self.supervisor.install_binding_retirement(owner);
1478 }
1479 self.supervisor.set_activity(self.activity.clone());
1480 self
1481 }
1482
1483 pub fn with_daemon_provenance(
1484 mut self,
1485 pid: u32,
1486 started_at_ms: u64,
1487 executable_path: Option<PathBuf>,
1488 build_git_sha: Option<String>,
1489 build_lock_digest: Option<String>,
1490 ) -> Self {
1491 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1492 let process_start_time = process_start_time(pid);
1493 self.daemon_provenance = DaemonProvenanceFacts {
1494 build: DaemonBuildProvenance {
1495 build_git_sha,
1496 build_lock_digest,
1497 },
1498 pid: Some(pid),
1499 started_at_ms: Some(started_at_ms),
1500 start_clock: None,
1501 executable_path,
1502 executable_identity,
1503 process_start_time,
1504 probe: ExecutableIdentityProbe::default(),
1505 };
1506 self
1507 }
1508
1509 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1510 self.daemon_provenance.start_clock = Some(clock);
1511 self
1512 }
1513
1514 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1516 self.scopes
1517 .write()
1518 .unwrap_or_else(|poisoned| poisoned.into_inner())
1519 .set_wall_clock(Arc::new(clock));
1520 self
1521 }
1522
1523 #[cfg(test)]
1524 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1525 self.provenance_probe_override = Some(result);
1526 self
1527 }
1528
1529 pub fn with_capability_config(
1533 self,
1534 modules: impl IntoIterator<Item = (String, bool)>,
1535 reserved_capabilities: BTreeMap<String, String>,
1536 ) -> Self {
1537 self.capability_evaluator
1538 .configure(modules, reserved_capabilities);
1539 self
1540 }
1541
1542 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1546 self.daemon_shutdown = flag;
1547 self
1548 }
1549
1550 pub fn with_supervisor_rescan(
1551 mut self,
1552 supervisor: Supervisor,
1553 config_path: impl Into<PathBuf>,
1554 configured_port: Option<u16>,
1555 ) -> Self {
1556 self.rescan = Some(SupervisorRescanContext {
1557 supervisor,
1558 config_path: config_path.into(),
1559 configured_port,
1560 storage_config: self.storage_config.clone(),
1561 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1562 admission_facts_targets: self.admission_facts_targets.clone(),
1563 scope_authority_owners: self.scope_authority_owners.clone(),
1564 federated_scope_owner: self.federated_scope_owner.clone(),
1565 privileged_operator_prompt: self.privileged_operator_prompt,
1566 generation: Arc::default(),
1567 module_entries: Arc::default(),
1568 });
1569 self
1570 }
1571
1572 pub(crate) fn with_supervisor_rescan_entries(
1573 self,
1574 modules: &[crate::daemon_config::ConfiguredModule],
1575 ) -> Self {
1576 if let Some(context) = &self.rescan {
1577 let mut entries = context
1578 .module_entries
1579 .lock()
1580 .unwrap_or_else(|e| e.into_inner());
1581 entries.seed(modules);
1582 }
1583 self
1584 }
1585
1586 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1587 self.connected_clients = connected_clients;
1588 self
1589 }
1590
1591 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1594 self.local_endpoint_unavailable = reason;
1595 self
1596 }
1597
1598 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1599 Arc::clone(&self.forwarding)
1600 }
1601
1602 pub(crate) fn counters(&self) -> DaemonCounters {
1603 self.counters.clone()
1604 }
1605
1606 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1609 let handler = Arc::downgrade(&self);
1610 let evaluator = self.capability_evaluator.clone();
1611 tokio::spawn(async move {
1612 loop {
1613 evaluator.wait_for_change_or_deadline().await;
1614 let Some(handler) = handler.upgrade() else {
1615 break;
1616 };
1617 handler.refresh_capability_requirements();
1618 }
1619 });
1620 }
1621
1622 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1628 let context = scope_expiry::ScopeExpiryContext::new(&self);
1629 let deadline_added = self.scope_deadline_added.clone();
1630 tokio::spawn(async move {
1631 loop {
1632 if !context.has_deadlines() {
1633 deadline_added.notified().await;
1634 continue;
1635 }
1636 tokio::time::sleep(Duration::from_secs(1)).await;
1637 if let Err(error) = context.sweep() {
1638 warn!(%error, "scope expiry sweep failed");
1639 }
1640 }
1641 });
1642 }
1643
1644 #[cfg(test)]
1645 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1646 scope_expiry::ScopeExpiryContext::new(self).sweep()
1647 }
1648
1649 fn publish_scope_changes(
1650 &self,
1651 changes: &[crate::scopes::ScopeTagChange],
1652 expired: &[crate::scopes::ScopeExpired],
1653 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1654 scope_expiry::ScopeExpiryContext::new(self).publish_scope_changes(changes, expired)
1655 }
1656
1657 fn runtime_capability_snapshot(
1658 &self,
1659 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1660 let runtime = self
1661 .supervisor
1662 .list()
1663 .into_iter()
1664 .map(|module| {
1665 let status = module.status().map_err(|err| {
1666 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1667 })?;
1668 Ok(RuntimeModule {
1669 module_id: status.module_id,
1670 state: status.state,
1671 enabled: status.enabled,
1672 })
1673 })
1674 .collect::<Result<Vec<_>, RouterError>>()?;
1675 let registrations = self.registry.list_registered_modules().map_err(|err| {
1681 RouterError::backend(
1682 0,
1683 0,
1684 format!("failed to list capability registrations: {err}"),
1685 )
1686 })?;
1687 let registrations = registrations
1688 .into_iter()
1689 .map(|registration| RegisteredModule {
1690 module_id: registration.manifest.module_id,
1691 module_version: registration.manifest.module_version,
1692 capabilities: registration.manifest.capabilities,
1693 })
1694 .collect();
1695 Ok((runtime, registrations))
1696 }
1697
1698 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1705 let cached_registration = RegisteredModule {
1706 module_id: registration.manifest.module_id.clone(),
1707 module_version: registration.manifest.module_version.clone(),
1708 capabilities: registration.manifest.capabilities.clone(),
1709 };
1710 let change = self.capability_evaluator.record_hello(&cached_registration);
1711 log_manifest_change(&cached_registration.module_id, &change);
1712 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1713 self.enforce_capability_denies();
1714 }
1715 self.refresh_capability_requirements();
1716 }
1717
1718 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1722 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1723 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1724 self.supervisor.set_swap_promotion_observer(observer);
1725 }
1726
1727 pub fn refresh_capability_requirements(&self) {
1728 match self.runtime_capability_snapshot() {
1729 Ok((runtime, registrations)) => {
1730 log_requirement_events(
1731 self.capability_evaluator
1732 .evaluate_now(&runtime, ®istrations),
1733 &self.daemon_shutdown,
1734 );
1735 }
1736 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1737 }
1738 }
1739
1740 fn enforce_capability_denies(&self) {
1744 let registrations = match self.registry.list_registered_modules() {
1745 Ok(snapshot) => snapshot,
1746 Err(err) => {
1747 warn!(error = %err, "failed to read registrations for capability deny census");
1748 return;
1749 }
1750 };
1751 let manifests = registrations
1752 .into_iter()
1753 .map(|registration| {
1754 (
1755 registration.manifest.module_id.clone(),
1756 registration.manifest,
1757 )
1758 })
1759 .collect::<BTreeMap<_, _>>();
1760 let census = match self.forwarding.route_census(None) {
1761 Ok(census) => census,
1762 Err(err) => {
1763 warn!(error = %err, "failed to read route census for capability deny enforcement");
1764 return;
1765 }
1766 };
1767
1768 for (target_module_id, routes) in census {
1769 let Some(target_manifest) = manifests.get(&target_module_id) else {
1770 continue;
1771 };
1772 let mut closed_routes = Vec::new();
1773 let mut goodbyes = Vec::new();
1774 for route in routes {
1775 let Principal::Reserved {
1776 module_id: opening_module_id,
1777 } = &route.principal
1778 else {
1779 continue;
1780 };
1781 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1782 continue;
1783 };
1784 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1785 continue;
1786 };
1787
1788 match self.forwarding.release_client_route(
1789 route.goodbye_target.connection_id,
1790 route.goodbye_target.channel,
1791 route.goodbye_target.epoch,
1792 ) {
1793 Ok(RouteRelease::Removed(module_goodbye)) => {
1794 warn!(
1795 opening_module_id,
1796 target_module_id,
1797 capability,
1798 "force-closing route because an attested capability deny edge now matches"
1799 );
1800 goodbyes.push(module_goodbye);
1801 goodbyes.push(route.goodbye_target.clone());
1802 closed_routes.push(route);
1803 }
1804 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1805 Err(err) => warn!(
1806 opening_module_id,
1807 target_module_id,
1808 capability,
1809 error = %err,
1810 "failed to force-close capability-denied route"
1811 ),
1812 }
1813 }
1814
1815 if closed_routes.is_empty() {
1816 continue;
1817 }
1818 send_route_control_pushes(
1819 &self.forwarding,
1820 closed_routes,
1821 ClientControlPush::RouteClosed {
1822 module_id: target_module_id,
1823 channels: Vec::new(),
1824 reason: RouteCloseReason::CapabilityDenied,
1825 drained: false,
1826 abandoned: 0,
1827 excluded_subscriptions: 0,
1828 terminal: Some(false),
1829 },
1830 );
1831 self.emit_route_goodbyes(goodbyes);
1834 }
1835 }
1836
1837 fn not_ready_reason(
1843 &self,
1844 registration: &crate::registry::ModuleRegistration,
1845 ) -> Option<NotReadyReason> {
1846 if !registration.ready {
1847 return Some(NotReadyReason {
1848 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1849 capability: None,
1850 });
1851 }
1852 self.first_unprovided_required_capability(registration)
1853 .map(|capability| NotReadyReason {
1854 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1855 capability: Some(capability),
1856 })
1857 }
1858
1859 fn first_unprovided_required_capability(
1877 &self,
1878 registration: &crate::registry::ModuleRegistration,
1879 ) -> Option<String> {
1880 let required = registration
1881 .manifest
1882 .capabilities
1883 .iter()
1884 .flat_map(|declarations| declarations.requires.iter())
1885 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1886 .map(|requirement| requirement.capability.as_str())
1887 .collect::<BTreeSet<_>>();
1888 if required.is_empty() {
1889 return None;
1890 }
1891 let module_id = registration.manifest.module_id.as_str();
1892 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1893 if required
1894 .iter()
1895 .any(|capability| verdict(capability).is_none())
1896 {
1897 self.refresh_capability_requirements();
1898 }
1899 required
1900 .into_iter()
1901 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1902 .map(str::to_string)
1903 }
1904
1905 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1906 self.capability_evaluator
1907 .statuses()
1908 .into_iter()
1909 .map(capability_requirement_status)
1910 .collect()
1911 }
1912
1913 fn deregister_connection(
1920 &self,
1921 connection_id: ConnectionId,
1922 reason: RegistrationEndReason,
1923 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1924 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
1925 spawn.end_primary_registration();
1926 }
1927 self.registry
1928 .deregister_connection_with_reason(connection_id, reason)
1929 }
1930
1931 fn rollback_unpublished_hello(
1936 &self,
1937 connection: ConnectionId,
1938 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1939 self.registry.deregister_connection_with_reason(
1940 connection,
1941 RegistrationEndReason::RegistrationFailed,
1942 )
1943 }
1944
1945 fn request_refusal_close(
1946 &self,
1947 connection: ConnectionId,
1948 sink: Option<&crate::FrameSink>,
1949 reason: CloseReason,
1950 ) {
1951 if let Some(sink) = sink {
1952 sink.terminate_ingress();
1953 }
1954 if self.claims.close_notifications_enabled() {
1955 self.forwarding.request_connection_close(connection, reason);
1956 }
1957 }
1958
1959 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1960 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1961 return None;
1962 }
1963 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1964 parse_client_control_request(&frame.body)
1965 else {
1966 return None;
1967 };
1968 Some(target_module_id(&target).to_string())
1969 }
1970
1971 pub(crate) fn route_open_capacity_refusal(
1972 &self,
1973 ctx: &RouteCtx,
1974 frame: &Frame,
1975 target_module_id: &str,
1976 in_flight: usize,
1977 limit: usize,
1978 ) -> Result<Frame, RouterError> {
1979 let caller = self.route_open_caller_from_frame(ctx, frame);
1982 self.route_open_admission_refusal_frame(
1983 ctx,
1984 &caller,
1985 frame,
1986 target_module_id,
1987 "open_admission_full",
1988 (in_flight, limit),
1989 format!(
1990 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1991 ),
1992 )
1993 }
1994
1995 fn route_open_target_capacity_refusal(
1996 &self,
1997 ctx: &RouteCtx,
1998 caller: &RouteOpenCaller,
1999 frame: &Frame,
2000 target_module_id: &str,
2001 in_flight: usize,
2002 ) -> Result<Frame, RouterError> {
2003 self.route_open_admission_refusal_frame(
2004 ctx,
2005 caller,
2006 frame,
2007 target_module_id,
2008 "target_binds_full",
2009 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
2010 format!(
2011 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2012 ),
2013 )
2014 }
2015
2016 #[allow(clippy::too_many_arguments)]
2024 fn route_open_admission_refusal_frame(
2025 &self,
2026 ctx: &RouteCtx,
2027 caller: &RouteOpenCaller,
2028 frame: &Frame,
2029 target_module_id: &str,
2030 reason: &'static str,
2031 (in_flight, limit): (usize, usize),
2032 message: impl Into<String>,
2033 ) -> Result<Frame, RouterError> {
2034 let code = error_codes::TARGET_UNAVAILABLE;
2035 self.counters.increment_route_open_refused(code);
2036 info!(
2037 target: "control",
2038 code,
2039 reason,
2040 module_id = ?target_module_id,
2041 connection_id = ctx.connection_id.get(),
2042 principal = %caller.principal,
2043 scope = %caller.scope,
2044 harness = %caller.harness,
2045 in_flight,
2046 limit,
2047 "route.open refused"
2048 );
2049 control_error_frame(frame, code, message.into())
2050 }
2051
2052 #[cfg(test)]
2058 pub fn handle_control(
2059 &self,
2060 connection_id: ConnectionId,
2061 frame: Frame,
2062 ) -> Result<Vec<Frame>, RouterError> {
2063 match frame.header.ty {
2064 FrameType::Ping => Ok(vec![pong(&frame)?]),
2065 FrameType::Hello => self.handle_hello(connection_id, None, frame),
2066 FrameType::Goodbye => self.handle_goodbye(connection_id),
2067 ty => Ok(vec![control_error_frame(
2068 &frame,
2069 "unsupported_control_frame",
2070 format!("unsupported channel-0 frame {ty:?}"),
2071 )?]),
2072 }
2073 }
2074
2075 pub async fn handle_control_frame(
2076 &self,
2077 ctx: &RouteCtx,
2078 frame: Frame,
2079 ) -> Result<Vec<Frame>, RouterError> {
2080 self.handle_control_frame_timed(ctx, frame, None).await
2081 }
2082
2083 #[cfg(any(test, feature = "test-support"))]
2084 pub fn with_test_operator_app_policy(
2085 mut self,
2086 policy: crate::operator_apps::TestOperatorAppPolicy,
2087 ) -> Self {
2088 self.operator_app_policy = policy.into();
2089 self
2090 }
2091
2092 #[cfg(all(windows, any(test, feature = "test-support")))]
2096 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
2097 self.windows_verify = Some(verify);
2098 self
2099 }
2100
2101 #[cfg(all(windows, any(test, feature = "test-support")))]
2103 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
2104 self.windows_flash = Some(flash);
2105 self
2106 }
2107
2108 #[cfg(all(windows, any(test, feature = "test-support")))]
2112 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
2113 self.windows_track = Some(track);
2114 self
2115 }
2116
2117 pub(crate) fn record_captured_peer(
2118 &self,
2119 connection: ConnectionId,
2120 peer: subc_os::local_peer::CapturedPeer,
2121 ) {
2122 self.record_observed_peer(connection, peer.observation().clone());
2123 let identity = self.operator_app_policy.config_identity(peer.clone());
2124 #[cfg(windows)]
2125 let identity = identity.or_else(|| self.config_windows_launch_identity(&peer));
2126 if let Some(identity) = identity {
2127 self.config_desktop_stamps
2130 .lock()
2131 .unwrap_or_else(|p| p.into_inner())
2132 .insert(connection, identity);
2133 }
2134 self.captured_peers
2135 .lock()
2136 .unwrap_or_else(|p| p.into_inner())
2137 .insert(connection, peer.clone());
2138 #[cfg(windows)]
2139 if let Some(stamp) = self.stamp_windows_launch(&peer) {
2140 self.operator_apps
2141 .lock()
2142 .unwrap_or_else(|p| p.into_inner())
2143 .insert(connection, stamp);
2144 }
2145 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
2146 self.operator_apps
2147 .lock()
2148 .unwrap_or_else(|p| p.into_inner())
2149 .insert(connection, stamp);
2150 }
2151 }
2152
2153 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
2154 self.local_connections
2155 .lock()
2156 .unwrap_or_else(|p| p.into_inner())
2157 .insert(connection);
2158 }
2159
2160 fn connection_operator_app(
2163 &self,
2164 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
2165 ) -> Option<subc_protocol::peer::OperatorApp> {
2166 stamp
2167 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
2168 .map(|stamp| stamp.app.clone())
2169 }
2170
2171 fn handle_connection_describe(
2172 &self,
2173 ctx: &RouteCtx,
2174 frame: Frame,
2175 ) -> Result<Vec<Frame>, RouterError> {
2176 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
2177 return Ok(vec![control_error_frame(
2178 &frame,
2179 "invalid_control_body",
2180 "malformed connection.describe body",
2181 )?]);
2182 }
2183 let local = self
2184 .local_connections
2185 .lock()
2186 .unwrap_or_else(|p| p.into_inner())
2187 .contains(&ctx.connection_id);
2188 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
2189 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
2190 Ok(vec![control_response_body_frame(
2191 &frame,
2192 &subc_control::ConnectionDescribeReply::new(local, operator_app),
2193 ops::CONNECTION_DESCRIBE,
2194 )?])
2195 }
2196
2197 pub(crate) fn record_observed_peer(
2198 &self,
2199 connection: ConnectionId,
2200 peer: subc_os::local_peer::ObservedPeer,
2201 ) {
2202 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
2203 let peer = match peer {
2204 OsPeer::Unobserved => ObservedPeer::unobserved(),
2205 OsPeer::Observed(peer) => {
2206 let observed = match peer.user {
2207 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
2208 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
2209 };
2210 observed
2211 .with_exe(
2212 peer.exe
2213 .and_then(|path| path.into_os_string().into_string().ok()),
2214 )
2215 .with_signing_id(peer.signing_id)
2216 }
2217 };
2218 self.observed_peers
2219 .lock()
2220 .unwrap_or_else(|p| p.into_inner())
2221 .insert(connection, peer);
2222 }
2223
2224 pub(crate) fn record_caller_endpoints(
2225 &self,
2226 connection: ConnectionId,
2227 endpoints: Option<(SocketAddr, SocketAddr)>,
2228 ) {
2229 let mut callers = self
2230 .caller_endpoints
2231 .lock()
2232 .unwrap_or_else(|p| p.into_inner());
2233 callers.remove(&connection);
2234 if let Some(endpoints) = endpoints {
2235 callers.insert(connection, endpoints);
2236 }
2237 }
2238
2239 #[cfg(test)]
2240 pub(crate) fn with_caller_lookup_for_test(
2241 mut self,
2242 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2243 ) -> Self {
2244 self.caller_lookup = Some(Arc::new(lookup));
2245 self
2246 }
2247
2248 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2249 let endpoints = self
2250 .caller_endpoints
2251 .lock()
2252 .unwrap_or_else(|p| p.into_inner())
2253 .get(&connection)
2254 .copied();
2255 #[cfg(test)]
2256 let lookup = self.caller_lookup.clone();
2257 tokio::task::spawn_blocking(move || {
2258 #[cfg(test)]
2259 if let Some(lookup) = lookup {
2260 return lookup(endpoints);
2261 }
2262 endpoints
2263 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2264 .unwrap_or_default()
2265 })
2266 .await
2267 .unwrap_or_default()
2268 }
2269
2270 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2271 frame.header.channel == 0
2272 && frame.header.ty == FrameType::Request
2273 && self
2274 .forwarding
2275 .module_endpoint_for_connection(ctx.connection_id)
2276 .is_ok_and(|endpoint| endpoint.is_none())
2277 && parse_client_control_request(&frame.body)
2278 .ok()
2279 .is_some_and(|request| privileged_operation(&request).is_some())
2280 }
2281
2282 pub(crate) async fn handle_control_frame_timed(
2283 &self,
2284 ctx: &RouteCtx,
2285 frame: Frame,
2286 dispatch_started_at: Option<StdInstant>,
2287 ) -> Result<Vec<Frame>, RouterError> {
2288 let first = ctx.egress.first_ingress_frame();
2289 self.handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
2290 .await
2291 }
2292
2293 pub(crate) async fn handle_control_frame_ingress(
2294 &self,
2295 ctx: &RouteCtx,
2296 frame: Frame,
2297 dispatch_started_at: Option<StdInstant>,
2298 first: bool,
2299 ) -> Result<Vec<Frame>, RouterError> {
2300 if ctx.egress.ingress_terminal() {
2301 return Ok(Vec::new());
2302 }
2303 #[cfg(test)]
2304 self.claims.record_dispatch(frame.header.corr);
2305 let claim_binding = self.claims.binding(ctx.connection_id);
2306 let mut prefix_attempt = (frame.header.ty == FrameType::Hello
2307 && claim_binding
2308 .as_ref()
2309 .is_some_and(|binding| binding.prefix_module_id().is_some()))
2310 .then(|| self.prefix_hello_attempt(ctx));
2311 if claim_binding
2312 .as_ref()
2313 .is_some_and(|binding| binding.prefix_module_id().is_some())
2314 {
2315 let registered = self
2316 .forwarding
2317 .module_endpoint_for_connection(ctx.connection_id)
2318 .map_err(RouterError::Forwarding)?
2319 .is_some();
2320 if !registered && frame.header.ty != FrameType::Hello {
2321 return self
2322 .refuse_claim_connection(ctx, &frame, "claim_hello_required")
2323 .await;
2324 }
2325 if registered && frame.header.ty == FrameType::Goodbye {
2326 ctx.egress.terminate_ingress();
2327 let result = self.handle_goodbye(ctx.connection_id);
2328 self.claims.connection_closed(ctx.connection_id);
2329 self.request_refusal_close(
2330 ctx.connection_id,
2331 Some(&ctx.egress),
2332 CloseReason::new("claim_registration_ended", "prefix registration ended"),
2333 );
2334 return result;
2335 }
2336 if registered && frame.header.ty == FrameType::Request {
2337 let allowed =
2338 serde_json::from_slice::<ControlOpProbe>(&frame.body).is_ok_and(|probe| {
2339 matches!(
2340 probe.op.as_str(),
2341 "catalog.update"
2342 | "catalog.update_role_declarations"
2343 | "scope.describe"
2344 | "supervisor.live_roots"
2345 | "activity.record"
2346 )
2347 });
2348 if !allowed {
2349 return Ok(vec![control_error_frame(
2350 &frame,
2351 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2352 "prefix registrations have only brokered module operations",
2353 )?]);
2354 }
2355 }
2356 if registered
2357 && frame.header.ty == FrameType::Push
2358 && !serde_json::from_slice::<ControlOpProbe>(&frame.body)
2359 .is_ok_and(|probe| probe.op == "route.status")
2360 {
2361 return Ok(vec![control_error_frame(
2362 &frame,
2363 channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2364 "prefix registrations may push only route.status for routes they serve",
2365 )?]);
2366 }
2367 }
2368 match frame.header.ty {
2369 FrameType::Ping => Ok(vec![pong(&frame)?]),
2370 FrameType::Hello => {
2371 if claim_binding
2372 .as_ref()
2373 .is_some_and(|binding| binding.prefix_module_id().is_none())
2374 {
2375 return self
2376 .refuse_claim_connection(ctx, &frame, "claim_hello_not_allowed")
2377 .await;
2378 }
2379 let channel_spawn = self.supervisor.registration_spawn(ctx.connection_id);
2383 if let Some(spawn) = &channel_spawn {
2384 spawn.wait_launch().await;
2385 if !spawn.launch_ready() {
2388 ctx.egress.terminate_ingress();
2389 return Ok(Vec::new());
2390 }
2391 #[cfg(test)]
2392 self.pause_hello_ingress_for_test().await;
2393 } else if claim_binding.is_none() {
2394 if let Ok(hello) = serde_json::from_slice::<serde_json::Value>(&frame.body) {
2395 if let (Some(module), Some(nonce)) = (
2396 hello
2397 .get("manifest")
2398 .and_then(|value| value.get("module_id"))
2399 .and_then(|value| value.as_str()),
2400 hello.get("launch_nonce").and_then(|value| value.as_str()),
2401 ) {
2402 if !self
2403 .config_runtime
2404 .as_ref()
2405 .is_some_and(|owner| owner.module_is_gated(module))
2406 && !self.supervisor.wait_program_binding(module, nonce).await
2407 {
2408 self.request_refusal_close(
2409 ctx.connection_id,
2410 Some(&ctx.egress),
2411 CloseReason::new(
2412 "program_refused",
2413 "the spawned program binding failed",
2414 ),
2415 );
2416 return Ok(vec![control_error_frame(
2417 &frame,
2418 if self
2419 .config_runtime
2420 .as_ref()
2421 .is_some_and(|owner| owner.is_recovering())
2422 {
2423 "config_gated"
2424 } else {
2425 "program_refused"
2426 },
2427 "the spawned program binding failed",
2428 )?]);
2429 }
2430 }
2431 }
2432 }
2433 let handler = self.clone();
2434 let connection = ctx.connection_id;
2435 let egress = ctx.egress.clone();
2436 let correlation = frame.header.corr;
2437 let dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
2438 let span = tracing::Span::current();
2439 let hello_binding = claim_binding.clone();
2440 let replies = tokio::task::spawn_blocking(move || {
2441 tracing::dispatcher::with_default(&dispatcher, || {
2442 span.in_scope(|| {
2443 handler.handle_hello_checked(
2444 connection,
2445 Some(egress),
2446 frame,
2447 true,
2448 hello_binding,
2449 channel_spawn,
2450 )
2451 })
2452 })
2453 })
2454 .await
2455 .map_err(|error| RouterError::backend(0, correlation, error.to_string()))??;
2456 #[cfg(test)]
2457 if prefix_attempt.is_some() {
2458 self.claims.pause_prefix_commit_for_test(false).await;
2459 }
2460 if let Some(attempt) = &mut prefix_attempt {
2461 let accepted = !replies
2462 .iter()
2463 .any(|reply| reply.header.ty == FrameType::Error)
2464 && self
2465 .registry
2466 .get_module_by_connection(ctx.connection_id)
2467 .map_err(|err| RouterError::backend(0, correlation, err.to_string()))?
2468 .is_some();
2469 if accepted {
2470 attempt.accepted = true;
2471 } else {
2472 ctx.egress.terminate_ingress();
2473 for reply in replies {
2474 ctx.egress.send_flushed(reply).await?;
2475 }
2476 return Ok(Vec::new());
2477 }
2478 }
2479 Ok(replies)
2480 }
2481 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2482 FrameType::Cancel => {
2483 if self
2489 .agent_declarations
2490 .cancel(ctx.connection_id, frame.header.corr)
2491 || self
2492 .forwarding
2493 .operator_confirms()
2494 .cancel(ctx.connection_id, frame.header.corr)
2495 || self.cancel_config_approval(ctx.connection_id, frame.header.corr)
2496 || self
2497 .supervisor
2498 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2499 || self.cancel_activity_subscription(ctx, &frame)?
2500 || self.cancel_catalog_subscription(ctx.connection_id, frame.header.corr)
2501 {
2502 Ok(Vec::new())
2503 } else {
2504 Ok(vec![control_error_frame(
2505 &frame,
2506 "unknown_subscription",
2507 "no subscription has this correlation id",
2508 )?])
2509 }
2510 }
2511 FrameType::Request => {
2512 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2515 let claim_present = op.as_ref().is_some_and(|probe| probe.op == "claim.present");
2516 if let Some(refusal) = self
2517 .recovery_request_refusal(
2518 ctx,
2519 &frame,
2520 op.as_ref().map(|probe| probe.op.as_str()),
2521 )
2522 .await?
2523 {
2524 if claim_present {
2525 self.reclaim_presented_expiry(&frame);
2526 return self
2527 .refuse_claim_connection(ctx, &frame, "config_gated")
2528 .await;
2529 }
2530 return Ok(vec![refusal]);
2531 }
2532 if claim_present {
2533 return self.handle_claim_present(ctx, &frame, first).await;
2534 }
2535 if op.as_ref().is_some_and(|probe| probe.op == "claim.mint") {
2536 return self.handle_claim_mint(ctx, &frame).await;
2537 }
2538 if let Some(refusal) = self.registration_operation_refusal(
2539 ctx,
2540 &frame,
2541 op.as_ref().map(|probe| probe.op.as_str()),
2542 )? {
2543 return Ok(vec![refusal]);
2544 }
2545 if op
2546 .as_ref()
2547 .is_some_and(|probe| probe.op == ops::CATALOG_WATCH)
2548 {
2549 return self.handle_catalog_watch(ctx, frame);
2550 }
2551 if op
2552 .as_ref()
2553 .is_some_and(|probe| probe.op == MODULE_TO_SUBC_OP_CATALOG_UPDATE)
2554 && subc_protocol::provider_declarations::catalog_update_has_provider_declarations(&frame.body)
2555 .unwrap_or(false)
2556 {
2557 return Ok(vec![control_error_frame(
2558 &frame,
2559 error_codes::CATALOG_UPDATE_FROZEN_DECLARATIONS,
2560 "provider_declarations are fixed at HELLO; restart the module to change them",
2561 )?]);
2562 }
2563 if let Some(probe) = op
2564 .as_ref()
2565 .filter(|probe| config_gate::is_module_config_op(&probe.op))
2566 {
2567 return self
2568 .handle_module_config_request(ctx, frame, &probe.op)
2569 .await;
2570 }
2571 if op
2572 .as_ref()
2573 .is_some_and(|probe| config_gate::is_config_op(&probe.op))
2574 {
2575 return self.handle_config_gate_request(ctx, frame).await;
2576 }
2577 if op
2578 .as_ref()
2579 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2580 {
2581 return self.handle_activity_record(ctx, frame).await;
2582 }
2583 if op.as_ref().is_some_and(|probe| {
2584 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2585 }) {
2586 return self.handle_activity_read(ctx, frame).await;
2587 }
2588 if op
2589 .as_ref()
2590 .is_some_and(|probe| probe.op == "operator.confirm")
2591 {
2592 return self.handle_operator_confirm(ctx, frame);
2593 }
2594 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2595 return self
2596 .observe_control_wait(
2597 ctx,
2598 None,
2599 ops::APP_OPEN,
2600 self.handle_app_open(ctx, frame),
2601 )
2602 .await;
2603 }
2604 if op
2605 .as_ref()
2606 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2607 {
2608 return self.handle_connection_describe(ctx, frame);
2609 }
2610 if op
2611 .as_ref()
2612 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2613 {
2614 return self.handle_scope_served_by(ctx.connection_id, frame);
2615 }
2616 if self
2617 .forwarding
2618 .module_endpoint_for_connection(ctx.connection_id)
2619 .map_err(RouterError::Forwarding)?
2620 .is_some()
2621 {
2622 if !is_known_module_request_op(&frame.body) {
2623 return Ok(vec![control_error_frame(
2624 &frame,
2625 "unsupported_control_frame",
2626 "module-originated channel-0 REQUEST is not supported",
2627 )?]);
2628 }
2629 let request = match parse_module_control_request_from_module(&frame.body) {
2630 Ok(request) => request,
2631 Err((err, ControlRequestBodyError::UnknownOp)) => {
2632 return Ok(vec![control_error_frame(
2633 &frame,
2634 "unsupported_control_frame",
2635 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2636 )?])
2637 }
2638 Err((err, ControlRequestBodyError::InvalidBody)) => {
2639 return Ok(vec![control_error_frame(
2640 &frame,
2641 "invalid_control_body",
2642 format!("malformed module control body: {err}"),
2643 )?])
2644 }
2645 };
2646 let op = module_control_request_op(&request);
2647 let corr = frame.header.corr;
2648 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2649 if let ModuleControlRequestFromModule::AgentCommands {
2650 started,
2651 finished,
2652 known,
2653 } = request
2654 {
2655 return self
2656 .agent_declarations
2657 .enqueue(
2658 self,
2659 ctx,
2660 frame,
2661 crate::resource_totals::Declaration {
2662 started,
2663 finished,
2664 known,
2665 },
2666 )
2667 .await;
2668 }
2669 let result =
2670 self.handle_module_control_request(ctx.connection_id, frame, request);
2671 log_slow_control_dispatch(
2672 dispatch_started_at,
2673 op,
2674 ctx.connection_id,
2675 corr,
2676 None,
2677 );
2678 return result;
2679 }
2680
2681 if is_known_module_request_op(&frame.body) {
2682 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2683 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2684 let code = if is_agents {
2685 "not_supervised"
2686 } else if op
2687 .as_ref()
2688 .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2689 {
2690 error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2691 } else {
2692 "not_registered"
2693 };
2694 return Ok(vec![control_error_frame(
2695 &frame,
2696 code,
2697 "module-originated control requests require the module's own registered connection",
2698 )?]);
2699 }
2700
2701 let request = match parse_client_control_request(&frame.body) {
2702 Ok(request) => request,
2703 Err((err, ControlRequestBodyError::UnknownOp)) => {
2704 return Ok(vec![control_error_frame(
2705 &frame,
2706 "unknown_control_op",
2707 format!("unknown client control op: {err}"),
2708 )?])
2709 }
2710 Err((err, ControlRequestBodyError::InvalidBody)) => {
2711 return Ok(vec![control_error_frame(
2712 &frame,
2713 "invalid_control_body",
2714 format!("malformed client control body: {err}"),
2715 )?])
2716 }
2717 };
2718 let op = client_control_request_op(&request);
2719 let corr = frame.header.corr;
2720 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2721 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2722 #[cfg(test)]
2723 if let Some(delay) = self.control_dispatch_delay {
2724 tokio::time::sleep(delay).await;
2725 }
2726 let result = self
2727 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2728 .await;
2729 if let Some(timings) = route_open_timings.as_mut() {
2730 timings.finish();
2731 }
2732 log_slow_control_dispatch(
2733 dispatch_started_at,
2734 op,
2735 ctx.connection_id,
2736 corr,
2737 route_open_timings.as_ref(),
2738 );
2739 result
2740 }
2741 FrameType::Push => {
2742 let Some(endpoint) = self
2743 .forwarding
2744 .module_endpoint_for_connection(ctx.connection_id)
2745 .map_err(RouterError::Forwarding)?
2746 else {
2747 return Ok(vec![control_error_frame(
2748 &frame,
2749 "unsupported_control_frame",
2750 "client-originated channel-0 PUSH is not supported",
2751 )?]);
2752 };
2753 self.handle_status_update(endpoint, frame)
2754 }
2755 FrameType::Response | FrameType::Error
2756 if self
2757 .forwarding
2758 .module_endpoint_for_connection(ctx.connection_id)
2759 .map_err(RouterError::Forwarding)?
2760 .is_some() =>
2761 {
2762 self.handle_module_relay_response(ctx.connection_id, frame)
2763 }
2764 ty => Ok(vec![control_error_frame(
2765 &frame,
2766 "unsupported_control_frame",
2767 format!("unsupported channel-0 frame {ty:?}"),
2768 )?]),
2769 }
2770 }
2771
2772 #[cfg(test)]
2773 fn pause_cleanup_for_test(&self, stage: u8) {
2774 let mut slot = self.cleanup_pause.lock().unwrap_or_else(|p| p.into_inner());
2775 if slot.as_ref().is_some_and(|(at, _)| *at == stage) {
2776 let (_, (reached, release)) = slot.take().unwrap();
2777 drop(slot);
2778 let _ = reached.send(());
2779 let _ = release.recv_timeout(Duration::from_secs(5));
2780 }
2781 }
2782
2783 #[cfg(all(test, unix, feature = "test-support"))]
2786 pub(crate) fn after_nonce_attestation_for_test(&self, hook: impl FnOnce() + Send + 'static) {
2787 *self
2788 .after_nonce_attestation
2789 .lock()
2790 .unwrap_or_else(|p| p.into_inner()) = Some(Box::new(hook));
2791 }
2792
2793 #[cfg(test)]
2794 pub(super) fn pause_after_nonce_attestation_for_test(&self) {
2795 let hook = self
2796 .after_nonce_attestation
2797 .lock()
2798 .unwrap_or_else(|p| p.into_inner())
2799 .take();
2800 if let Some(hook) = hook {
2801 hook();
2802 }
2803 }
2804
2805 #[cfg(test)]
2806 async fn pause_hello_ingress_for_test(&self) {
2807 let pause = self
2808 .hello_ingress_pause
2809 .lock()
2810 .unwrap_or_else(|p| p.into_inner())
2811 .take();
2812 if let Some((reached, release)) = pause {
2813 let _ = reached.send(());
2814 let _ = release.await;
2815 }
2816 }
2817
2818 pub fn cleanup_connection(
2819 &self,
2820 connection_id: ConnectionId,
2821 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2822 self.agent_declarations.close(connection_id);
2836 self.close_catalog_subscriptions(connection_id);
2837 #[cfg(test)]
2838 self.pause_cleanup_for_test(0);
2839 if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
2840 spawn.retire_registration_channel();
2841 }
2842 self.supervisor
2843 .forget_registration_connection(connection_id);
2844 #[cfg(test)]
2845 self.pause_cleanup_for_test(1);
2846 self.local_connections
2847 .lock()
2848 .unwrap_or_else(|p| p.into_inner())
2849 .remove(&connection_id);
2850 self.caller_endpoints
2851 .lock()
2852 .unwrap_or_else(|p| p.into_inner())
2853 .remove(&connection_id);
2854 self.build_class_warned
2855 .lock()
2856 .unwrap_or_else(|p| p.into_inner())
2857 .remove(&connection_id);
2858 self.observed_peers
2859 .lock()
2860 .unwrap_or_else(|p| p.into_inner())
2861 .remove(&connection_id);
2862 self.captured_peers
2863 .lock()
2864 .unwrap_or_else(|p| p.into_inner())
2865 .remove(&connection_id);
2866 self.config_desktop_stamps
2867 .lock()
2868 .unwrap_or_else(|p| p.into_inner())
2869 .remove(&connection_id);
2870 self.operator_apps
2871 .lock()
2872 .unwrap_or_else(|p| p.into_inner())
2873 .remove(&connection_id);
2874 self.forwarding
2875 .operator_confirms()
2876 .connection_closed(connection_id);
2877 let cleanup = self.cleanup_connection_with_end_reason(
2878 connection_id,
2879 RegistrationEndReason::ConnectionClosed,
2880 );
2881 self.claims.connection_closed(connection_id);
2884 cleanup
2885 }
2886
2887 fn cleanup_connection_with_end_reason(
2888 &self,
2889 connection_id: ConnectionId,
2890 requested_reason: RegistrationEndReason,
2891 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2892 #[cfg(test)]
2893 if let Some(pause) = self
2894 .activity_request_pause
2895 .lock()
2896 .unwrap()
2897 .as_ref()
2898 .filter(|pause| pause.connection == connection_id)
2899 {
2900 pause
2901 .cleanup_started
2902 .store(true, std::sync::atomic::Ordering::Release);
2903 }
2904 if let Some(runtime) = &self.config_runtime {
2905 runtime.disconnected(connection_id);
2906 }
2907 self.agent_declarations.close(connection_id);
2908 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2909 self.registry
2910 .get_module_by_connection(connection_id)?
2911 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2912 .and_then(|module| module.registration_end_reason().ok().flatten())
2913 .unwrap_or(requested_reason)
2914 } else {
2915 requested_reason
2916 };
2917 let crash_closed = self
2918 .registry
2919 .get_module_by_connection(connection_id)?
2920 .and_then(|registration| {
2921 self.forwarding
2922 .module_endpoint_for_connection(connection_id)
2923 .ok()
2924 .flatten()
2925 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2926 .map(|routes| (registration.manifest.module_id, routes))
2927 });
2928 let crash_closed = crash_closed.map(|(module_id, routes)| {
2929 let terminal = match self.supervisor.get(&module_id) {
2930 None => false,
2931 Some(module) => match module.will_recover_after_connection_loss() {
2932 Ok(will_recover) => !will_recover,
2933 Err(err) => {
2934 warn!(
2935 %module_id,
2936 error = %err,
2937 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2938 );
2939 false
2940 }
2941 },
2942 };
2943 let (reason, drained) = match self
2947 .forwarding
2948 .daemon_drain_result_for_connection(connection_id)
2949 {
2950 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2951 Ok(None) => (RouteCloseReason::Crash, false),
2952 Err(err) => {
2953 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2954 (RouteCloseReason::Crash, false)
2955 }
2956 };
2957 (module_id, routes, reason, drained, terminal)
2958 });
2959 let registrations = self.deregister_connection(connection_id, end_reason);
2960 let ended_authority = self
2963 .forwarding
2964 .target_authority_for_connection(connection_id)
2965 .ok()
2966 .flatten();
2967 let cleanup = if crash_closed.is_some() {
2968 self.forwarding.cleanup_connection_counted(connection_id)
2969 } else {
2970 self.forwarding
2976 .cleanup_connection(connection_id)
2977 .map(|released| crate::forwarding::ConnectionCleanup {
2978 released,
2979 abandoned_relays: 0,
2980 })
2981 };
2982 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2987 let abandoned = cleanup
2988 .as_ref()
2989 .map_or(0, |cleanup| cleanup.abandoned_relays);
2990 send_route_control_pushes(
2991 &self.forwarding,
2992 routes,
2993 ClientControlPush::RouteClosed {
2994 module_id,
2995 channels: Vec::new(),
2996 reason,
2997 drained,
2998 abandoned,
2999 excluded_subscriptions: 0,
3000 terminal: Some(terminal),
3001 },
3002 );
3003 }
3004 if let Ok(cleanup) = cleanup {
3005 self.emit_route_goodbyes(cleanup.released);
3006 }
3007 if let Some(record) = ended_authority {
3008 self.change_spawn_authority(&record);
3009 }
3010 if matches!(®istrations, Ok(r) if !r.is_empty()) {
3014 crate::supervise::notify_registration_release();
3015 self.capability_evaluator.wake_deadline_loop();
3016 self.refresh_capability_requirements();
3017 }
3018 self.supervisor.remove_spawn_subscribers(connection_id);
3019 self.activity_streams
3020 .lock()
3021 .unwrap_or_else(|p| p.into_inner())
3022 .retain(|(id, _), task| {
3023 if *id == connection_id {
3024 task.abort();
3025 false
3026 } else {
3027 true
3028 }
3029 });
3030 self.hello_launch_nonces
3033 .lock()
3034 .unwrap_or_else(|poisoned| poisoned.into_inner())
3035 .forget(connection_id);
3036 self.scopes
3037 .write()
3038 .unwrap_or_else(|poisoned| poisoned.into_inner())
3039 .release_connection(connection_id);
3040 registrations
3041 }
3042
3043 pub(crate) fn handle_route_goodbye(
3044 &self,
3045 connection_id: ConnectionId,
3046 route_channel: u16,
3047 route_epoch: u32,
3048 ) -> Result<bool, RouterError> {
3049 debug!(
3050 connection_id = connection_id.get(),
3051 route_channel, route_epoch, "handling route GOODBYE"
3052 );
3053 let RouteRelease::Removed(released_route) = self
3054 .forwarding
3055 .release_client_route(connection_id, route_channel, route_epoch)
3056 .map_err(RouterError::Forwarding)?
3057 else {
3058 return Ok(false);
3059 };
3060 self.emit_route_goodbyes(vec![released_route]);
3061 Ok(true)
3062 }
3063
3064 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
3065 scope_expiry::ScopeExpiryContext::new(self).emit_route_goodbyes(released_routes);
3066 }
3067
3068 fn send_abandoned_route_bind_goodbye(
3080 &self,
3081 module_sink: &crate::FrameSink,
3082 negotiated_ver: u8,
3083 module_channel: u16,
3084 module_epoch: u32,
3085 ) {
3086 let frame = match Frame::build_with_version(
3087 negotiated_ver,
3088 FrameType::Goodbye,
3089 control_flags(),
3090 module_channel,
3091 module_epoch,
3092 0,
3093 Vec::new(),
3094 ) {
3095 Ok(frame) => frame,
3096 Err(err) => {
3097 warn!(
3098 route_channel = module_channel,
3099 error = %err,
3100 "failed to build GOODBYE for abandoned route.bind"
3101 );
3102 return;
3103 }
3104 };
3105 crate::forwarding::send_module_route_goodbye(
3106 &self.counters,
3107 module_sink,
3108 frame,
3109 None,
3110 "abandoned route.bind",
3111 );
3112 }
3113
3114 #[cfg(test)]
3115 fn handle_hello(
3116 &self,
3117 connection_id: ConnectionId,
3118 egress: Option<crate::router::FrameSink>,
3119 frame: Frame,
3120 ) -> Result<Vec<Frame>, RouterError> {
3121 let binding = self.claims.binding(connection_id);
3122 let channel_spawn = self.supervisor.registration_spawn(connection_id);
3123 self.handle_hello_checked(connection_id, egress, frame, false, binding, channel_spawn)
3124 }
3125
3126 fn handle_hello_checked(
3127 &self,
3128 connection_id: ConnectionId,
3129 sink: Option<crate::FrameSink>,
3130 frame: Frame,
3131 wait_for_gate: bool,
3132 prefix_binding: Option<claims::ClaimBinding>,
3133 channel_spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3134 ) -> Result<Vec<Frame>, RouterError> {
3135 let primary_sink = channel_spawn.as_ref().and_then(|_| sink.clone());
3136 #[cfg(test)]
3137 if prefix_binding.is_some() {
3138 self.claims.pause_prefix_hello_for_test(5);
3139 match self.claims.take_prefix_hello_failure_for_test() {
3140 Some("error") => {
3141 return Err(RouterError::backend(
3142 0,
3143 frame.header.corr,
3144 "injected HELLO failure",
3145 ))
3146 }
3147 Some("panic") => panic!("injected HELLO task panic"),
3148 _ => {}
3149 }
3150 }
3151 if let Some(binding) = &prefix_binding {
3152 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3153 let module = value
3154 .as_ref()
3155 .and_then(|value| value["manifest"]["module_id"].as_str());
3156 if !binding.admit_prefix_hello(module)
3157 || !self.prefix_binding_live(binding, connection_id)
3158 || !module.is_some_and(|module| {
3159 self.supervisor
3160 .prefix_claim_authorized(&binding.registration.module_id, module)
3161 })
3162 || binding.registration.check().is_err()
3163 {
3164 return Ok(vec![control_error_frame(
3165 &frame,
3166 "claim_prefix_refused",
3167 "HELLO must name the prefix claim's exact module id once",
3168 )?]);
3169 }
3170 }
3171 struct RefuseUnregisteredChannel {
3172 spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3173 sink: Option<crate::FrameSink>,
3174 }
3175 impl Drop for RefuseUnregisteredChannel {
3176 fn drop(&mut self) {
3177 if let Some(spawn) = &self.spawn {
3178 if !spawn.registered() {
3179 if let Some(sink) = &self.sink {
3180 sink.terminate_ingress();
3181 }
3182 spawn.retire_registration_channel();
3183 }
3184 }
3185 }
3186 }
3187 let _channel_attempt = RefuseUnregisteredChannel {
3188 spawn: channel_spawn.clone(),
3189 sink: sink.clone(),
3190 };
3191 if let Some(spawn) = &channel_spawn {
3192 let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3193 if let Err(code) = spawn.admit_hello(
3194 value
3195 .as_ref()
3196 .and_then(|value| value["manifest"]["module_id"].as_str()),
3197 ) {
3198 return Ok(vec![control_error_frame(
3199 &frame,
3200 code,
3201 "registration channel HELLO refused",
3202 )?]);
3203 }
3204 }
3205 debug!(
3206 connection_id = connection_id.get(),
3207 corr = frame.header.corr,
3208 "handling HELLO"
3209 );
3210 if self
3214 .registry
3215 .get_module_by_connection(connection_id)
3216 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3217 .is_some()
3218 {
3219 return Ok(vec![control_error_frame(
3220 &frame,
3221 "invalid_hello",
3222 "connection is already registered as a module",
3223 )?]);
3224 }
3225 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
3226 Ok(value) => value,
3227 Err(err) => {
3228 return Ok(vec![control_error_frame(
3229 &frame,
3230 "invalid_hello",
3231 format!("malformed HELLO body: {err}"),
3232 )?])
3233 }
3234 };
3235 if let Err(detail) = catalog::validate_hello_declarations(&hello_value) {
3236 return Ok(vec![control_error_frame(
3237 &frame,
3238 error_codes::INVALID_PROVIDER_DECLARATIONS,
3239 detail,
3240 )?]);
3241 }
3242 if let Some(module) = hello_value
3243 .get("manifest")
3244 .and_then(|value| value.get("module_id"))
3245 .and_then(|value| value.as_str())
3246 {
3247 let recovering = self
3248 .config_runtime
3249 .as_ref()
3250 .is_some_and(|owner| owner.check_recovery().unwrap_or(true));
3251 if recovering
3255 && channel_spawn.as_ref().is_none_or(|spawn| {
3256 !self.supervisor.has_live_verified_channel_binding(spawn)
3257 || spawn.check().is_err()
3258 })
3259 && !(channel_spawn.is_none()
3260 && hello_value
3261 .get("launch_nonce")
3262 .and_then(|value| value.as_str())
3263 .is_some_and(|nonce| {
3264 self.supervisor.spawned_consumer_authorized(module, nonce)
3265 && self
3266 .supervisor
3267 .has_live_verified_program_binding(module, nonce)
3268 }))
3269 {
3270 return Ok(vec![control_error_frame(
3271 &frame,
3272 "config_gated",
3273 "recovery admits only previously bound supervised launches",
3274 )?]);
3275 }
3276 if self
3277 .config_runtime
3278 .as_ref()
3279 .is_some_and(|owner| owner.module_is_gated(module))
3280 && (channel_spawn
3281 .as_ref()
3282 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3283 || self.config_runtime.as_ref().is_some_and(|owner| {
3284 !recovering && owner.check_module_hold(module).is_err()
3285 }))
3286 {
3287 return Ok(vec![control_error_frame(
3288 &frame,
3289 "reserved_module",
3290 "a gated identity belongs only to its supervised child",
3291 )?]);
3292 }
3293 }
3294 if channel_spawn.is_none() {
3295 if let (Some(module), Some(nonce)) = (
3296 hello_value
3297 .get("manifest")
3298 .and_then(|value| value.get("module_id"))
3299 .and_then(|value| value.as_str()),
3300 hello_value
3301 .get("launch_nonce")
3302 .and_then(|value| value.as_str()),
3303 ) {
3304 let binding = self.supervisor.has_program_binding_record(module, nonce);
3305 let required = self
3306 .config_runtime
3307 .as_ref()
3308 .is_some_and(|owner| owner.module_is_gated(module));
3309 if let Some(error) = (binding || required)
3310 .then(|| {
3311 self.supervisor
3312 .check_current_program_binding(module, nonce)
3313 .err()
3314 })
3315 .flatten()
3316 {
3317 let code = if error.code == "program_refused" {
3318 "program_refused"
3319 } else {
3320 "config_delivery_deferred"
3321 };
3322 self.request_refusal_close(
3323 connection_id,
3324 sink.as_ref(),
3325 CloseReason::new(code, error.detail.clone()),
3326 );
3327 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3328 }
3329 }
3330 }
3331 if let Err(err) = validate_hello_display_name(&hello_value) {
3332 return Ok(vec![control_error_frame(
3333 &frame,
3334 "invalid_manifest",
3335 err.to_string(),
3336 )?]);
3337 }
3338 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
3339 return Ok(vec![control_error_frame(
3340 &frame,
3341 "invalid_capability_grammar",
3342 err.to_string(),
3343 )?]);
3344 }
3345 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
3346 return Ok(vec![control_error_frame(
3347 &frame,
3348 "invalid_manifest",
3349 err.to_string(),
3350 )?]);
3351 }
3352 if let Err(err) = validate_hello_event_declarations(&hello_value) {
3353 return Ok(vec![control_error_frame(
3354 &frame,
3355 "invalid_event_declaration",
3356 err.to_string(),
3357 )?]);
3358 }
3359 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
3360 return Ok(vec![control_error_frame(
3361 &frame,
3362 "invalid_manifest",
3363 err.to_string(),
3364 )?]);
3365 }
3366 if let Some(provenance) = hello_value
3367 .get("manifest")
3368 .and_then(|manifest| manifest.get("provenance"))
3369 {
3370 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
3371 return Ok(vec![control_error_frame(
3372 &frame,
3373 "invalid_manifest",
3374 format!("malformed manifest provenance: {err}"),
3375 )?]);
3376 }
3377 }
3378 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
3379 let module_id = hello_value["manifest"]["module_id"]
3380 .as_str()
3381 .unwrap_or("<unknown>");
3382 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
3383 }
3384 if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
3385 let module_id = hello_value
3386 .get("manifest")
3387 .and_then(|m| m.get("module_id"))
3388 .and_then(serde_json::Value::as_str)
3389 .unwrap_or("unknown");
3390 warn!(module_id, %reason, "dropping operation_deferral from HELLO");
3391 }
3392 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
3393 let module_id = hello_value["manifest"]["module_id"]
3394 .as_str()
3395 .unwrap_or("<unknown>");
3396 warn!(module_id, %reason, "dropping flow_operations from HELLO");
3397 }
3398 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
3399 let module_id = hello_value["manifest"]["module_id"]
3400 .as_str()
3401 .unwrap_or("<unknown>");
3402 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
3403 }
3404 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
3405 Ok(hello) => hello,
3406 Err(err) => {
3407 return Ok(vec![control_error_frame(
3408 &frame,
3409 "invalid_hello",
3410 format!("malformed HELLO body: {err}"),
3411 )?])
3412 }
3413 };
3414
3415 if hello.protocol_ver != hello.manifest.protocol_ver {
3416 return Ok(vec![control_error_frame(
3417 &frame,
3418 "invalid_manifest",
3419 format!(
3420 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
3421 hello.protocol_ver, hello.manifest.protocol_ver
3422 ),
3423 )?]);
3424 }
3425
3426 if hello.manifest.module_id.trim().is_empty() {
3427 return Ok(vec![control_error_frame(
3428 &frame,
3429 "invalid_manifest",
3430 "manifest module_id must not be empty",
3431 )?]);
3432 }
3433
3434 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
3435 Ok(negotiated_ver) => negotiated_ver,
3436 Err(message) => {
3437 return Ok(vec![control_error_frame(
3438 &frame,
3439 "version_unsupported",
3440 message,
3441 )?])
3442 }
3443 };
3444
3445 let _publication_gate = if let Some(owner) = &self.config_runtime {
3451 let gate = owner
3452 .module_gate(&hello.manifest.module_id)
3453 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3454 Some(if wait_for_gate {
3455 gate.blocking_lock_owned()
3456 } else {
3457 match gate.try_lock_owned() {
3458 Ok(guard) => guard,
3459 Err(_) => {
3460 return Ok(vec![control_error_frame(
3461 &frame,
3462 "config_delivery_deferred",
3463 "module admission is busy",
3464 )?])
3465 }
3466 }
3467 })
3468 } else {
3469 None
3470 };
3471 #[cfg(test)]
3472 if let Some(hold) = &self.hello_before_publication_hold {
3473 hold.wait();
3474 hold.wait();
3475 }
3476 let module = &hello.manifest.module_id;
3477 let required = self
3478 .config_runtime
3479 .as_ref()
3480 .is_some_and(|owner| owner.module_is_gated(module));
3481 let recovering = self
3482 .config_runtime
3483 .as_ref()
3484 .is_some_and(|owner| owner.is_recovering());
3485 if recovering
3493 && channel_spawn.as_ref().is_none_or(|spawn| {
3494 !self.supervisor.has_live_verified_channel_binding(spawn) || spawn.check().is_err()
3495 })
3496 && !(channel_spawn.is_none()
3497 && hello.launch_nonce.as_deref().is_some_and(|nonce| {
3498 self.supervisor.spawned_consumer_authorized(module, nonce)
3499 && self
3500 .supervisor
3501 .has_live_verified_program_binding(module, nonce)
3502 }))
3503 {
3504 return Ok(vec![control_error_frame(
3505 &frame,
3506 "config_gated",
3507 "recovery admits only previously bound supervised launches",
3508 )?]);
3509 }
3510 if required
3511 && (channel_spawn
3512 .as_ref()
3513 .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3514 || self
3515 .config_runtime
3516 .as_ref()
3517 .is_some_and(|owner| !recovering && owner.check_module_hold(module).is_err()))
3518 {
3519 return Ok(vec![control_error_frame(
3520 &frame,
3521 "reserved_module",
3522 "gated admission requires the current supervised child's registration channel and verified image binding",
3523 )?]);
3524 }
3525 if let Some(spawn) = &channel_spawn {
3526 if let Err(code) = spawn.check() {
3527 return Ok(vec![control_error_frame(
3528 &frame,
3529 code,
3530 "registration launch is no longer live",
3531 )?]);
3532 }
3533 } else if let Some(nonce) = hello.launch_nonce.as_deref() {
3534 if required || self.supervisor.has_program_binding_record(module, nonce) {
3535 if let Err(error) = self.supervisor.check_current_program_binding(module, nonce) {
3536 let code = if error.code == "program_refused" {
3537 "program_refused"
3538 } else {
3539 "config_delivery_deferred"
3540 };
3541 self.request_refusal_close(
3542 connection_id,
3543 sink.as_ref(),
3544 CloseReason::new(code, error.detail.clone()),
3545 );
3546 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3547 }
3548 }
3549 }
3550
3551 let swap_admission = match &channel_spawn {
3559 Some(spawn) if spawn.is_candidate() => SwapHelloAdmission::Candidate,
3560 Some(_) => SwapHelloAdmission::NotSwapping,
3561 None if prefix_binding.is_some() => SwapHelloAdmission::NotSwapping,
3562 None => self
3563 .supervisor
3564 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref()),
3565 };
3566 if swap_admission == SwapHelloAdmission::ProtocolNone {
3567 warn!(
3568 module_id = %hello.manifest.module_id,
3569 connection_id = connection_id.get(),
3570 "HELLO refused: the module_id belongs to a protocol:none module, which never registers"
3571 );
3572 return Ok(vec![control_error_frame(
3575 &frame,
3576 "swap_token_invalid",
3577 format!(
3578 "module_id '{}' is a protocol:none module; a protocol:none module may not register",
3579 hello.manifest.module_id
3580 ),
3581 )?]);
3582 }
3583 if swap_admission == SwapHelloAdmission::ChannelRequired {
3584 warn!(
3585 module_id = %hello.manifest.module_id,
3586 connection_id = connection_id.get(),
3587 "HELLO refused: the module_id is gated and the HELLO did not arrive over its registration channel"
3588 );
3589 return Ok(vec![control_error_frame(
3592 &frame,
3593 "swap_token_invalid",
3594 format!(
3595 "module_id '{}' is gated; it must register over its supervised registration channel",
3596 hello.manifest.module_id
3597 ),
3598 )?]);
3599 }
3600 if swap_admission == SwapHelloAdmission::Refused {
3601 warn!(
3602 module_id = %hello.manifest.module_id,
3603 connection_id = connection_id.get(),
3604 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
3605 );
3606 return Ok(vec![control_error_frame(
3607 &frame,
3608 "swap_token_invalid",
3609 format!(
3610 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
3611 hello.manifest.module_id
3612 ),
3613 )?]);
3614 }
3615 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
3616
3617 let verification = if swap_candidate && channel_spawn.is_none() {
3622 ReservedHelloVerification {
3623 nonce_verified: true,
3624 prefix_owner: None,
3625 }
3626 } else {
3627 let verification = if let Some(binding) = &prefix_binding {
3628 if !self.supervisor.prefix_claim_authorized(
3629 &binding.registration.module_id,
3630 &hello.manifest.module_id,
3631 ) {
3632 return Ok(vec![control_error_frame(
3633 &frame,
3634 "claim_prefix_refused",
3635 "prefix ownership changed",
3636 )?]);
3637 }
3638 Ok(ReservedHelloVerification {
3639 nonce_verified: true,
3640 prefix_owner: Some(binding.registration.module_id.clone()),
3641 })
3642 } else if channel_spawn.is_some() {
3643 self.supervisor
3644 .channel_hello_verification(&hello.manifest.module_id)
3645 } else {
3646 self.supervisor.reserved_hello_verification(
3647 &hello.manifest.module_id,
3648 hello.launch_nonce.as_deref(),
3649 )
3650 };
3651 match verification {
3652 Ok(verified) => verified,
3653 Err(rejection) => {
3654 let message = match rejection {
3655 ReservedHelloRejection::Exact { module_id } => format!(
3656 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
3657 ),
3658 ReservedHelloRejection::Prefix {
3659 prefix,
3660 owner_module_id,
3661 } => format!(
3662 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
3663 hello.manifest.module_id
3664 ),
3665 };
3666 return Ok(vec![control_error_frame(
3667 &frame,
3668 "reserved_module",
3669 message,
3670 )?]);
3671 }
3672 }
3673 };
3674
3675 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
3676 &hello.manifest.module_id,
3677 hello.manifest.capabilities.as_ref(),
3678 );
3679 if let Some(refusal) = reserved_capability_refusals.first() {
3680 let capability = refusal.capability.clone();
3681 let bound_module = refusal.claimants[0].clone();
3682 log_duplicate_claim_events(reserved_capability_refusals);
3683 return Ok(vec![control_error_frame(
3684 &frame,
3685 "reserved_capability",
3686 format!(
3687 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3688 capability, bound_module, hello.manifest.module_id
3689 ),
3690 )?]);
3691 }
3692
3693 if self
3696 .forwarding
3697 .connection_has_client_routes(connection_id)
3698 .map_err(RouterError::Forwarding)?
3699 {
3700 return Ok(vec![control_error_frame(
3701 &frame,
3702 "invalid_hello",
3703 "connection has open client routes and cannot also register as a module",
3704 )?]);
3705 }
3706
3707 self.hello_launch_nonces
3715 .lock()
3716 .unwrap_or_else(|poisoned| poisoned.into_inner())
3717 .record(
3718 connection_id,
3719 if prefix_binding.is_some() || channel_spawn.is_some() {
3720 None
3721 } else {
3722 hello.launch_nonce.as_deref()
3723 },
3724 );
3725 let control_ops = effective_module_control_ops(hello.control_ops);
3726 let prefix_submodule = verification.prefix_owner.is_some();
3727 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
3730 #[cfg(test)]
3731 struct PublicationFinished(Arc<std::sync::Barrier>);
3732 #[cfg(test)]
3733 impl Drop for PublicationFinished {
3734 fn drop(&mut self) {
3735 self.0.wait();
3736 }
3737 }
3738 #[cfg(test)]
3739 let _publication_finished = self.hello_after_launch_check_hold.as_ref().map(|hold| {
3740 hold.wait();
3741 hold.wait();
3742 PublicationFinished(hold.clone())
3743 });
3744 let publication_spawn = channel_spawn
3745 .as_ref()
3746 .or_else(|| prefix_binding.as_ref().map(|binding| &binding.registration));
3747 #[cfg(test)]
3748 if prefix_binding.is_some() {
3749 self.claims.pause_prefix_hello_for_test(7);
3750 }
3751 let _registration_publication = match publication_spawn {
3752 Some(spawn) => Some(match spawn.begin_publication() {
3753 Ok(guard) => guard,
3754 Err(code) => {
3755 return Ok(vec![control_error_frame(
3756 &frame,
3757 code,
3758 "registration ended before HELLO publication",
3759 )?])
3760 }
3761 }),
3762 None => None,
3763 };
3764 let (_prefix_live, _prefix_ownership) = if let Some(binding) = &prefix_binding {
3765 let live = self.claims.lock_prefix_binding(binding, connection_id);
3766 let ownership = self.supervisor.prefix_claim_publication(
3767 &binding.registration.module_id,
3768 &hello.manifest.module_id,
3769 );
3770 if live.is_none()
3771 || ownership.is_none()
3772 || !self.claim_primary_registered(binding.spawn, &binding.registration)
3773 || binding.prefix_module_id() != Some(hello.manifest.module_id.as_str())
3774 {
3775 return Ok(vec![control_error_frame(
3776 &frame,
3777 "claim_prefix_refused",
3778 "prefix authority ended before publication",
3779 )?]);
3780 }
3781 (live, ownership)
3782 } else {
3783 (None, None)
3784 };
3785 if let Some(spawn) = publication_spawn {
3786 self.forwarding
3787 .bind_spawn(connection_id, spawn)
3788 .map_err(RouterError::Forwarding)?;
3789 }
3790 let hello_authority = match (&channel_spawn, &prefix_binding) {
3800 (Some(spawn), _) => Some(spawn.authority.clone()),
3801 (None, None) => hello.launch_nonce.as_deref().and_then(|nonce| {
3802 self.supervisor
3803 .spawned_consumer_record(&hello.manifest.module_id, nonce)
3804 }),
3805 (None, Some(_)) => None,
3806 }
3807 .map(|record| HelloAuthority {
3808 record,
3809 requested: hello
3810 .manifest
3811 .provider_declarations
3812 .clone()
3813 .unwrap_or_default(),
3814 froze: Default::default(),
3815 });
3816 #[cfg(test)]
3817 if prefix_binding.is_some() {
3818 self.claims.pause_prefix_hello_for_test(8);
3819 }
3820 if swap_candidate {
3821 let module_id = hello.manifest.module_id.clone();
3822 let mut differing_declarations = Default::default();
3823 let reply = self.register_swap_candidate(
3824 connection_id,
3825 sink,
3826 &frame,
3827 hello.manifest,
3828 negotiated_ver,
3829 control_ops,
3830 hello_ack,
3831 hello_authority.as_ref(),
3832 &mut differing_declarations,
3833 #[cfg(windows)]
3834 hello.launch_nonce.as_deref(),
3835 )?;
3836 if let Some(spawn) = &channel_spawn {
3837 if self
3838 .registry
3839 .get_module_by_connection(connection_id)
3840 .ok()
3841 .flatten()
3842 .is_some()
3843 {
3844 if let Some(sink) = primary_sink {
3845 spawn.bind_primary_sink(sink);
3846 }
3847 spawn.hello_accepted();
3848 }
3849 } else if self
3850 .registry
3851 .get_module_by_connection(connection_id)
3852 .ok()
3853 .flatten()
3854 .is_some()
3855 {
3856 self.supervisor
3857 .retire_unused_registration_channel(&module_id, true);
3858 }
3859 if self
3860 .registry
3861 .get_module_by_connection(connection_id)
3862 .ok()
3863 .flatten()
3864 .is_some()
3865 {
3866 self.file_development_widening(
3867 hello_authority.map(|authority| authority.record),
3868 differing_declarations,
3869 );
3870 }
3871 return Ok(reply);
3872 }
3873 let registration = match self
3880 .registry
3881 .register_unpublished_verified_with_control_ops(
3882 hello.manifest,
3883 negotiated_ver,
3884 connection_id,
3885 control_ops,
3886 verification.nonce_verified,
3887 ) {
3888 Ok(registration) => registration,
3889 Err(RegistryError::DuplicateModuleId { module_id }) => {
3890 return Ok(vec![control_error_frame(
3891 &frame,
3892 "duplicate_module_id",
3893 format!(
3894 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
3895 ),
3896 )?])
3897 }
3898 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3899 return Ok(vec![control_error_frame(
3900 &frame,
3901 "invalid_module_id",
3902 err.to_string(),
3903 )?])
3904 }
3905 Err(err) => {
3906 return Ok(vec![control_error_frame(
3907 &frame,
3908 "registry_error",
3909 err.to_string(),
3910 )?])
3911 }
3912 };
3913 let differing_declarations =
3914 match self.admit_hello_authority(connection_id, hello_authority.as_ref()) {
3915 Ok(differing) => differing,
3916 Err(err) => {
3917 if matches!(
3918 self.rollback_unpublished_hello(connection_id),
3919 Ok(r) if !r.is_empty()
3920 ) {
3921 crate::supervise::notify_registration_release();
3922 }
3923 return Err(RouterError::Forwarding(err));
3924 }
3925 };
3926 #[cfg(test)]
3927 self.hold_between_registry_and_forwarding();
3928
3929 let mut reply = Vec::new();
3930 let publication = self
3933 .registry
3934 .publish_registration_after(connection_id, || {
3935 if let Some(sink) = sink {
3936 let concurrency = manifest_concurrency(®istration.manifest);
3952 self.forwarding.register_module_connection_acked(
3953 connection_id,
3954 registration.manifest.module_id.clone(),
3955 negotiated_ver,
3956 concurrency,
3957 sink,
3958 crate::forwarding::ModuleHelloAck {
3959 frame: hello_ack,
3960 verified_prefix_owner: verification.prefix_owner,
3961 },
3962 )?;
3963 } else {
3964 reply.push(hello_ack);
3967 }
3968 #[cfg(test)]
3969 self.hold_between_ack_and_publication();
3970 Ok(())
3971 })
3972 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3973 let published = match publication {
3974 Ok(published) => published,
3975 Err(err) => {
3976 if matches!(
3983 self.rollback_unpublished_hello(connection_id),
3984 Ok(r) if !r.is_empty()
3985 ) {
3986 crate::supervise::notify_registration_release();
3987 }
3988 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
3989 return Ok(vec![control_error_frame(
3990 &frame,
3991 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
3992 "invalid_hello"
3993 } else {
3994 forwarding_error_code(&err)
3995 },
3996 err.to_string(),
3997 )?]);
3998 }
3999 };
4000 if !published {
4001 debug!(
4002 module_id = %registration.manifest.module_id,
4003 connection_id = connection_id.get(),
4004 "registration was removed before it could be published"
4005 );
4006 self.refuse_hello_authority(connection_id, hello_authority.as_ref());
4007 return Ok(reply);
4008 }
4009
4010 drop(_prefix_ownership);
4011 drop(_prefix_live);
4012
4013 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
4022 info!(
4023 module_id = %registration.manifest.module_id,
4024 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
4025 );
4026 }
4027 if registration.manifest.registration_channel == Some(false)
4028 && self
4029 .supervisor
4030 .module_is_gated(®istration.manifest.module_id)
4031 && self
4032 .config_runtime
4033 .as_ref()
4034 .and_then(|owner| owner.registration_declaration(®istration.manifest.module_id))
4035 .as_deref()
4036 == Some("declared")
4037 {
4038 warn!(module_id = %registration.manifest.module_id,
4039 "HELLO registration_channel is false despite the approved channel declaration; registration remains admitted");
4040 }
4041
4042 self.apply_registration_capabilities(®istration);
4043
4044 self.supervisor
4045 .record_activity_registration(®istration.manifest.module_id);
4046
4047 info!(
4048 module_id = %registration.manifest.module_id,
4049 module_version = %registration.manifest.module_version,
4050 negotiated_ver,
4051 routable_provider = manifest_provides_routable_role(®istration.manifest),
4052 connection_id = connection_id.get(),
4053 "module registered"
4054 );
4055
4056 if let Some(spawn) = &channel_spawn {
4057 if let Some(sink) = primary_sink {
4058 spawn.bind_primary_sink(sink);
4059 }
4060 spawn.hello_accepted();
4061 } else if !prefix_submodule {
4062 self.supervisor
4063 .retire_unused_registration_channel(®istration.manifest.module_id, false);
4064 }
4065 self.file_development_widening(
4066 hello_authority.map(|authority| authority.record),
4067 differing_declarations,
4068 );
4069 #[cfg(windows)]
4070 if let Some(nonce) = hello.launch_nonce.as_deref() {
4071 self.supervisor
4072 .program_hello_accepted(®istration.manifest.module_id, nonce);
4073 }
4074
4075 #[cfg(test)]
4076 if let Some(hold) = &self.hello_inside_publication_hold {
4077 hold.wait();
4078 hold.wait();
4079 }
4080
4081 Ok(reply)
4082 }
4083
4084 #[allow(clippy::too_many_arguments)]
4101 fn register_swap_candidate(
4102 &self,
4103 connection_id: ConnectionId,
4104 sink: Option<crate::FrameSink>,
4105 frame: &Frame,
4106 manifest: ModuleManifest,
4107 negotiated_ver: u8,
4108 control_ops: Vec<String>,
4109 hello_ack: Frame,
4110 hello_authority: Option<&HelloAuthority>,
4111 differing_declarations: &mut ProviderDeclarations,
4112 #[cfg(windows)] launch_nonce: Option<&str>,
4113 ) -> Result<Vec<Frame>, RouterError> {
4114 let module_id = manifest.module_id.clone();
4115 self.supervisor
4118 .with_swap_candidate_admission(&module_id, || {
4119 let reply = self.publish_swap_candidate(
4120 connection_id,
4121 sink,
4122 frame,
4123 manifest,
4124 negotiated_ver,
4125 control_ops,
4126 hello_ack,
4127 hello_authority,
4128 differing_declarations,
4129 #[cfg(windows)]
4130 launch_nonce,
4131 );
4132 let admitted = self
4133 .registry
4134 .get_candidate(&module_id)
4135 .ok()
4136 .flatten()
4137 .is_some_and(|candidate| candidate.connection_id == connection_id);
4138 (reply, admitted)
4139 })
4140 .unwrap_or_else(|| {
4141 Ok(vec![control_error_frame(
4142 frame,
4143 "swap_token_invalid",
4144 "swap ended before candidate HELLO publication",
4145 )?])
4146 })
4147 }
4148
4149 #[allow(clippy::too_many_arguments)]
4150 fn publish_swap_candidate(
4151 &self,
4152 connection_id: ConnectionId,
4153 sink: Option<crate::FrameSink>,
4154 frame: &Frame,
4155 manifest: ModuleManifest,
4156 negotiated_ver: u8,
4157 control_ops: Vec<String>,
4158 hello_ack: Frame,
4159 hello_authority: Option<&HelloAuthority>,
4160 differing_declarations: &mut ProviderDeclarations,
4161 #[cfg(windows)] launch_nonce: Option<&str>,
4162 ) -> Result<Vec<Frame>, RouterError> {
4163 let module_id = manifest.module_id.clone();
4164 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
4165 manifest,
4166 negotiated_ver,
4167 connection_id,
4168 control_ops,
4169 ) {
4170 Ok(registration) => registration,
4171 Err(RegistryError::DuplicateModuleId { module_id }) => {
4172 return Ok(vec![control_error_frame(
4173 frame,
4174 "duplicate_module_id",
4175 format!(
4176 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
4177 ),
4178 )?])
4179 }
4180 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
4181 return Ok(vec![control_error_frame(
4182 frame,
4183 "invalid_module_id",
4184 err.to_string(),
4185 )?])
4186 }
4187 Err(err) => {
4188 return Ok(vec![control_error_frame(
4189 frame,
4190 "registry_error",
4191 err.to_string(),
4192 )?])
4193 }
4194 };
4195 *differing_declarations = match self.admit_hello_authority(connection_id, hello_authority) {
4196 Ok(differing) => differing,
4197 Err(err) => {
4198 if matches!(
4199 self.rollback_unpublished_hello(connection_id),
4200 Ok(r) if !r.is_empty()
4201 ) {
4202 crate::supervise::notify_registration_release();
4203 }
4204 return Err(RouterError::Forwarding(err));
4205 }
4206 };
4207 #[cfg(test)]
4208 self.hold_between_registry_and_forwarding();
4209 let mut reply = Vec::new();
4210 let publication = self
4211 .registry
4212 .publish_registration_after(connection_id, || {
4213 if let Some(sink) = sink {
4214 let concurrency = manifest_concurrency(®istration.manifest);
4218 self.forwarding.register_candidate_module_connection_acked(
4219 connection_id,
4220 module_id.clone(),
4221 negotiated_ver,
4222 concurrency,
4223 sink,
4224 hello_ack,
4225 )?;
4226 } else {
4227 reply.push(hello_ack);
4228 }
4229 #[cfg(test)]
4230 self.hold_between_ack_and_publication();
4231 Ok(())
4232 })
4233 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4234 let published = match publication {
4235 Ok(published) => published,
4236 Err(err) => {
4237 if matches!(
4238 self.rollback_unpublished_hello(connection_id),
4239 Ok(r) if !r.is_empty()
4240 ) {
4241 crate::supervise::notify_registration_release();
4242 }
4243 self.refuse_hello_authority(connection_id, hello_authority);
4244 return Ok(vec![control_error_frame(
4245 frame,
4246 forwarding_error_code(&err),
4247 err.to_string(),
4248 )?]);
4249 }
4250 };
4251 if !published {
4252 debug!(
4253 module_id = %module_id,
4254 connection_id = connection_id.get(),
4255 "swap candidate registration was removed before it could be published"
4256 );
4257 self.refuse_hello_authority(connection_id, hello_authority);
4258 return Ok(reply);
4259 }
4260 info!(
4261 module_id = %module_id,
4262 module_version = %registration.manifest.module_version,
4263 negotiated_ver,
4264 ready = registration.ready,
4265 connection_id = connection_id.get(),
4266 "swap candidate registered; not routable until cutover"
4267 );
4268 #[cfg(windows)]
4269 if let Some(nonce) = launch_nonce {
4270 self.supervisor.program_hello_accepted(&module_id, nonce);
4271 }
4272 Ok(reply)
4273 }
4274
4275 fn admit_hello_authority(
4281 &self,
4282 connection: ConnectionId,
4283 authority: Option<&HelloAuthority>,
4284 ) -> Result<ProviderDeclarations, ForwardingError> {
4285 let Some(authority) = authority else {
4286 return Ok(ProviderDeclarations::new());
4287 };
4288 let froze = authority
4289 .record
4290 .freeze_declarations(authority.requested.clone());
4291 authority
4292 .froze
4293 .store(froze.is_some(), std::sync::atomic::Ordering::Relaxed);
4294 self.forwarding
4295 .bind_target_authority(connection, authority.record.clone())
4296 .inspect_err(|_| self.refuse_hello_authority(connection, Some(authority)))?;
4297 Ok(froze.unwrap_or_default())
4298 }
4299
4300 fn refuse_hello_authority(&self, connection: ConnectionId, authority: Option<&HelloAuthority>) {
4305 let Some(authority) = authority else {
4306 return;
4307 };
4308 if let Err(err) = self.forwarding.unbind_target_authority(connection) {
4309 warn!(error = %err, "failed to unbind a refused HELLO from its spawn");
4310 }
4311 if authority
4312 .froze
4313 .swap(false, std::sync::atomic::Ordering::Relaxed)
4314 {
4315 authority.record.unfreeze_refused_declarations();
4316 }
4317 }
4318
4319 fn build_hello_ack(
4320 &self,
4321 frame: &Frame,
4322 negotiated_ver: u8,
4323 module_id: &str,
4324 ) -> Result<Frame, RouterError> {
4325 let ack = ModuleHelloAckBody {
4326 negotiated_ver,
4327 subc_ops: module_subc_ops(),
4328 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
4329 storage: self
4330 .storage_config
4331 .as_ref()
4332 .map(|cfg| cfg.descriptor_for(module_id)),
4333 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4334 };
4335 let body = serde_json::to_vec(&ack).map_err(|err| {
4336 RouterError::backend(
4337 0,
4338 frame.header.corr,
4339 format!("failed to encode HELLO_ACK: {err}"),
4340 )
4341 })?;
4342
4343 Frame::build_with_version(
4344 negotiated_ver,
4345 FrameType::HelloAck,
4346 control_flags(),
4347 0,
4348 0,
4349 frame.header.corr,
4350 body,
4351 )
4352 .map_err(RouterError::FrameBuild)
4353 }
4354
4355 async fn handle_client_control_request(
4356 &self,
4357 ctx: &RouteCtx,
4358 frame: Frame,
4359 request: ClientControlRequest,
4360 route_open_timings: &mut Option<RouteOpenTimings>,
4361 ) -> Result<Vec<Frame>, RouterError> {
4362 if let Some(owner) = &self.config_runtime {
4363 let module = match &request {
4364 ClientControlRequest::SupervisorRestart { module_id, .. }
4365 | ClientControlRequest::SupervisorReload { module_id }
4366 | ClientControlRequest::SupervisorSwap { module_id, .. }
4367 | ClientControlRequest::SupervisorSetEnabled {
4368 module_id,
4369 enabled: true,
4370 } => Some(module_id.as_str()),
4371 _ => None,
4372 };
4373 let rescan = matches!(
4374 &request,
4375 ClientControlRequest::SupervisorRescan { preview: false }
4376 | ClientControlRequest::SupervisorRescanChecked { preview: false, .. }
4377 );
4378 let held = if module.is_some() || rescan {
4379 let module = module.map(str::to_owned);
4380 owner
4381 .on_blocking(move |owner| owner.checked_admission_hold(module.as_deref()))
4382 .await
4383 } else {
4384 Ok(())
4385 };
4386 if let Err(error) = held {
4387 if error.code != "config_delivery_deferred" {
4388 let code = if error.code == "store_tampered" {
4389 "store_tampered"
4390 } else {
4391 "config_gated"
4392 };
4393 return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
4394 }
4395 }
4396 }
4397 if let Some(operation) = privileged_operation(&request) {
4398 let caller = self.resolve_caller(ctx.connection_id).await;
4399 let op = client_control_request_op(&request);
4400 let text = prompt_text(operation, &caller);
4401 match self.privileged_operator_prompt {
4402 PrivilegedOperatorPrompt::Log => {
4403 info!(target: "subc_daemon::privileged_prompt", operation = op,
4404 target = ?operation, caller = %caller.description(), summary = %text,
4405 "privileged request would prompt the operator");
4406 }
4407 PrivilegedOperatorPrompt::Enforce => {
4408 let key =
4409 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
4410 let stamp = self
4411 .operator_apps
4412 .lock()
4413 .unwrap_or_else(|p| p.into_inner())
4414 .get(&ctx.connection_id)
4415 .cloned();
4416 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
4417 &self.operator_app_policy,
4418 &self.launch_table,
4419 stamp.as_ref(),
4420 )
4421 .then(|| self.shortcut_context(stamp, None, None));
4422 let outcome = self
4423 .forwarding
4424 .operator_confirms()
4425 .approve_privileged_with_shortcut(
4426 ctx,
4427 &frame,
4428 op,
4429 text,
4430 caller.description(),
4431 key,
4432 shortcut,
4433 )
4434 .await;
4435 if outcome.code != "confirmed" {
4436 return Ok(vec![outcome.frame(&frame)]);
4437 }
4438 }
4439 }
4440 }
4441 match request {
4442 ClientControlRequest::DaemonShutdownIntent { intent } => {
4443 if intent.as_str() != Some("restart") {
4444 return Ok(vec![control_error_frame(
4445 &frame,
4446 "invalid_params",
4447 "intent must be 'restart'",
4448 )?]);
4449 }
4450 if let Some(owner) = &self.config_runtime {
4455 match owner
4456 .on_blocking(|owner| owner.assign_daemon_restart())
4457 .await
4458 {
4459 Ok(Some(generation)) => {
4460 info!(
4461 generation,
4462 "daemon restart will warm up approved generation"
4463 )
4464 }
4465 Ok(None) => {}
4466 Err(error) => warn!(
4467 %error,
4468 "daemon restart intent not recorded; the next start boots the last active generation"
4469 ),
4470 }
4471 }
4472 let response = ClientControlResponse::DaemonShutdownIntent {
4473 intent: "restart".to_owned(),
4474 expires_at_ms: self.supervisor.announce_daemon_restart(),
4475 };
4476 Ok(vec![control_response_body_frame(
4477 &frame,
4478 &response,
4479 "ClientControlResponse::DaemonShutdownIntent",
4480 )?])
4481 }
4482 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
4483 ClientControlRequest::CatalogList { module_id } => {
4484 self.handle_catalog_list(frame, module_id)
4485 }
4486 ClientControlRequest::RouteOpen {
4487 target,
4488 identity,
4489 consumer_identity,
4490 consumer_capabilities,
4491 role_versions,
4492 busy_refusals,
4493 admission_facts,
4494 scope,
4495 ..
4496 } => {
4497 let extensions = match serde_json::from_slice::<subc_control::RouteOpenRequestExt>(
4498 &frame.body,
4499 ) {
4500 Ok(extensions) => extensions,
4501 Err(err) => {
4502 return Ok(vec![control_error_frame(
4503 &frame,
4504 "invalid_control_body",
4505 format!("malformed route.open extensions: {err}"),
4506 )?]);
4507 }
4508 };
4509 self.handle_route_open(
4510 ctx,
4511 frame,
4512 RouteOpenRequest {
4513 target,
4514 identity,
4515 consumer_identity,
4516 consumer_capabilities,
4517 role_versions,
4518 busy_refusals,
4519 admission_facts,
4520 scope,
4521 accepts_close_reasons: extensions.accepts_close_reasons,
4522 },
4523 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
4524 )
4525 .await
4526 }
4527 ClientControlRequest::RoutePoll {
4528 route_channel,
4529 route_epoch,
4530 kind,
4531 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
4532 ClientControlRequest::SupervisorList {} => {
4533 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
4534 self.agent_declarations.refuse(
4535 self,
4536 ctx,
4537 control_error_frame(
4538 &frame,
4539 "busy",
4540 "supervisor list work is at capacity; retry",
4541 )?,
4542 );
4543 return Ok(Vec::new());
4544 };
4545 let handler = self.clone();
4547 let sink = ctx.egress.clone();
4548 tokio::spawn(
4549 async move {
4550 let _permit = permit;
4551 let result = handler.handle_supervisor_list(frame.clone()).await;
4552 let replies = match result {
4553 Ok(replies) => replies,
4554 Err(error) => {
4555 control_error_frame(&frame, "backend_error", error.to_string())
4556 .into_iter()
4557 .collect()
4558 }
4559 };
4560 for reply in replies {
4561 if !matches!(
4562 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
4563 .await,
4564 Ok(Ok(()))
4565 ) {
4566 break;
4567 }
4568 }
4569 }
4570 .with_current_subscriber(),
4571 );
4572 Ok(Vec::new())
4573 }
4574 ClientControlRequest::SupervisorSpawnSnapshot {} => {
4575 self.handle_supervisor_spawn_snapshot(frame)
4576 }
4577 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
4578 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
4579 }
4580 ClientControlRequest::SupervisorRestart {
4581 module_id,
4582 drain_timeout_ms,
4583 } => {
4584 self.log_supervisor_request_received(
4585 ctx,
4586 frame.header.corr,
4587 ops::SUPERVISOR_RESTART,
4588 Some(&module_id),
4589 None,
4590 )?;
4591 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4592 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
4593 .await
4594 }
4595 ClientControlRequest::SupervisorSwap {
4596 module_id,
4597 ready_timeout_ms,
4598 } => {
4599 self.log_supervisor_request_received(
4600 ctx,
4601 frame.header.corr,
4602 ops::SUPERVISOR_SWAP,
4603 Some(&module_id),
4604 None,
4605 )?;
4606 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4607 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
4608 .await
4609 }
4610 ClientControlRequest::SupervisorReload { module_id } => {
4611 self.log_supervisor_request_received(
4612 ctx,
4613 frame.header.corr,
4614 ops::SUPERVISOR_RELOAD,
4615 Some(&module_id),
4616 None,
4617 )?;
4618 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4619 self.handle_supervisor_reload(frame, module_id, requester)
4620 .await
4621 }
4622 ClientControlRequest::SupervisorRescan { preview } => {
4623 if !preview {
4624 self.log_supervisor_request_received(
4625 ctx,
4626 frame.header.corr,
4627 ops::SUPERVISOR_RESCAN,
4628 None,
4629 None,
4630 )?;
4631 }
4632 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4633 let operation = self.handle_supervisor_rescan(
4634 frame,
4635 if preview {
4636 RescanMode::Preview
4637 } else {
4638 RescanMode::Execute
4639 },
4640 requester,
4641 );
4642 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
4643 .await
4644 }
4645 ClientControlRequest::SupervisorRescanChecked {
4646 preview,
4647 candidate,
4648 expect,
4649 } => {
4650 self.observe_control_wait(
4651 ctx,
4652 None,
4653 ops::SUPERVISOR_RESCAN,
4654 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
4655 )
4656 .await
4657 }
4658 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
4659 self.log_supervisor_request_received(
4660 ctx,
4661 frame.header.corr,
4662 ops::SUPERVISOR_RELEASE_RESERVED,
4663 Some(&module_id),
4664 None,
4665 )?;
4666 self.handle_supervisor_release_reserved(frame, module_id)
4667 .await
4668 }
4669 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
4670 self.log_supervisor_request_received(
4671 ctx,
4672 frame.header.corr,
4673 ops::SUPERVISOR_SET_ENABLED,
4674 Some(&module_id),
4675 Some(enabled),
4676 )?;
4677 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4678 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
4679 .await
4680 }
4681 ClientControlRequest::SupervisorHealthProbe { module_id } => {
4682 self.handle_supervisor_health_probe(ctx, frame, module_id)
4683 .await
4684 }
4685 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
4686 ClientControlRequest::SupervisorRoutes { module_id } => {
4687 self.handle_supervisor_routes(frame, module_id)
4688 }
4689 ClientControlRequest::SupervisorProvenance { module_id } => {
4690 self.handle_supervisor_provenance(ctx, frame, module_id)
4691 .await
4692 }
4693 ClientControlRequest::SupervisorStderrTail {
4694 module_id,
4695 max_lines,
4696 max_bytes,
4697 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
4698 ClientControlRequest::SupervisorTerminals { module_id } => {
4699 self.handle_supervisor_terminals(ctx, frame, module_id)
4700 .await
4701 }
4702 }
4703 }
4704
4705 fn log_supervisor_request_received(
4706 &self,
4707 ctx: &RouteCtx,
4708 corr: u64,
4709 op: &'static str,
4710 module_id: Option<&str>,
4711 enabled: Option<bool>,
4712 ) -> Result<(), RouterError> {
4713 let caller = self.supervisor_requester(ctx, corr)?;
4714 let caller = principal_label(&caller);
4715 info!(
4718 target: "control",
4719 op,
4720 module_id,
4721 enabled,
4722 connection_id = ctx.connection_id.get(),
4723 caller = %caller,
4724 "supervisor request received"
4725 );
4726 Ok(())
4727 }
4728
4729 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
4730 Ok(self
4731 .registry
4732 .get_module_by_connection(ctx.connection_id)
4733 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4734 .map(|registration| Principal::Reserved {
4735 module_id: registration.manifest.module_id,
4736 })
4737 .unwrap_or(Principal::Direct))
4738 }
4739
4740 fn handle_module_control_request(
4741 &self,
4742 connection_id: ConnectionId,
4743 frame: Frame,
4744 request: ModuleControlRequestFromModule,
4745 ) -> Result<Vec<Frame>, RouterError> {
4746 match request {
4747 ModuleControlRequestFromModule::RouteAuthorize {
4748 principal,
4749 scope,
4750 target,
4751 expected,
4752 } => self.handle_route_authorize(
4753 connection_id,
4754 frame,
4755 principal,
4756 scope,
4757 target,
4758 expected,
4759 ),
4760 ModuleControlRequestFromModule::CatalogUpdate {
4761 provides,
4762 capabilities,
4763 ready,
4764 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
4765 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
4766 self.handle_catalog_update_role_declarations(
4767 connection_id,
4768 frame,
4769 role_declarations,
4770 )
4771 }
4772 ModuleControlRequestFromModule::LiveRoots {} => {
4773 let registered = self
4774 .registry
4775 .get_module_by_connection(connection_id)
4776 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4777 let Some(registration) = registered else {
4778 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
4779 };
4780 let response = self
4781 .forwarding
4782 .live_roots(®istration.manifest.module_id)
4783 .map_err(RouterError::Forwarding)?;
4784 Ok(vec![control_response_body_frame(
4785 &frame,
4786 &response,
4787 "ModuleControlResponseToModule::LiveRoots",
4788 )?])
4789 }
4790 ModuleControlRequestFromModule::AgentCommands {
4791 started,
4792 finished,
4793 known,
4794 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
4795 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
4796 self.handle_scope_sync(connection_id, frame, generation, scopes)
4797 }
4798 ModuleControlRequestFromModule::ScopeApply {
4799 generation,
4800 upsert,
4801 end,
4802 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
4803 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
4804 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
4805 }
4806 _ => Ok(vec![control_error_frame(
4807 &frame,
4808 "unsupported_control_frame",
4809 "unsupported module control operation",
4810 )?]),
4811 }
4812 }
4813
4814 fn handle_scope_served_by(
4815 &self,
4816 connection_id: ConnectionId,
4817 frame: Frame,
4818 ) -> Result<Vec<Frame>, RouterError> {
4819 let registered = self
4820 .registry
4821 .get_module_by_connection(connection_id)
4822 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4823 let Some(registered) = registered else {
4824 return Ok(vec![control_error_frame(
4825 &frame,
4826 "not_registered",
4827 "scope.served_by requires an active module registration owned by this connection",
4828 )?]);
4829 };
4830 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
4831 {
4832 Ok(request) => request,
4833 Err(err) => {
4834 return Ok(vec![control_error_frame(
4835 &frame,
4836 "invalid_control_body",
4837 format!("malformed scope.served_by body: {err}"),
4838 )?])
4839 }
4840 };
4841 let answer = self
4842 .scopes
4843 .read()
4844 .unwrap_or_else(|poisoned| poisoned.into_inner())
4845 .served_by(
4846 ®istered.manifest.module_id,
4847 &request.scope_ref,
4848 request.scope_epoch,
4849 &request.module_id,
4850 );
4851 let reply = subc_control::ScopeServedByReply::new(
4852 answer,
4853 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4854 );
4855 Ok(vec![control_response_body_frame(
4856 &frame,
4857 &reply,
4858 "ScopeServedByReply",
4859 )?])
4860 }
4861
4862 fn handle_operator_confirm(
4863 &self,
4864 ctx: &RouteCtx,
4865 frame: Frame,
4866 ) -> Result<Vec<Frame>, RouterError> {
4867 use crate::operator_confirm::{audit, Outcome};
4868 let registration = self
4869 .registry
4870 .get_module_by_connection(ctx.connection_id)
4871 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4872 let Some(registration) = registration else {
4873 let outcome = Outcome::refusal("not_registered");
4874 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4875 return Ok(vec![outcome.frame(&frame)]);
4876 };
4877 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
4878 Ok(request) => request,
4879 Err(_) => {
4880 let outcome = Outcome::refusal("invalid_control_body");
4881 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4882 return Ok(vec![outcome.frame(&frame)]);
4883 }
4884 };
4885 let declared = matches!(
4886 request.kind,
4887 Some(subc_protocol::session::OperatorConfirmKind::Operation)
4888 ) && request.operation.as_ref().is_some_and(|operation| {
4889 registration
4890 .manifest
4891 .confirm_operations
4892 .as_ref()
4893 .is_some_and(|operations| operations.contains(operation))
4894 });
4895 let module_id = registration.manifest.module_id;
4896 let nonce = self
4900 .hello_launch_nonces
4901 .lock()
4902 .unwrap_or_else(|p| p.into_inner())
4903 .nonce(ctx.connection_id)
4904 .map(str::to_owned);
4905 let nonce_proven = self
4908 .confirm_channel_proven(ctx.connection_id, &module_id)
4909 .unwrap_or_else(|| {
4910 nonce.as_deref().is_some_and(|nonce| {
4911 self.supervisor
4912 .spawned_consumer_authorized(&module_id, nonce)
4913 })
4914 });
4915 let confirms = self.forwarding.operator_confirms();
4916 self.forwarding
4917 .with_operator_route(
4918 ctx.connection_id,
4919 request.route_channel,
4920 request.route_epoch,
4921 |binding| {
4922 let shortcut = binding
4923 .filter(|binding| {
4924 nonce_proven
4925 && crate::operator_confirm::valid_summary(&request.summary)
4926 && declared
4927 && crate::operator_confirm::ShortcutContext::route_eligible(
4928 &self.operator_app_policy,
4929 &self.launch_table,
4930 binding,
4931 )
4932 })
4933 .map(|binding| {
4934 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
4935 &self.operator_app_policy,
4936 binding,
4937 );
4938 let stamp = if phone.is_some() {
4939 None
4940 } else {
4941 binding.operator_app.clone()
4942 };
4943 self.shortcut_context(
4944 stamp,
4945 phone,
4946 Some((request.route_channel, request.route_epoch)),
4947 )
4948 });
4949 confirms.admit_with_shortcut(
4950 ctx,
4951 frame,
4952 module_id,
4953 nonce_proven,
4954 request,
4955 binding,
4956 shortcut,
4957 )
4958 },
4959 )
4960 .map_err(RouterError::Forwarding)
4961 }
4962
4963 fn shortcut_context(
4964 &self,
4965 stamp: Option<crate::operator_apps::OperatorAppStamp>,
4966 phone_fingerprint: Option<String>,
4967 route: Option<(u16, u32)>,
4968 ) -> crate::operator_confirm::ShortcutContext {
4969 crate::operator_confirm::ShortcutContext {
4970 ledger: self.activity.clone(),
4971 forwarding: self.forwarding.clone(),
4972 launches: self.launch_table.clone(),
4973 stamp,
4974 phone_fingerprint,
4975 route,
4976 connections: self.operator_apps.clone(),
4977 #[cfg(test)]
4978 hooks: Default::default(),
4979 }
4980 }
4981
4982 #[cfg(any(test, feature = "test-support"))]
4984 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
4985 self.operator_app_policy.phone_enabled = enabled;
4986 self
4987 }
4988
4989 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
4992 let mut scopes = ScopeRecords::from(scopes);
4993 for (index, record) in scopes.records.iter_mut().enumerate() {
4994 if let Some(refusal) = Self::canonicalize_workdir(record) {
4995 scopes.refusals.insert(index, refusal);
4996 }
4997 }
4998 scopes
4999 }
5000
5001 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
5002 let workdir = record.workdir.as_mut()?;
5003 let path = std::path::Path::new(workdir);
5004 if !path.is_absolute() {
5005 return Some((
5006 error_codes::INVALID_WORKDIR,
5007 "workdir must be absolute".to_string(),
5008 ));
5009 }
5010 match ProjectRootId::from_path_allowing_missing(path) {
5011 Ok(root) => match root.as_path().to_str() {
5012 Some(canonical) => {
5013 *workdir = canonical.to_string();
5014 None
5015 }
5016 None => Some((
5017 error_codes::INVALID_WORKDIR,
5018 "canonical workdir is not UTF-8".to_string(),
5019 )),
5020 },
5021 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
5022 }
5023 }
5024
5025 fn handle_scope_sync(
5029 &self,
5030 connection_id: ConnectionId,
5031 frame: Frame,
5032 generation: u64,
5033 scopes: Vec<ScopeRecord>,
5034 ) -> Result<Vec<Frame>, RouterError> {
5035 self.handle_scope_change(connection_id, frame, generation, scopes, None)
5036 }
5037
5038 fn handle_scope_change(
5039 &self,
5040 connection_id: ConnectionId,
5041 frame: Frame,
5042 generation: u64,
5043 scopes: Vec<ScopeRecord>,
5044 end: Option<Vec<ScopeEnd>>,
5045 ) -> Result<Vec<Frame>, RouterError> {
5046 let is_apply = end.is_some();
5047 let Some(registration) = self
5048 .registry
5049 .get_module_by_connection(connection_id)
5050 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5051 else {
5052 return Ok(vec![control_error_frame(
5053 &frame,
5054 "not_registered",
5055 "scope.sync and scope.apply require an active module registration owned by this connection",
5056 )?]);
5057 };
5058 let owner = registration.manifest.module_id;
5059 let scopes = Self::prepare_scope_records(scopes);
5060 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
5061 let is_current_launch = |connection: ConnectionId| {
5062 self.connection_is_current_launch(&owner, connection, current_nonce.as_deref())
5063 };
5064 let mut table = self
5069 .scopes
5070 .write()
5071 .unwrap_or_else(|poisoned| poisoned.into_inner());
5072 let outcome = match end {
5073 Some(end) => table.apply(
5074 &owner,
5075 connection_id,
5076 is_current_launch,
5077 generation,
5078 scopes,
5079 end,
5080 ),
5081 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
5082 };
5083 let drained = match &outcome {
5084 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
5085 Err(_) => Vec::new(),
5086 };
5087 let has_deadlines = outcome.is_ok() && table.has_deadlines();
5088 drop(table);
5089 if has_deadlines {
5090 self.scope_deadline_added.notify_one();
5091 }
5092 match outcome {
5093 Ok(applied) => {
5094 let counts = ScopeOutcomeCounts::of(&applied.results);
5095 info!(
5096 owner = %owner,
5097 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5098 generation,
5099 records = applied.results.len(),
5100 created = counts.created,
5101 replaced = counts.replaced,
5102 updated = counts.updated,
5103 unchanged = counts.unchanged,
5104 refused = counts.refused,
5105 ended = applied.ended.len(),
5106 tag_changes = applied.tag_changes.len(),
5107 routes_closed = drained.len(),
5108 "scope change accepted"
5109 );
5110 for refused in applied
5116 .results
5117 .iter()
5118 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
5119 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
5120 {
5121 warn!(
5122 owner = %owner,
5123 generation,
5124 scope_ref = %refused.scope_ref,
5125 scope_epoch = refused.scope_epoch,
5126 code = refused.code.as_deref().unwrap_or(""),
5127 "scope record refused"
5128 );
5129 }
5130 self.close_scope_drained_routes(drained);
5131 let response = if is_apply {
5132 ModuleControlResponseToModule::ScopeApply {
5133 generation,
5134 results: applied.results,
5135 end_results: applied.end_results,
5136 ended: applied.ended,
5137 }
5138 } else {
5139 ModuleControlResponseToModule::ScopeSync {
5140 generation,
5141 results: applied.results,
5142 ended: applied.ended,
5143 }
5144 };
5145 Ok(vec![control_response_body_frame(
5146 &frame,
5147 &response,
5148 "scope change response",
5149 )?])
5150 }
5151 Err(refusal) => {
5152 info!(
5153 owner = %owner,
5154 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5155 generation,
5156 code = refusal.code,
5157 "scope change refused"
5158 );
5159 Ok(vec![control_error_frame(
5160 &frame,
5161 refusal.code,
5162 refusal.message,
5163 )?])
5164 }
5165 }
5166 }
5167
5168 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
5170 scope_expiry::ScopeExpiryContext::new(self).close_scope_drained_routes(drained);
5171 }
5172
5173 fn handle_route_authorize(
5179 &self,
5180 connection_id: ConnectionId,
5181 frame: Frame,
5182 principal: Principal,
5183 scope: subc_protocol::scope::RouteAuthorizeScope,
5184 target: String,
5185 expected: subc_protocol::scope::RouteAuthorizeExpected,
5186 ) -> Result<Vec<Frame>, RouterError> {
5187 use crate::scopes::admission::{admit_route, AdmissionMode};
5188 use subc_protocol::scope::{
5189 RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
5190 };
5191 let registry = self
5192 .registry
5193 .route_admission_snapshot()
5194 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5195 let owner_configured = match &scope.owner {
5196 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5197 _ => false,
5198 };
5199 let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
5200 let answer = {
5201 let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
5202 let permitted = registry
5203 .iter()
5204 .find(|r| r.connection_id == connection_id)
5205 .is_some_and(|r| {
5206 scopes.is_federated_owner(&Principal::Reserved {
5207 module_id: r.manifest.module_id.clone(),
5208 }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
5209 caps.provides
5210 .iter()
5211 .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
5212 })
5213 });
5214 if !permitted {
5215 drop(scopes);
5216 return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
5217 "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
5218 }
5219 admit_route(
5220 &scopes,
5221 ®istry,
5222 &principal,
5223 &target,
5224 Some(&selector),
5225 owner_configured,
5226 AdmissionMode::Authorize(&expected),
5227 )
5228 };
5229 let response = match answer {
5230 Ok(admission) => {
5231 let mut response = RouteAuthorizeResponse::new(
5232 RouteAuthorizeOutcome::Authorized,
5233 "route admission authorized",
5234 );
5235 response.content_version = admission.content_version;
5236 response
5237 }
5238 Err(refusal) => refusal.response(),
5239 };
5240 Ok(vec![control_response_body_frame(
5241 &frame,
5242 &response,
5243 "RouteAuthorizeResponse",
5244 )?])
5245 }
5246
5247 fn handle_scope_describe(
5250 &self,
5251 connection_id: ConnectionId,
5252 frame: Frame,
5253 owner: Principal,
5254 scope_ref: String,
5255 ) -> Result<Vec<Frame>, RouterError> {
5256 let registered = self
5257 .registry
5258 .get_module_by_connection(connection_id)
5259 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5260 let Some(registered) = registered else {
5261 return Ok(vec![control_error_frame(
5262 &frame,
5263 "not_registered",
5264 "scope.describe requires an active module registration owned by this connection",
5265 )?]);
5266 };
5267 let scopes = self
5268 .scopes
5269 .read()
5270 .unwrap_or_else(|poisoned| poisoned.into_inner());
5271 if scopes.is_federated_owner(&owner)
5272 && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
5273 {
5274 return Ok(vec![control_error_frame(
5275 &frame,
5276 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
5277 "scope.describe of the federated owner requires scope-federated/v1",
5278 )?]);
5279 }
5280 let mut description = scopes.describe(&owner, &scope_ref);
5281 drop(scopes);
5282 if owner
5283 != (Principal::Reserved {
5284 module_id: registered.manifest.module_id.clone(),
5285 })
5286 {
5287 if let Some(facts) = description
5288 .stamp
5289 .as_mut()
5290 .and_then(|stamp| stamp.attributes.federated.as_mut())
5291 {
5292 facts.stamp_required = false;
5293 }
5294 }
5295 if !registered
5298 .manifest
5299 .capabilities
5300 .as_ref()
5301 .is_some_and(|caps| {
5302 caps.provides
5303 .iter()
5304 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5305 })
5306 {
5307 if let Some(stamp) = description.stamp.as_mut() {
5308 stamp.attributes.on_behalf_of_agent = None;
5309 }
5310 }
5311 if !registered
5312 .manifest
5313 .capabilities
5314 .as_ref()
5315 .is_some_and(|caps| {
5316 caps.provides
5317 .iter()
5318 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5319 })
5320 {
5321 if let Some(stamp) = description.stamp.as_mut() {
5322 stamp.attributes.flow_version = None;
5323 stamp.attributes.flow_code_hash = None;
5324 stamp.attributes.flow_install_id = None;
5325 }
5326 }
5327 let owner_configured = match &owner {
5328 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5333 _ => false,
5334 };
5335 let response = ModuleControlResponseToModule::ScopeDescribe {
5336 status: description.status,
5337 scope_epoch: description.scope_epoch,
5338 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
5339 owner_synced: description.owner_synced,
5340 owner_configured,
5341 scope: description.stamp,
5342 };
5343 Ok(vec![control_response_body_frame(
5344 &frame,
5345 &response,
5346 "ModuleControlResponseToModule::ScopeDescribe",
5347 )?])
5348 }
5349
5350 fn handle_catalog_update_role_declarations(
5351 &self,
5352 connection_id: ConnectionId,
5353 frame: Frame,
5354 role_declarations: serde_json::Value,
5355 ) -> Result<Vec<Frame>, RouterError> {
5356 if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
5357 return Ok(vec![control_error_frame(
5358 &frame,
5359 "invalid_role_declarations",
5360 err.to_string(),
5361 )?]);
5362 }
5363 let declarations = serde_json::from_value(role_declarations)
5364 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5365 if self
5366 .registry
5367 .replace_role_declarations_for_connection(connection_id, declarations)
5368 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5369 .is_none()
5370 {
5371 return Ok(vec![control_error_frame(
5372 &frame,
5373 "not_registered",
5374 "catalog.update_role_declarations requires an active module registration owned by this connection",
5375 )?]);
5376 }
5377 control_response_body_frame(
5378 &frame,
5379 &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
5380 "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
5381 )
5382 .map(|frame| vec![frame])
5383 }
5384
5385 fn handle_catalog_update(
5386 &self,
5387 connection_id: ConnectionId,
5388 frame: Frame,
5389 provides: Vec<ProviderRole>,
5390 capabilities: Option<CapabilityDeclarations>,
5391 ready: Option<bool>,
5392 ) -> Result<Vec<Frame>, RouterError> {
5393 self.refresh_capability_requirements();
5394 let Some(registration) = self
5395 .registry
5396 .get_module_by_connection(connection_id)
5397 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5398 else {
5399 return Ok(vec![control_error_frame(
5400 &frame,
5401 "not_registered",
5402 "catalog.update requires an active module registration owned by this connection",
5403 )?]);
5404 };
5405
5406 if let Some(message) =
5407 catalog_update_frozen_field_message(®istration.manifest, &provides)
5408 {
5409 return Ok(vec![control_error_frame(
5410 &frame,
5411 "catalog_update_frozen_field",
5412 message,
5413 )?]);
5414 }
5415
5416 let mut candidate = registration.manifest.clone();
5417 candidate.provides = provides.clone();
5418 candidate.capabilities = capabilities
5419 .clone()
5420 .or_else(|| registration.manifest.capabilities.clone());
5421 if let Some(rich_ui) = &candidate.rich_ui {
5422 if let Err(err) = rich_ui.validate() {
5423 return Ok(vec![control_error_frame(
5424 &frame,
5425 "invalid_manifest",
5426 err.to_string(),
5427 )?]);
5428 }
5429 }
5430 if let Err(err) = candidate.validate_capability_grammar() {
5431 return Ok(vec![control_error_frame(
5432 &frame,
5433 "invalid_capability_grammar",
5434 err.to_string(),
5435 )?]);
5436 }
5437
5438 let mut conflicts = self
5441 .capability_evaluator
5442 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
5443 if let Some(conflict) = conflicts.first() {
5444 let message = format!(
5445 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
5446 conflict.capability, conflict.claimants[0], candidate.module_id
5447 );
5448 for conflict in &mut conflicts {
5449 conflict.source = DuplicateClaimSource::CatalogUpdate;
5450 }
5451 log_duplicate_claim_events(conflicts);
5452 return Ok(vec![control_error_frame(
5453 &frame,
5454 "reserved_capability",
5455 message,
5456 )?]);
5457 }
5458
5459 let updated = match self.registry.replace_catalog_for_connection(
5460 connection_id,
5461 provides,
5462 capabilities,
5463 ready,
5464 ) {
5465 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => {
5466 return Ok(vec![control_error_frame(
5467 &frame,
5468 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5469 "scope-federated/v1 is fixed at HELLO; restart the module to change it",
5470 )?])
5471 }
5472 Err(crate::registry::RegistryError::CallerAttestationCapabilityChanged) => {
5473 return Ok(vec![control_error_frame(
5474 &frame,
5475 error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5476 "caller-attestation/v1 is fixed at HELLO; restart the module to change it",
5477 )?])
5478 }
5479 result => {
5480 result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5481 }
5482 };
5483 if updated.is_none() {
5484 return Ok(vec![control_error_frame(
5485 &frame,
5486 "not_registered",
5487 "catalog.update requires an active module registration owned by this connection",
5488 )?]);
5489 }
5490 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
5491 log_duplicate_claim_events(
5492 self.capability_evaluator
5493 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
5494 );
5495 }
5496 if capability_census_trigger(
5497 registration.manifest.capabilities.as_ref(),
5498 updated
5499 .as_ref()
5500 .and_then(|entry| entry.manifest.capabilities.as_ref()),
5501 ) {
5502 self.enforce_capability_denies();
5503 }
5504 self.refresh_capability_requirements();
5505
5506 let response = ModuleControlResponseToModule::CatalogUpdate {};
5507 control_response_body_frame(
5508 &frame,
5509 &response,
5510 "ModuleControlResponseToModule::CatalogUpdate",
5511 )
5512 .map(|frame| vec![frame])
5513 }
5514
5515 pub(crate) fn with_activity_ledger_config(
5516 mut self,
5517 run_dir: &Path,
5518 incarnation: String,
5519 config: crate::activity::ActivityConfig,
5520 ) -> Self {
5521 self.activity = Some(crate::activity::ActivityLedger::open_config(
5522 run_dir.join("activity.jsonl"),
5523 incarnation,
5524 config,
5525 ));
5526 self.supervisor.set_activity(self.activity.clone());
5527 self
5528 }
5529
5530 pub(crate) async fn configure_activity(
5531 &self,
5532 config: crate::activity::ActivityConfig,
5533 ) -> Result<(), &'static str> {
5534 match &self.activity {
5535 Some(ledger) => ledger.configure(config).await,
5536 None => Ok(()),
5537 }
5538 }
5539
5540 fn cancel_activity_subscription(
5541 &self,
5542 ctx: &RouteCtx,
5543 frame: &Frame,
5544 ) -> Result<bool, RouterError> {
5545 let task = self
5546 .activity_streams
5547 .lock()
5548 .unwrap_or_else(|p| p.into_inner())
5549 .remove(&(ctx.connection_id, frame.header.corr));
5550 let Some(task) = task else {
5551 return Ok(false);
5552 };
5553 task.abort();
5554 let end = Frame::build_with_version(
5555 response_version(frame),
5556 FrameType::StreamEnd,
5557 control_flags(),
5558 0,
5559 0,
5560 frame.header.corr,
5561 Vec::new(),
5562 )
5563 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
5564 let sink = ctx.egress.clone();
5565 tokio::spawn(async move {
5566 let _ = sink.send(end).await;
5567 });
5568 Ok(true)
5569 }
5570
5571 async fn handle_activity_read(
5572 &self,
5573 ctx: &RouteCtx,
5574 frame: Frame,
5575 ) -> Result<Vec<Frame>, RouterError> {
5576 #[cfg(test)]
5577 {
5578 let pause = self
5579 .activity_request_pause
5580 .lock()
5581 .unwrap()
5582 .clone()
5583 .filter(|pause| pause.connection == ctx.connection_id);
5584 if let Some(pause) = pause {
5585 let _cancelled = ActivityRequestCancellation(pause.clone());
5586 pause.started.notify_one();
5587 std::future::pending::<()>().await;
5588 }
5589 }
5590 use subc_control::{
5591 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
5592 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
5593 };
5594 #[derive(Deserialize)]
5595 struct Identity {
5596 #[serde(default)]
5597 consumer_identity: Option<ConsumerIdentity>,
5598 }
5599 let mut bound_spawn = None;
5603 let legacy = match self.activity_reader(ctx.connection_id) {
5604 channel_operations::ActivityReader::Module { module_id, spawn } => {
5605 bound_spawn = Some(spawn);
5606 Some(Ok(Principal::Reserved { module_id }))
5607 }
5608 channel_operations::ActivityReader::Refused => {
5610 return Ok(vec![control_error_frame(
5611 &frame,
5612 ACTIVITY_READ_FORBIDDEN,
5613 "helper, relay and retired connections cannot read activity",
5614 )?]);
5615 }
5616 channel_operations::ActivityReader::Legacy => None,
5617 };
5618 let principal = match legacy {
5622 Some(principal) => principal,
5623 None => match self
5624 .forwarding
5625 .module_id_for_connection(ctx.connection_id)
5626 .map_err(RouterError::Forwarding)?
5627 {
5628 Some(module_id) => Ok(Principal::Reserved { module_id }),
5629 None => {
5630 let identity: Identity = match serde_json::from_slice(&frame.body) {
5631 Ok(identity) => identity,
5632 Err(_) => {
5633 return Ok(vec![control_error_frame(
5634 &frame,
5635 ACTIVITY_MALFORMED,
5636 "malformed activity read",
5637 )?])
5638 }
5639 };
5640 self.route_open_principal(
5641 ctx.connection_id,
5642 &frame,
5643 identity.consumer_identity,
5644 )?
5645 }
5646 },
5647 };
5648 let admitted = matches!(principal, Ok(Principal::Direct))
5649 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
5650 if !admitted {
5651 return Ok(vec![control_error_frame(
5652 &frame,
5653 ACTIVITY_READ_FORBIDDEN,
5654 "activity reads require an operator or trusted mirror principal",
5655 )?]);
5656 }
5657 let Some(ledger) = &self.activity else {
5658 return Ok(vec![control_error_frame(
5659 &frame,
5660 ACTIVITY_UNAVAILABLE,
5661 "activity ledger unavailable",
5662 )?]);
5663 };
5664 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
5665 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
5666 {
5667 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
5668 Ok(request) => ledger.query(request).await,
5669 Err(_) => Err(ACTIVITY_MALFORMED),
5670 };
5671 return match result {
5672 Ok(reply) => Ok(vec![control_response_body_frame(
5673 &frame,
5674 &reply,
5675 ops::ACTIVITY_QUERY,
5676 )?]),
5677 Err(code) => Ok(vec![control_error_frame(
5678 &frame,
5679 code,
5680 "activity query refused",
5681 )?]),
5682 };
5683 }
5684 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
5685 Ok(request) => request,
5686 Err(_) => {
5687 return Ok(vec![control_error_frame(
5688 &frame,
5689 ACTIVITY_MALFORMED,
5690 "malformed activity subscription",
5691 )?])
5692 }
5693 };
5694 let mut subscription = match ledger.subscribe(request.since).await {
5695 Ok(subscription) => subscription,
5696 Err(code) => {
5697 return Ok(vec![control_error_frame(
5698 &frame,
5699 code,
5700 "activity subscription refused",
5701 )?])
5702 }
5703 };
5704 let key = (ctx.connection_id, frame.header.corr);
5705 let sink = ctx.egress.clone();
5706 let mut streams = self
5707 .activity_streams
5708 .lock()
5709 .unwrap_or_else(|p| p.into_inner());
5710 streams.retain(|_, task| !task.is_finished());
5711 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
5712 return Ok(vec![control_error_frame(
5713 &frame,
5714 "duplicate_subscription",
5715 "activity subscription correlation id already in use",
5716 )?]);
5717 }
5718 let stream = async move {
5719 loop {
5720 let entry = if let Some(entry) = subscription.replay.pop_front() {
5721 Some(entry)
5722 } else {
5723 subscription.receiver.recv().await
5724 };
5725 let Some(entry) = entry else {
5726 break;
5727 };
5728 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
5729 return;
5730 };
5731 let Ok(next) = Frame::build_with_version(
5732 response_version(&frame),
5733 FrameType::StreamData,
5734 control_flags(),
5735 0,
5736 0,
5737 frame.header.corr,
5738 body,
5739 ) else {
5740 return;
5741 };
5742 if sink.send(next).await.is_err() {
5743 return;
5744 }
5745 }
5746 if let Some(code) = subscription.terminal_code() {
5749 if let Ok(end) =
5750 control_error_frame(&frame, code, "activity subscriber fell behind")
5751 {
5752 let _ = sink.send(end).await;
5753 }
5754 }
5755 };
5756 let task = tokio::spawn(async move {
5760 match bound_spawn {
5761 Some(spawn) => {
5762 tokio::select! {
5763 () = stream => {}
5764 () = spawn.wait_closed() => {}
5765 }
5766 }
5767 None => stream.await,
5768 }
5769 });
5770 streams.insert(key, task.abort_handle());
5771 Ok(Vec::new())
5772 }
5773
5774 async fn handle_activity_record(
5775 &self,
5776 ctx: &RouteCtx,
5777 frame: Frame,
5778 ) -> Result<Vec<Frame>, RouterError> {
5779 use subc_control::{
5780 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
5781 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
5782 };
5783 let result = async {
5784 let module = self
5785 .forwarding
5786 .module_id_for_connection(ctx.connection_id)
5787 .ok()
5788 .flatten()
5789 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
5790 let request: ActivityRecordRequest =
5791 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
5792 crate::activity::validate(&module, &request)?;
5793 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
5794 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
5795 let at_ms = std::time::SystemTime::now()
5796 .duration_since(std::time::UNIX_EPOCH)
5797 .unwrap_or_default()
5798 .as_millis() as u64;
5799 ledger.record(module, request, actor, at_ms).await
5800 }
5801 .await;
5802 match result {
5803 Ok(entry) => Ok(vec![control_response_body_frame(
5804 &frame,
5805 &ActivityRecordResponse { seq: entry.seq },
5806 "activity.record",
5807 )?]),
5808 Err(code) => Ok(vec![control_error_frame(
5809 &frame,
5810 code,
5811 "activity recording refused",
5812 )?]),
5813 }
5814 }
5815
5816 fn activity_actor(
5817 &self,
5818 connection: ConnectionId,
5819 module: &str,
5820 on: Option<&subc_control::ActivityOn>,
5821 ) -> Result<subc_control::ActivityActor, &'static str> {
5822 use crate::forwarding::{DataRoute, DataRouteState};
5823 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
5824 let principal = |id: &str| Principal::Reserved {
5825 module_id: id.to_owned(),
5826 };
5827 let mut actor = ActivityActor {
5828 principal: principal(module),
5829 scope_owner: None,
5830 agent_id: None,
5831 run_id: None,
5832 flow_id: None,
5833 project_root: None,
5834 workdir: None,
5835 project_id: None,
5836 };
5837 let scope = match on {
5838 None => return Ok(actor),
5839 Some(ActivityOn::Scope(scope)) => {
5840 let selector = subc_protocol::scope::ScopeSelector::new(
5841 principal(module),
5842 scope.scope_ref.clone(),
5843 Some(scope.epoch),
5844 );
5845 self.scopes
5846 .read()
5847 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5848 .attribute(&principal(module), module, &selector, true)
5849 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5850 }
5851 Some(ActivityOn::Route(route)) => {
5852 let DataRoute::Module(DataRouteState::Bound(binding)) = self
5853 .forwarding
5854 .lookup_data_route(connection, route.channel, route.epoch)
5855 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5856 else {
5857 return Err(ACTIVITY_ACTOR_UNKNOWN);
5858 };
5859 if binding.module_id != module {
5860 return Err(ACTIVITY_ACTOR_UNKNOWN);
5861 }
5862 actor.principal = binding.principal.clone();
5863 actor.project_root = binding
5864 .project_root
5865 .as_ref()
5866 .map(|root| root.as_path().to_path_buf());
5867 let Some(bound) = &binding.scope else {
5868 return Ok(actor);
5869 };
5870 let owner = principal(&bound.owner);
5871 let selector = subc_protocol::scope::ScopeSelector::new(
5872 owner.clone(),
5873 bound.scope_ref.clone(),
5874 Some(bound.tag.scope_epoch),
5875 );
5876 let admitted = self
5877 .scopes
5878 .read()
5879 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5880 .attribute(&owner, module, &selector, true)
5881 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
5882 if admitted.tag != bound.tag {
5883 return Err(ACTIVITY_ACTOR_UNKNOWN);
5884 }
5885 admitted
5886 }
5887 };
5888 actor.scope_owner = Some(scope.stamp.owner);
5889 actor.agent_id = scope.stamp.attributes.agent_id;
5890 actor.run_id = scope.stamp.attributes.run_id;
5891 actor.flow_id = scope.stamp.attributes.flow_id;
5892 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
5893 actor.project_id = scope.stamp.project_id;
5894 Ok(actor)
5895 }
5896
5897 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5898 self.refresh_capability_requirements();
5899 let mut counters = self.counters.snapshot();
5905 let losses = self.resource_tracker.loss_diagnostics();
5906 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
5907 if let Some(obj) = counters.as_object_mut() {
5908 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
5909 }
5910 }
5911 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
5912 obj.insert(
5913 "activity_unparseable_lines".into(),
5914 ledger.read_errors().into(),
5915 );
5916 obj.insert(
5917 "activity_lifecycle_append_failures".into(),
5918 ledger.lifecycle_failures().into(),
5919 );
5920 obj.insert(
5921 "activity_started_digest_skips".into(),
5922 ledger.lifecycle_digest_skips().into(),
5923 );
5924 }
5925 if let (Ok((connections_with_routes, max)), Some(obj)) = (
5926 self.forwarding.client_route_concentration(),
5927 counters.as_object_mut(),
5928 ) {
5929 obj.insert(
5930 "client_connections_with_routes".into(),
5931 connections_with_routes.into(),
5932 );
5933 obj.insert("max_routes_on_one_connection".into(), max.into());
5934 }
5935 if let (Some(open_breakers), Some(obj)) = (
5941 self.route_bind_breakers.open_snapshot(),
5942 counters.as_object_mut(),
5943 ) {
5944 obj.insert("route_bind_breakers_open".into(), open_breakers);
5945 }
5946 if let (Some(reason), Some(obj)) = (
5951 self.local_endpoint_unavailable.clone(),
5952 counters.as_object_mut(),
5953 ) {
5954 obj.insert("local_endpoint_unavailable".into(), reason.into());
5955 }
5956 let response = ClientControlResponse::ServerDescribe {
5957 protocol_ver: PROTOCOL_VERSION,
5958 subc_ops: subc_ops(),
5959 capabilities: self.describe_capabilities(),
5960 connected_clients: self.connected_clients.count(),
5961 counters: Some(counters),
5962 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
5963 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
5964 capability_requirements: self.capability_requirement_statuses(),
5965 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
5966 config_path: self
5967 .rescan
5968 .as_ref()
5969 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
5970 };
5971 Ok(vec![control_response_body_frame(
5972 &frame,
5973 &response,
5974 "ClientControlResponse::ServerDescribe",
5975 )?])
5976 }
5977
5978 fn handle_catalog_list(
5979 &self,
5980 frame: Frame,
5981 module_id: Option<String>,
5982 ) -> Result<Vec<Frame>, RouterError> {
5983 let (generation, modules) = self
5984 .registry
5985 .list_modules_with_nonce_verification()
5986 .map_err(|err| {
5987 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
5988 })?;
5989 let entries = modules
5990 .into_iter()
5991 .filter(|(registration, _)| {
5992 module_id
5993 .as_deref()
5994 .map(|wanted| registration.manifest.module_id == wanted)
5995 .unwrap_or(true)
5996 })
5997 .map(|(registration, nonce_verified)| {
5998 let entry = self
5999 .catalog_discovery_entry(®istration, module_id.is_some())
6000 .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
6001 let not_ready = self.not_ready_reason(®istration);
6002 let mut entry = entry;
6003 entry.roles = registration.manifest.provides;
6004 entry.control_ops = registration.control_ops;
6005 Ok(entry
6006 .with_ready(not_ready.is_none())
6007 .with_display_name(registration.manifest.display_name)
6008 .with_not_ready(not_ready)
6009 .with_module_version(Some(registration.manifest.module_version))
6010 .with_provenance(registration.manifest.provenance)
6011 .with_capabilities(registration.manifest.capabilities)
6012 .with_self_signals(registration.manifest.self_signals)
6013 .with_events(registration.manifest.events)
6014 .with_operation_delivery(registration.manifest.operation_delivery)
6015 .with_operation_payload(registration.manifest.operation_payload)
6016 .with_flow_operations(registration.manifest.flow_operations)
6017 .with_operation_deferral(registration.manifest.operation_deferral)
6018 .with_confirm_operations(registration.manifest.confirm_operations)
6019 .with_rich_ui(registration.manifest.rich_ui)
6020 .with_role_declarations(registration.manifest.role_declarations)
6021 .with_nonce_verified(nonce_verified))
6022 })
6023 .collect::<Result<Vec<_>, RouterError>>()?;
6024 let response = ClientControlResponse::CatalogList {
6025 generation,
6026 modules: entries,
6027 subc_ops: subc_ops(),
6028 };
6029 Ok(vec![control_response_body_frame(
6030 &frame,
6031 &response,
6032 "ClientControlResponse::CatalogList",
6033 )?])
6034 }
6035
6036 fn route_open_caller_from_frame(&self, ctx: &RouteCtx, frame: &Frame) -> RouteOpenCaller {
6040 let Ok(ClientControlRequest::RouteOpen {
6041 identity,
6042 consumer_identity,
6043 scope,
6044 ..
6045 }) = parse_client_control_request(&frame.body)
6046 else {
6047 return RouteOpenCaller::unknown();
6048 };
6049 let principal = self
6050 .attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref())
6051 .map(|attested| attested.principal);
6052 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
6053 }
6054
6055 fn route_open_principal(
6056 &self,
6057 connection_id: ConnectionId,
6058 frame: &Frame,
6059 consumer_identity: Option<ConsumerIdentity>,
6060 ) -> Result<Result<Principal, Frame>, RouterError> {
6061 let attested = self
6062 .attest_route_open_consumer(connection_id, consumer_identity.as_ref())
6063 .map(|attested| attested.principal);
6064 self.route_open_principal_attested(frame, consumer_identity, attested)
6065 }
6066
6067 fn route_open_principal_attested(
6071 &self,
6072 frame: &Frame,
6073 consumer_identity: Option<ConsumerIdentity>,
6074 attested: Option<Principal>,
6075 ) -> Result<Result<Principal, Frame>, RouterError> {
6076 if let Some(principal) = attested {
6077 return Ok(Ok(principal));
6078 }
6079 Ok(Err(control_error_frame(
6080 frame,
6081 "bad_consumer_identity",
6082 consumer_identity.map_or_else(
6083 || "connection claim is no longer authorized for routes".to_string(),
6084 |identity| format!(
6085 "consumer_identity for module_id '{}' conflicts with its connection claim or did not match a supervised launch nonce",
6086 identity.module_id
6087 ),
6088 ),
6089 )?))
6090 }
6091
6092 #[allow(clippy::too_many_arguments)]
6099 fn route_open_refusal_frame(
6100 &self,
6101 ctx: &RouteCtx,
6102 caller: &RouteOpenCaller,
6103 frame: &Frame,
6104 module_id: &str,
6105 reason: &'static str,
6106 code: &'static str,
6107 message: impl Into<String>,
6108 ) -> Result<Frame, RouterError> {
6109 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
6110 let detail = if matches!(
6111 reason,
6112 "registration_not_active"
6113 | "supervisor_not_live"
6114 | "no_forwarding_connection"
6115 | "reloading"
6116 ) {
6117 let expected_back = (reason == "reloading").then_some(true);
6120 self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
6121 } else {
6122 None
6123 };
6124 control_error_body_frame(
6125 frame,
6126 ErrorBody {
6127 code: code.to_string(),
6128 message: message.into(),
6129 detail,
6130 },
6131 )
6132 }
6133
6134 fn route_open_recovery_detail(
6135 &self,
6136 module_id: &str,
6137 corr: u64,
6138 detail: Option<serde_json::Value>,
6139 expected_back: Option<bool>,
6140 ) -> Result<Option<serde_json::Value>, RouterError> {
6141 let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
6142 return Ok(detail);
6143 };
6144 let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
6145 detail["state"] = serde_json::json!(status.state.to_string());
6146 if let Some(expected_back) = expected_back {
6147 detail["expected_back"] = serde_json::json!(expected_back);
6148 } else if let Some(module) = self.supervisor.get(module_id) {
6149 match module.will_recover_after_connection_loss() {
6150 Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
6151 Err(err) => warn!(
6152 %module_id,
6153 error = %err,
6154 "failed to read route.open recovery verdict; omitting expected_back"
6155 ),
6156 }
6157 }
6158 Ok(Some(detail))
6159 }
6160
6161 #[allow(clippy::too_many_arguments)]
6170 fn route_bind_rejection(
6171 &self,
6172 ctx: &RouteCtx,
6173 caller: &RouteOpenCaller,
6174 frame: &Frame,
6175 target_module_id: &str,
6176 body: ErrorBody,
6177 daemon_authored: bool,
6178 ) -> Result<Vec<Frame>, RouterError> {
6179 if daemon_authored {
6185 let scope_code = match body.code.as_str() {
6186 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
6187 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
6188 _ => None,
6189 };
6190 if let Some(code) = scope_code {
6191 self.observe_route_open_refusal(
6192 ctx,
6193 caller,
6194 target_module_id,
6195 "scope_changed_before_commit",
6196 code,
6197 );
6198 return Ok(vec![control_error_body_frame(frame, body)?]);
6199 }
6200 }
6201 self.counters
6202 .increment_route_open_refused("module_rejected");
6203 info!(
6204 target: "control",
6205 code = "module_rejected",
6206 module_code = ?body.code,
6207 module_id = ?target_module_id,
6208 connection_id = ctx.connection_id.get(),
6209 principal = %caller.principal,
6210 scope = %caller.scope,
6211 harness = %caller.harness,
6212 "route.open refused"
6213 );
6214 let refusal = if daemon_authored {
6215 control_error_body_frame(frame, body)?
6216 } else {
6217 control_relayed_error_body_frame(frame, body)?
6218 };
6219 Ok(vec![refusal])
6220 }
6221
6222 #[allow(clippy::too_many_arguments)]
6240 fn route_open_breaker_refusal_frame(
6241 &self,
6242 ctx: &RouteCtx,
6243 caller: &RouteOpenCaller,
6244 frame: &Frame,
6245 module_id: &str,
6246 consecutive_timeouts: u32,
6247 retry_in: Duration,
6248 probe_in_flight: bool,
6249 ) -> Result<Frame, RouterError> {
6250 self.counters
6251 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
6252 debug!(
6253 target: "control",
6254 code = "module_timeout",
6255 module_id = ?module_id,
6256 connection_id = ctx.connection_id.get(),
6257 principal = %caller.principal,
6258 scope = %caller.scope,
6259 harness = %caller.harness,
6260 consecutive_timeouts,
6261 retry_in_ms = retry_in.as_millis() as u64,
6262 probe_in_flight,
6263 "route.open refused by open bind-relay breaker"
6264 );
6265 let detail = if probe_in_flight {
6272 "one test route is already being tried; retry once it settles".to_string()
6273 } else {
6274 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
6275 };
6276 control_error_frame(
6277 frame,
6278 "module_timeout",
6279 format!(
6280 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
6281 timed out in a row); {detail}; its established routes are unaffected"
6282 ),
6283 )
6284 }
6285
6286 fn observe_route_open_refusal(
6299 &self,
6300 ctx: &RouteCtx,
6301 caller: &RouteOpenCaller,
6302 module_id: &str,
6303 reason: &'static str,
6304 code: &'static str,
6305 ) {
6306 self.counters.increment_route_open_refused(code);
6307 info!(
6308 target: "control",
6309 code,
6310 reason,
6311 module_id = ?module_id,
6312 connection_id = ctx.connection_id.get(),
6313 principal = %caller.principal,
6314 scope = %caller.scope,
6315 harness = %caller.harness,
6316 "route.open refused"
6317 );
6318 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
6319 self.route_outages.record_not_serving(module_id, reason);
6320 }
6321 }
6322
6323 fn observe_route_open_accept(
6368 &self,
6369 ctx: &RouteCtx,
6370 module_id: &str,
6371 principal: &str,
6372 scope: &str,
6373 ) {
6374 self.route_outages.record_accepted(module_id);
6375 self.counters.increment_route_open_accepted(principal);
6376 info!(
6377 target: "control",
6378 principal,
6379 module_id,
6380 connection_id = ctx.connection_id.get(),
6381 scope = %scope,
6382 "route.open accepted"
6383 );
6384 }
6385
6386 fn supervised_absent_route_open_refusal_frame(
6387 &self,
6388 ctx: &RouteCtx,
6389 caller: &RouteOpenCaller,
6390 frame: &Frame,
6391 module_id: &str,
6392 code: &'static str,
6393 status: &crate::supervise::ModuleStatus,
6394 ) -> Result<Frame, RouterError> {
6395 self.counters.increment_route_open_refused(code);
6396 info!(
6397 target: "control",
6398 code,
6399 reason = "supervised_not_registered",
6400 module_id = ?module_id,
6401 connection_id = ctx.connection_id.get(),
6402 principal = %caller.principal,
6403 scope = %caller.scope,
6404 harness = %caller.harness,
6405 state = %status.state,
6406 enabled = status.enabled,
6407 live = status.live,
6408 "route.open refused"
6409 );
6410 self.route_outages
6414 .record_not_serving(module_id, "supervised_not_registered");
6415 let message = if let Some(pid) = status.waiting_for_previous_pid {
6416 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
6417 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
6418 })
6419 } else {
6420 format!(
6421 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
6422 status.state, status.enabled, status.live
6423 )
6424 };
6425 let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
6429 control_error_body_frame(
6430 frame,
6431 ErrorBody {
6432 code: code.to_string(),
6433 message,
6434 detail: self.route_open_recovery_detail(
6435 module_id,
6436 frame.header.corr,
6437 None,
6438 expected_back,
6439 )?,
6440 },
6441 )
6442 }
6443
6444 async fn handle_route_open(
6445 &self,
6446 ctx: &RouteCtx,
6447 frame: Frame,
6448 request: RouteOpenRequest,
6449 timings: &mut RouteOpenTimings,
6450 ) -> Result<Vec<Frame>, RouterError> {
6451 let RouteOpenRequest {
6452 target,
6453 mut identity,
6454 consumer_identity,
6455 consumer_capabilities,
6456 role_versions,
6457 busy_refusals,
6458 admission_facts,
6459 scope,
6460 accepts_close_reasons,
6461 } = request;
6462 let target_module_id = target_module_id(&target).to_string();
6463 timings.module_id.clone_from(&target_module_id);
6464 if self
6465 .registry
6466 .get_module_by_connection(ctx.connection_id)
6467 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6468 .is_some()
6469 {
6470 return Ok(vec![control_error_frame(
6471 &frame,
6472 "invalid_request",
6473 "module connections cannot open client routes",
6474 )?]);
6475 }
6476 debug!(
6477 connection_id = ctx.connection_id.get(),
6478 corr = frame.header.corr,
6479 module_id = %target_module_id,
6480 "handling route.open"
6481 );
6482
6483 let attested =
6489 self.attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref());
6490 let mut caller = RouteOpenCaller::new(
6491 attested.as_ref().map(|attested| &attested.principal),
6492 scope.as_ref(),
6493 &identity,
6494 );
6495
6496 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
6502 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
6503 self.observe_route_open_refusal(
6504 ctx,
6505 &caller,
6506 &target_module_id,
6507 "invalid_role_versions",
6508 error_codes::INVALID_REQUEST,
6509 );
6510 return Ok(vec![control_error_body_frame(
6511 &frame,
6512 ErrorBody {
6513 code: error_codes::INVALID_REQUEST.to_string(),
6514 message: error.to_string(),
6515 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
6516 },
6517 )?]);
6518 }
6519
6520 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
6521 if status.waiting_for_previous_pid.is_some() {
6522 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6523 ctx,
6524 &caller,
6525 &frame,
6526 &target_module_id,
6527 "module_warming",
6528 &status,
6529 )?]);
6530 }
6531 }
6532
6533 let admission_registry = self
6552 .registry
6553 .route_admission_snapshot()
6554 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6555 let Some(registration) =
6556 crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
6557 else {
6558 if let Some((status, warming)) =
6559 self.supervisor_status(&target_module_id, frame.header.corr)?
6560 {
6561 if status.protocol == ModuleProtocol::None {
6568 return Ok(vec![self.route_open_refusal_frame(
6569 ctx,
6570 &caller,
6571 &frame,
6572 &target_module_id,
6573 "protocol_none",
6574 error_codes::MODULE_NO_PROTOCOL,
6575 format!(
6576 "module_id '{target_module_id}' is declared protocol: none; \
6577 it speaks no subc wire and serves no routes"
6578 ),
6579 )?]);
6580 }
6581 let code = if warming {
6582 "module_warming"
6583 } else {
6584 "target_unavailable"
6585 };
6586 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6587 ctx,
6588 &caller,
6589 &frame,
6590 &target_module_id,
6591 code,
6592 &status,
6593 )?]);
6594 }
6595 if let Some(removed_ago_ms) =
6596 self.supervisor.removal_tombstone_age_ms(&target_module_id)
6597 {
6598 return Ok(vec![self.route_open_refusal_frame(
6599 ctx,
6600 &caller,
6601 &frame,
6602 &target_module_id,
6603 "removed",
6604 error_codes::MODULE_REMOVED,
6605 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
6606 )?]);
6607 }
6608 return Ok(vec![self.route_open_refusal_frame(
6609 ctx,
6610 &caller,
6611 &frame,
6612 &target_module_id,
6613 "not_registered",
6614 error_codes::UNKNOWN_MODULE,
6615 format!("module_id '{target_module_id}' is not registered"),
6616 )?]);
6617 };
6618
6619 if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
6623 .is_err()
6624 {
6625 self.counters
6626 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
6627 info!(
6628 target: "control",
6629 code = error_codes::MODULE_WARMING,
6630 module_id = ?target_module_id,
6631 connection_id = ctx.connection_id.get(),
6632 principal = %caller.principal,
6633 scope = %caller.scope,
6634 harness = %caller.harness,
6635 reason = "declared_not_ready",
6636 "route.open refused"
6637 );
6638 self.route_outages
6641 .record_not_serving(&target_module_id, "declared_not_ready");
6642 return Ok(vec![control_error_body_frame(
6645 &frame,
6646 ErrorBody {
6647 code: error_codes::MODULE_WARMING.to_string(),
6648 message: format!(
6649 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
6650 ),
6651 detail: self.route_open_recovery_detail(
6652 &target_module_id,
6653 frame.header.corr,
6654 Some(serde_json::json!({ "reason": "declared_not_ready" })),
6655 Some(true),
6656 )?,
6657 },
6658 )?]);
6659 }
6660
6661 if let Some(capability) = self.first_unprovided_required_capability(registration) {
6680 self.counters
6681 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
6682 info!(
6683 target: "control",
6684 code = error_codes::MODULE_WARMING,
6685 module_id = ?target_module_id,
6686 connection_id = ctx.connection_id.get(),
6687 principal = %caller.principal,
6688 scope = %caller.scope,
6689 harness = %caller.harness,
6690 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6691 capability = %capability,
6692 "route.open refused"
6693 );
6694 return Ok(vec![control_error_body_frame(
6695 &frame,
6696 ErrorBody {
6697 code: error_codes::MODULE_WARMING.to_string(),
6698 message: format!(
6699 "module_id '{target_module_id}' requires capability '{capability}', \
6700 which no registered module provides; retry"
6701 ),
6702 detail: Some(serde_json::json!({
6703 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6704 "capability": capability,
6705 })),
6706 },
6707 )?]);
6708 }
6709
6710 if !target_has_required_role(&target, ®istration.manifest.provides) {
6711 return Ok(vec![self.route_open_refusal_frame(
6712 ctx,
6713 &caller,
6714 &frame,
6715 &target_module_id,
6716 "role_not_provided",
6717 "target_unavailable",
6718 format!("module_id '{target_module_id}' does not provide the requested target"),
6719 )?]);
6720 }
6721
6722 if registration.state != ChannelState::Active {
6723 return Ok(vec![self.route_open_refusal_frame(
6724 ctx,
6725 &caller,
6726 &frame,
6727 &target_module_id,
6728 "registration_not_active",
6729 "target_unavailable",
6730 format!("module_id '{target_module_id}' is not active"),
6731 )?]);
6732 }
6733
6734 if self
6735 .forwarding
6736 .module_is_draining(&target_module_id)
6737 .map_err(RouterError::Forwarding)?
6738 {
6739 return Ok(vec![self.route_open_refusal_frame(
6740 ctx,
6741 &caller,
6742 &frame,
6743 &target_module_id,
6744 "reloading",
6745 "module_reloading",
6746 format!("module_id '{target_module_id}' is reloading"),
6747 )?]);
6748 }
6749
6750 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
6751 process_liveness.process_live(&target_module_id) == Some(false)
6752 }) {
6753 if process_liveness.process_replacing(&target_module_id) {
6760 return Ok(vec![self.route_open_refusal_frame(
6761 ctx,
6762 &caller,
6763 &frame,
6764 &target_module_id,
6765 "reloading",
6766 "module_reloading",
6767 format!("module_id '{target_module_id}' is reloading"),
6768 )?]);
6769 }
6770 return Ok(vec![self.route_open_refusal_frame(
6771 ctx,
6772 &caller,
6773 &frame,
6774 &target_module_id,
6775 "supervisor_not_live",
6776 "target_unavailable",
6777 format!("module_id '{target_module_id}' is not live"),
6778 )?]);
6779 }
6780
6781 if !self
6782 .forwarding
6783 .has_live_module_connection(&target_module_id)
6784 .map_err(RouterError::Forwarding)?
6785 {
6786 return Ok(vec![self.route_open_refusal_frame(
6787 ctx,
6788 &caller,
6789 &frame,
6790 &target_module_id,
6791 "no_forwarding_connection",
6792 "target_unavailable",
6793 format!("module_id '{target_module_id}' has no live forwarding connection"),
6794 )?]);
6795 }
6796
6797 if let Some(error) =
6798 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
6799 {
6800 self.observe_route_open_refusal(
6801 ctx,
6802 &caller,
6803 &target_module_id,
6804 "op_not_allowed",
6805 "op_not_allowed",
6806 );
6807 return Ok(vec![error]);
6808 }
6809
6810 let principal = match (&attested, consumer_identity.is_some()) {
6811 (None, false) => {
6814 self.observe_route_open_refusal(
6815 ctx,
6816 &caller,
6817 &target_module_id,
6818 "registration_channel_closed",
6819 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6820 );
6821 return Ok(vec![control_error_frame(
6822 &frame,
6823 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6824 "the claim binding's launch has ended",
6825 )?]);
6826 }
6827 _ => match self.route_open_principal_attested(
6828 &frame,
6829 consumer_identity,
6830 attested.as_ref().map(|attested| attested.principal.clone()),
6831 )? {
6832 Ok(principal) => principal,
6833 Err(error) => {
6834 self.observe_route_open_refusal(
6835 ctx,
6836 &caller,
6837 &target_module_id,
6838 "bad_consumer_identity",
6839 "bad_consumer_identity",
6840 );
6841 return Ok(vec![error]);
6842 }
6843 },
6844 };
6845 let attested = attested.expect("an attested principal was resolved above");
6846 timings.principal = principal_label(&principal);
6847
6848 let Ok(mut caller_capture) = attested.capture() else {
6851 self.observe_route_open_refusal(
6852 ctx,
6853 &caller,
6854 &target_module_id,
6855 "registration_channel_closed",
6856 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6857 );
6858 return Ok(vec![control_error_frame(
6859 &frame,
6860 crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6861 "the opener's launch has ended",
6862 )?]);
6863 };
6864 let caller_module_id = match &attested.spawn {
6868 Some((record, _)) => Some(record.module_id().to_owned()),
6869 None => match &principal {
6870 Principal::Reserved { module_id } => Some(module_id.clone()),
6871 _ => None,
6872 },
6873 };
6874
6875 let build_class = identity.build_class.take().filter(|build_class| {
6878 if caller_module_id.as_deref() == Some("callosum") {
6879 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
6880 } else {
6881 if self
6882 .build_class_warned
6883 .lock()
6884 .unwrap_or_else(|p| p.into_inner())
6885 .insert(ctx.connection_id)
6886 {
6887 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
6888 "ignoring build_class from a non-Callosum opener");
6889 }
6890 false
6891 }
6892 });
6893
6894 if let Some(opening_registration) = self.caller_registration(&attested, &admission_registry)
6898 {
6899 let opening_module_id = opening_registration.manifest.module_id.as_str();
6900 if let Some(capability) =
6901 denied_capability(&opening_registration.manifest, ®istration.manifest)
6902 {
6903 warn!(
6904 opening_module_id,
6905 target_module_id,
6906 capability,
6907 "refusing route.open because an attested capability deny edge matches"
6908 );
6909 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6910 &frame,
6911 &target_module_id,
6912 "capability_deny_edge",
6913 "capability_forbidden",
6914 format!(
6915 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
6916 ),
6917 )?]);
6918 }
6919 }
6920
6921 if admission_facts.is_some() {
6922 let carrier_matches = caller_module_id.is_some()
6923 && self.admission_facts_carrier_module_id.as_deref() == caller_module_id.as_deref();
6924 if !carrier_matches {
6925 return Ok(vec![self.route_open_refusal_frame(
6926 ctx,
6927 &caller,
6928 &frame,
6929 &target_module_id,
6930 "admission_facts_carrier_not_permitted",
6931 "admission_facts_not_permitted",
6932 "admission facts may only be carried by the configured reserved module",
6933 )?]);
6934 }
6935
6936 let target_allowed = self
6937 .admission_facts_targets
6938 .as_ref()
6939 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
6940 if !target_allowed {
6941 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6942 &frame,
6943 &target_module_id,
6944 "admission_facts_target_not_listed",
6945 "admission_facts_target_not_allowed",
6946 format!(
6947 "admission facts are not permitted for target module_id '{target_module_id}'"
6948 ),
6949 )?]);
6950 }
6951
6952 }
6956
6957 let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
6963 None => (None, None, false),
6964 Some(selector) => {
6965 let owner_configured = match &selector.owner {
6966 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
6972 _ => false,
6973 };
6974 let mut scopes = self
6975 .scopes
6976 .write()
6977 .unwrap_or_else(|poisoned| poisoned.into_inner());
6978 let admitted = crate::scopes::admission::admit_route(
6979 &scopes,
6980 &admission_registry,
6981 &principal,
6982 &target_module_id,
6983 Some(selector),
6984 owner_configured,
6985 crate::scopes::admission::AdmissionMode::Open,
6986 )
6987 .map(|route| {
6988 let admission = route.scope.expect("scoped admission");
6989 let bound = scopes.reserve_bind(
6990 &admission,
6991 &principal,
6992 ctx.connection_id,
6993 selector.hold_presence,
6994 );
6995 (admission, bound)
6996 });
6997 match admitted {
6998 Ok((admission, bound)) => {
6999 (Some(bound), Some(admission.stamp), admission.stamp_required)
7000 }
7001 Err(refusal) => {
7002 return Ok(vec![self.route_open_refusal_frame(
7003 ctx,
7004 &caller,
7005 &frame,
7006 &target_module_id,
7007 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7008 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7009 refusal.message,
7010 )?]);
7011 }
7012 }
7013 }
7014 };
7015 if let Some(stamp) = &scope_stamp {
7018 caller.scope = route_open_scope_label(stamp);
7019 }
7020 if scope_stamp.as_ref().is_some_and(scope_acts_for_another) {
7023 caller_capture = caller_capture.map(crate::spawn_authority::CallerCapture::relayed);
7024 }
7025
7026 timings.enter(RouteOpenPhase::ProjectRoot);
7034 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
7035 Ok(project) => project,
7036 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
7037 };
7038 timings.enter(RouteOpenPhase::Admission);
7039 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
7040 Ok(root) => root,
7041 Err(err) => {
7042 return Ok(vec![control_error_frame(
7043 &frame,
7044 "invalid_project_root",
7045 err.to_string(),
7046 )?])
7047 }
7048 };
7049 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
7055
7056 let _concurrency_guard = match self
7067 .route_bind_concurrency
7068 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
7069 {
7070 Ok(guard) => guard,
7071 Err(in_flight) => {
7072 return Ok(vec![self.route_open_target_capacity_refusal(
7073 ctx,
7074 &caller,
7075 &frame,
7076 &target_module_id,
7077 in_flight,
7078 )?]);
7079 }
7080 };
7081
7082 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
7085 RouteBindAdmission::Admitted { guard, probe } => {
7086 if probe {
7087 info!(
7088 module_id = %target_module_id,
7089 connection_id = ctx.connection_id.get(),
7090 "route.bind breaker half-open: admitting one probe"
7091 );
7092 }
7093 guard
7094 }
7095 RouteBindAdmission::Refused {
7096 consecutive_timeouts,
7097 retry_in,
7098 probe_in_flight,
7099 } => {
7100 return Ok(vec![self.route_open_breaker_refusal_frame(
7101 ctx,
7102 &caller,
7103 &frame,
7104 &target_module_id,
7105 consecutive_timeouts,
7106 retry_in,
7107 probe_in_flight,
7108 )?]);
7109 }
7110 };
7111
7112 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
7117 let relay_deadline = Instant::now() + route_bind_relay_timeout;
7118 let operator_app = self
7119 .operator_apps
7120 .lock()
7121 .unwrap_or_else(|p| p.into_inner())
7122 .get(&ctx.connection_id)
7123 .cloned();
7124 timings.enter(RouteOpenPhase::EgressReservation);
7125 let pending = match self
7126 .forwarding
7127 .begin_route_bind_relay_for(
7128 ctx.connection_id,
7129 ctx.egress.clone(),
7130 response_version(&frame),
7131 frame.header.corr,
7132 &target_module_id,
7133 principal.clone(),
7134 bound_scope,
7135 project_root,
7136 identity.harness.clone(),
7137 build_class,
7138 operator_app.clone(),
7139 crate::forwarding::RouteAuthority {
7140 incarnation: self.supervisor.daemon_incarnation(),
7141 caller: caller_capture
7142 .as_ref()
7143 .map(|caller| caller.authority.pair()),
7144 },
7145 relay_deadline,
7146 busy_refusals == Some(true),
7147 accepts_close_reasons,
7148 )
7149 .await
7150 {
7151 Ok(pending) => pending,
7152 Err(err) => {
7153 return Ok(vec![self.route_open_refusal_frame(
7154 ctx,
7155 &caller,
7156 &frame,
7157 &target_module_id,
7158 "relay_reservation_failed",
7159 forwarding_error_code(&err),
7160 err.to_string(),
7161 )?])
7162 }
7163 };
7164 timings.enter(RouteOpenPhase::Admission);
7165 let crate::forwarding::PendingRouteBindRelay {
7166 endpoint,
7167 module_sink,
7168 negotiated_ver,
7169 client_channel,
7170 client_epoch,
7171 module_channel,
7172 module_epoch,
7173 corr: relay_corr,
7174 receiver,
7175 } = pending;
7176 let mut reservation =
7177 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
7178
7179 let relay_registration = self
7184 .registry
7185 .get_module_by_connection(endpoint.connection_id)
7186 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7187 let relay_capabilities = relay_registration
7188 .as_ref()
7189 .and_then(|registration| registration.manifest.capabilities.as_ref());
7190 if let Some(selector) = &scope {
7191 if let Err(refusal) = self
7194 .scopes
7195 .read()
7196 .unwrap_or_else(|poisoned| poisoned.into_inner())
7197 .check_bind_proof(&principal, selector)
7198 {
7199 reservation.release_and_disarm();
7200 return Ok(vec![self.route_open_refusal_frame(
7201 ctx,
7202 &caller,
7203 &frame,
7204 &target_module_id,
7205 refusal.code,
7206 refusal.code,
7207 refusal.message,
7208 )?]);
7209 }
7210 }
7211 if let Some(stamp) = scope_stamp.as_ref() {
7212 if let Err(refusal) = crate::scopes::check_target_required_stamp(
7213 stamp_required,
7214 &target_module_id,
7215 relay_capabilities,
7216 )
7217 .and_then(|()| {
7218 crate::scopes::check_target_flow_support(
7219 stamp,
7220 &target_module_id,
7221 relay_capabilities,
7222 )
7223 })
7224 .and_then(|()| {
7225 crate::scopes::check_target_agent_run_support(
7226 stamp,
7227 &target_module_id,
7228 relay_capabilities,
7229 )
7230 }) {
7231 reservation.release_and_disarm();
7232 return Ok(vec![self.route_open_refusal_frame(
7233 ctx,
7234 &caller,
7235 &frame,
7236 &target_module_id,
7237 refusal.code,
7238 refusal.code,
7239 refusal.message,
7240 )?]);
7241 }
7242 }
7243
7244 if scope_stamp
7247 .as_ref()
7248 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
7249 && !crate::scopes::target_supports_federated(relay_capabilities)
7250 {
7251 scope_stamp = None;
7253 }
7254 if !relay_capabilities.is_some_and(|caps| {
7255 caps.provides
7256 .iter()
7257 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
7258 }) {
7259 if let Some(stamp) = scope_stamp.as_mut() {
7260 stamp.workdir = None;
7261 stamp.project_id = None;
7262 }
7263 }
7264 let worker_run = scope_stamp
7269 .as_ref()
7270 .is_some_and(crate::scopes::is_worker_run)
7271 && relay_capabilities.is_some_and(|caps| {
7272 caps.provides
7273 .iter()
7274 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
7275 });
7276 if !worker_run
7277 && !relay_capabilities.is_some_and(|caps| {
7278 caps.provides
7279 .iter()
7280 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
7281 })
7282 {
7283 if let Some(stamp) = scope_stamp.as_mut() {
7284 stamp.attributes.on_behalf_of_agent = None;
7285 }
7286 }
7287 if !relay_capabilities.is_some_and(|caps| {
7288 caps.provides
7289 .iter()
7290 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
7291 }) {
7292 if let Some(stamp) = scope_stamp.as_mut() {
7293 stamp.attributes.flow_version = None;
7294 stamp.attributes.flow_code_hash = None;
7295 stamp.attributes.flow_install_id = None;
7296 }
7297 }
7298 let call_mode = if worker_run
7299 || relay_capabilities
7300 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
7301 {
7302 Some(derive_call_mode(scope_stamp.as_ref()))
7303 } else {
7304 if let Some(stamp) = scope_stamp.as_mut() {
7305 stamp.expires_at_ms = None;
7306 }
7307 None
7308 };
7309
7310 debug!(
7311 connection_id = ctx.connection_id.get(),
7312 client_channel,
7313 client_epoch,
7314 module_channel,
7315 module_epoch,
7316 "reserved route handle pair"
7317 );
7318 let route_principal_label = principal_label(&principal);
7321 let relay = ModuleControlRequest::RouteBind {
7322 route_channel: module_channel,
7323 epoch: module_epoch,
7324 target,
7325 identity,
7326 principal: Some(principal),
7327 consumer_capabilities,
7328 role_versions,
7329 admission_facts,
7330 scope: scope_stamp,
7331 call_mode,
7332 };
7333 let peer = RouteBindPeer::new()
7334 .with_observed_peer(
7335 relay_capabilities
7336 .is_some_and(|caps| {
7337 caps.provides
7338 .iter()
7339 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
7340 })
7341 .then(|| {
7342 self.observed_peers
7343 .lock()
7344 .unwrap_or_else(|p| p.into_inner())
7345 .get(&ctx.connection_id)
7346 .cloned()
7347 .unwrap_or_else(ObservedPeer::unobserved)
7348 }),
7349 )
7350 .with_operator_app(
7351 self.connection_operator_app(operator_app.as_ref())
7352 .filter(|_| {
7353 relay_capabilities.is_some_and(|caps| {
7354 caps.provides
7355 .iter()
7356 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
7357 })
7358 }),
7359 )
7360 .with_caller(caller_for_target(
7361 caller_capture.as_ref(),
7362 relay_capabilities,
7363 &target_module_id,
7364 ));
7365 let relay_body = crate::bind_compat::encode_bind(
7366 &relay,
7367 &peer,
7368 &legacy_bind_project,
7369 relay_capabilities,
7370 )
7371 .map_err(|err| {
7372 RouterError::backend(
7373 0,
7374 frame.header.corr,
7375 format!("failed to encode route.bind request: {err}"),
7376 )
7377 })?;
7378 let relay_frame = Frame::build_with_version(
7379 negotiated_ver,
7380 FrameType::Request,
7381 control_flags(),
7382 0,
7383 0,
7384 relay_corr,
7385 relay_body,
7386 )
7387 .map_err(RouterError::FrameBuild)?;
7388
7389 timings.enter(RouteOpenPhase::BindSend);
7390 let sent = module_sink.send(relay_frame).await;
7391 timings.enter(RouteOpenPhase::Admission);
7392 if let Err(err) = sent {
7393 reservation.release_and_disarm();
7394 return Ok(vec![self.route_open_refusal_frame(
7395 ctx,
7396 &caller,
7397 &frame,
7398 &target_module_id,
7399 "relay_send_failed",
7400 "target_unavailable",
7401 err.to_string(),
7402 )?]);
7403 }
7404
7405 #[cfg(test)]
7406 if let Some(hold) = &self.route_bind_relay_sent_hold {
7407 hold.wait().await;
7411 hold.wait().await;
7412 }
7413
7414 if self
7419 .forwarding
7420 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
7421 .map_err(RouterError::Forwarding)?
7422 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
7423 {
7424 self.send_abandoned_route_bind_goodbye(
7425 &module_sink,
7426 negotiated_ver,
7427 module_channel,
7428 module_epoch,
7429 );
7430 }
7431
7432 timings.enter(RouteOpenPhase::BindAckWait);
7433 let outcome = timeout_at(relay_deadline, receiver).await;
7434 timings.enter(RouteOpenPhase::Admission);
7435 match outcome {
7436 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
7437 reservation.disarm();
7438 if breaker.record_accepted() {
7439 info!(
7440 module_id = %target_module_id,
7441 "route.bind breaker closed: the probe was accepted"
7442 );
7443 }
7444 self.observe_route_open_accept(
7445 ctx,
7446 &target_module_id,
7447 &route_principal_label,
7448 &caller.scope,
7449 );
7450 Ok(Vec::new())
7451 }
7452 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
7453 reservation.release_and_disarm();
7454 breaker.record_inconclusive();
7458 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
7459 }
7460 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
7461 reservation.release_and_disarm();
7462 breaker.record_inconclusive();
7463 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
7464 }
7465 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
7466 reservation.release_and_disarm();
7467 breaker.record_inconclusive();
7468 tracing::warn!(
7475 module_id = %target_module_id,
7476 "route.bind relay abandoned: {message}"
7477 );
7478 Ok(vec![self.route_open_refusal_frame(
7479 ctx,
7480 &caller,
7481 &frame,
7482 &target_module_id,
7483 "relay_abandoned",
7484 "target_unavailable",
7485 message,
7486 )?])
7487 }
7488 Ok(Err(_)) => {
7489 reservation.release_and_disarm();
7490 breaker.record_inconclusive();
7491 Ok(vec![self.route_open_refusal_frame(
7492 ctx,
7493 &caller,
7494 &frame,
7495 &target_module_id,
7496 "relay_waiter_canceled",
7497 "target_unavailable",
7498 "route.bind relay waiter was canceled before the module responded",
7499 )?])
7500 }
7501 Err(_) => {
7502 reservation.release_and_disarm();
7503 if let Some(opened) = breaker.record_timeout(
7507 self.route_bind_breaker_threshold,
7508 self.route_bind_breaker_cooldown,
7509 ) {
7510 warn!(
7511 module_id = %target_module_id,
7512 consecutive_timeouts = opened.consecutive_timeouts,
7513 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
7514 reopened_after_probe = opened.reopened_after_probe,
7515 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
7516 );
7517 }
7518 tracing::warn!(
7526 module_id = %target_module_id,
7527 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
7528 "route.bind relay timed out: module did not ack within budget"
7529 );
7530 Ok(vec![self.route_open_refusal_frame(
7531 ctx,
7532 &caller,
7533 &frame,
7534 &target_module_id,
7535 "relay_timed_out",
7536 "module_timeout",
7537 format!(
7538 "module_id '{target_module_id}' did not answer route.bind within {:?}",
7539 route_bind_relay_timeout
7540 ),
7541 )?])
7542 }
7543 }
7544 }
7545
7546 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7547 let response = ClientControlResponse::SupervisorSpawnSnapshot {
7548 snapshot: self.supervisor.spawn_snapshot(),
7549 };
7550 Ok(vec![control_response_body_frame(
7551 &frame,
7552 &response,
7553 "ClientControlResponse::SupervisorSpawnSnapshot",
7554 )?])
7555 }
7556
7557 fn handle_supervisor_spawn_subscribe(
7558 &self,
7559 ctx: &RouteCtx,
7560 frame: Frame,
7561 since: Option<SpawnCursor>,
7562 ) -> Result<Vec<Frame>, RouterError> {
7563 match self.supervisor.subscribe_spawns(
7564 ctx.connection_id,
7565 frame.header.corr,
7566 response_version(&frame),
7567 since,
7568 ctx.egress.clone(),
7569 ) {
7570 Ok(()) => Ok(Vec::new()),
7571 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
7572 Ok(vec![control_error_body_frame(
7573 &frame,
7574 ErrorBody {
7575 code: "spawn_cursor_incarnation_mismatch".to_string(),
7576 message: "spawn cursor belongs to a different daemon incarnation"
7577 .to_string(),
7578 detail: Some(serde_json::json!({
7579 "current_daemon_incarnation": current
7580 })),
7581 },
7582 )?])
7583 }
7584 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
7585 &frame,
7586 ErrorBody {
7587 code: "spawn_cursor_too_old".to_string(),
7588 message: "spawn cursor predates the retained event ring".to_string(),
7589 detail: Some(serde_json::json!({
7590 "oldest_retained_cursor": oldest
7591 })),
7592 },
7593 )?]),
7594 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
7595 0,
7596 frame.header.corr,
7597 format!("failed to open supervisor spawn subscription: {error}"),
7598 )),
7599 }
7600 }
7601
7602 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7603 let handler = self.clone();
7606 let runtime = tokio::runtime::Handle::current();
7607 let subscriber = tracing::dispatcher::get_default(Clone::clone);
7610 let corr = frame.header.corr;
7611 tokio::task::spawn_blocking(move || {
7612 runtime.block_on(
7613 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
7614 .with_subscriber(subscriber),
7615 )
7616 })
7617 .await
7618 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
7619 }
7620
7621 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7622 use crate::config_gate::runtime::ApprovedDaemonView;
7623 let publication = self
7624 .config_runtime
7625 .as_ref()
7626 .and_then(|owner| owner.polled_snapshot().ok());
7627 let approved = publication
7628 .as_ref()
7629 .map(|view| view.approved_daemon.clone())
7630 .unwrap_or_else(|| {
7631 if self.config_runtime.is_some() {
7632 ApprovedDaemonView::Unavailable
7633 } else {
7634 ApprovedDaemonView::NotStamped
7635 }
7636 });
7637 let generation = self
7638 .registry
7639 .generation()
7640 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7641 let supervised = self.supervisor.list();
7642 let mut statuses = BTreeMap::new();
7643 let mut not_running = BTreeSet::new();
7644 for module in &supervised {
7645 let status = module.status_for_control("list").map_err(|err| {
7646 RouterError::backend(
7647 0,
7648 frame.header.corr,
7649 format!("failed to read supervisor status: {err}"),
7650 )
7651 })?;
7652 if !status.enabled || !status.process_alive {
7653 not_running.insert(module.module_id().to_owned());
7654 }
7655 statuses.insert(module.module_id().to_owned(), status);
7656 }
7657 let mut process_resources = BTreeMap::new();
7660 for module in &supervised {
7661 process_resources.insert(
7662 module.module_id().to_owned(),
7663 crate::subtree_resources::read_process(module.clone()).await,
7664 );
7665 }
7666 let mut image_readings = BTreeMap::new();
7669 for module in &supervised {
7670 image_readings.insert(
7671 module.module_id().to_owned(),
7672 module.running_image_agreement().await,
7673 );
7674 }
7675 let readings = self.resource_tracker.list(supervised.clone());
7676 let mut modules = Vec::new();
7677 for module in supervised {
7678 let status = statuses
7679 .remove(module.module_id())
7680 .expect("each supervised module has a status snapshot");
7681 let (configured, _) = module.configuration().map_err(|err| {
7682 RouterError::backend(
7683 0,
7684 frame.header.corr,
7685 format!("failed to read module configuration: {err}"),
7686 )
7687 })?;
7688 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
7689 RouterError::backend(
7690 0,
7691 frame.header.corr,
7692 format!("failed to read retry time: {err}"),
7693 )
7694 })?;
7695 let image = image_readings
7696 .remove(module.module_id())
7697 .expect("each supervised module has an image snapshot");
7698 let resources = process_resources.remove(module.module_id());
7699 let mut verdict =
7700 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
7701 verdict.spec = Some(launch_spec_agreement(
7702 &configured,
7703 status.process_alive,
7704 status.launch_spec_digest.as_ref(),
7705 ));
7706 let pending_reload = Some(verdict);
7707 let pending_entry = match &approved {
7708 ApprovedDaemonView::Approved { config, .. } => config
7709 .modules
7710 .iter()
7711 .find(|m| m.module_id == module.module_id()),
7712 _ => None,
7713 };
7714 let pending = self
7715 .pending_module_fields(
7716 &module,
7717 pending_entry,
7718 false,
7719 Self::registration_channel_in_force(
7720 publication
7721 .as_ref()
7722 .and_then(|view| view.registration_declaration(module.module_id()))
7723 .as_deref(),
7724 pending_entry.map(|entry| entry.registration_channel),
7725 ),
7726 )
7727 .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
7728 let script = configured.script.as_ref().map(|entry| {
7729 let approved = self
7730 .config_runtime
7731 .as_ref()
7732 .and_then(|owner| owner.approved_script(&status.module_id, &entry.path));
7733 crate::supervise::script::status(entry, approved.as_ref())
7734 });
7735 let configured_enabled = match &approved {
7736 ApprovedDaemonView::Approved {
7737 config: approved, ..
7738 } => approved
7739 .modules
7740 .iter()
7741 .find(|module| module.module_id == status.module_id)
7742 .map(|module| module.enabled),
7743 ApprovedDaemonView::NotStamped => self
7744 .capability_evaluator
7745 .configured_enabled(&status.module_id),
7746 ApprovedDaemonView::Unavailable => None,
7747 };
7748 let admission_refused = status
7749 .admission_refused
7750 .clone()
7751 .filter(|_| status.enabled && !status.process_alive)
7752 .filter(|_| {
7753 publication
7754 .as_ref()
7755 .and_then(|view| view.registration_declaration(&status.module_id))
7756 .as_deref()
7757 == Some("undeclared")
7758 });
7759 let mut entry = SupervisorEntry::new(
7762 status.module_id,
7763 status.state.to_string(),
7764 status.enabled,
7765 status.live,
7766 status.health.status,
7767 )
7768 .with_configured_enabled(configured_enabled)
7769 .with_script(script)
7770 .with_admission_refused(admission_refused)
7771 .with_next_retry_at_ms(next_retry_at_ms)
7772 .with_launch_nonce_env(Some(!cfg!(unix)))
7773 .with_launch_nonce_source(status.launch_nonce_source)
7774 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
7775 .with_protocol(status.protocol)
7776 .with_pending_reload(pending_reload)
7777 .with_last_probe_ms(status.health.last_probe_ms)
7778 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
7779 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
7780 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
7781 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
7782 .with_restart_count(Some(status.restart_count))
7783 .with_max_restarts(Some(status.max_restarts))
7784 .with_lifetime_restarts(Some(status.lifetime_restarts))
7785 .with_spawn_generation(Some(status.spawn_generation))
7786 .with_restart_window_secs(Some(status.restart_window.as_secs()))
7787 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
7788 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
7789 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
7790 .with_resources(resources);
7791 entry.pending = pending;
7792 modules.push(entry);
7793 }
7794 for module in &mut modules {
7795 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
7796 module.tree = tree.clone();
7797 module.cpu_last_hour = hour.clone();
7798 }
7799 module.tree_unavailable = readings
7800 .reasons
7801 .get(&module.module_id)
7802 .map(|reason| (*reason).to_owned());
7803 if module.tree.is_none() {
7804 module.tree_unavailable = Some(
7805 if not_running.contains(&module.module_id) {
7806 "not_running"
7807 } else if let Some(reason) = readings.reasons.get(&module.module_id) {
7808 reason
7809 } else {
7810 "not_yet_measured"
7811 }
7812 .to_owned(),
7813 );
7814 }
7815 }
7816 let response = ClientControlResponse::SupervisorList {
7817 generation,
7818 modules,
7819 daemon: readings.daemon,
7820 agent_commands: readings.agent_commands,
7821 };
7822 Ok(vec![control_response_body_frame(
7823 &frame,
7824 &response,
7825 "ClientControlResponse::SupervisorList",
7826 )?])
7827 }
7828
7829 fn handle_supervisor_stderr_tail(
7830 &self,
7831 frame: Frame,
7832 module_id: String,
7833 max_lines: Option<u32>,
7834 max_bytes: Option<u32>,
7835 ) -> Result<Vec<Frame>, RouterError> {
7836 let Some(module) = self.supervisor.get(&module_id) else {
7837 return Ok(vec![control_error_frame(
7838 &frame,
7839 "unknown_module",
7840 format!("module_id '{module_id}' is not supervised"),
7841 )?]);
7842 };
7843
7844 let snapshot = module.stderr_tail(
7845 max_lines.map(|value| value as usize),
7846 max_bytes.map(|value| value as usize),
7847 );
7848
7849 let response = ClientControlResponse::SupervisorStderrTail {
7850 module_id,
7851 tail: StderrTail {
7852 capture: match snapshot.capture {
7853 CaptureState::Captured => StderrCaptureState::Captured,
7854 CaptureState::Incomplete { reason } => {
7855 StderrCaptureState::Incomplete { reason }
7856 }
7857 CaptureState::NotCaptured { reason } => {
7858 StderrCaptureState::NotCaptured { reason }
7859 }
7860 },
7861 entries: snapshot
7862 .entries
7863 .into_iter()
7864 .map(|entry| match entry {
7865 TailEntry::Line {
7866 text,
7867 truncated,
7868 at_ms,
7869 } => StderrTailEntry::Line {
7870 text,
7871 truncated,
7872 at_ms,
7873 },
7874 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
7875 })
7876 .collect(),
7877 dropped_lines: snapshot.dropped_lines,
7878 },
7879 };
7880 Ok(vec![control_response_body_frame(
7881 &frame,
7882 &response,
7883 "ClientControlResponse::SupervisorStderrTail",
7884 )?])
7885 }
7886
7887 async fn observe_control_wait<F: std::future::Future>(
7888 &self,
7889 ctx: &RouteCtx,
7890 target: Option<&str>,
7891 kind: &'static str,
7892 future: F,
7893 ) -> F::Output {
7894 self.counters
7895 .observe_reader_wait(
7896 || {
7897 let module = self
7898 .forwarding
7899 .module_id_for_connection(ctx.connection_id)
7900 .ok()
7901 .flatten();
7902 let principal = if module.is_some() { "module" } else { "direct" };
7903 (
7904 self.resolve_reader_wait_target(kind, target)
7905 .or(module)
7906 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
7907 principal.into(),
7908 )
7909 },
7910 kind,
7911 future,
7912 )
7913 .await
7914 }
7915
7916 pub(crate) fn resolve_reader_wait_target(
7919 &self,
7920 op: &str,
7921 selector: Option<&str>,
7922 ) -> Option<String> {
7923 let selector = selector?;
7924 match op {
7925 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
7926 .supervisor
7927 .get(selector)
7928 .map(|module| module.module_id().to_owned()),
7929 ops::SUPERVISOR_HEALTH_PROBE => self
7930 .registry
7931 .get_module(selector)
7932 .ok()
7933 .flatten()
7934 .map(|registration| registration.manifest.module_id),
7935 _ => None,
7936 }
7937 }
7938
7939 async fn handle_supervisor_terminals(
7940 &self,
7941 ctx: &RouteCtx,
7942 frame: Frame,
7943 module_id: String,
7944 ) -> Result<Vec<Frame>, RouterError> {
7945 let Some(module) = self.supervisor.get(&module_id) else {
7946 return Ok(vec![control_error_frame(
7947 &frame,
7948 "unknown_module",
7949 format!("module_id '{module_id}' is not supervised"),
7950 )?]);
7951 };
7952
7953 let terminals = self
7956 .observe_control_wait(
7957 ctx,
7958 Some(module.module_id()),
7959 ops::SUPERVISOR_TERMINALS,
7960 module.read_durable_terminal_history(),
7961 )
7962 .await
7963 .map_err(|error| {
7964 RouterError::backend(
7965 0,
7966 frame.header.corr,
7967 format!("failed to read terminal history: {error}"),
7968 )
7969 })?;
7970 let response = ClientControlResponse::SupervisorTerminals {
7971 module_id,
7972 terminals,
7973 };
7974 Ok(vec![control_response_body_frame(
7975 &frame,
7976 &response,
7977 "ClientControlResponse::SupervisorTerminals",
7978 )?])
7979 }
7980
7981 fn handle_supervisor_routes(
7982 &self,
7983 frame: Frame,
7984 module_id: Option<String>,
7985 ) -> Result<Vec<Frame>, RouterError> {
7986 let modules = self
7987 .forwarding
7988 .route_census(module_id.as_deref())
7989 .map_err(RouterError::Forwarding)?
7990 .into_iter()
7991 .map(|(module_id, routes)| SupervisorRouteModule {
7992 module_id,
7993 routes: routes
7994 .into_iter()
7995 .map(|route| SupervisorRoute {
7996 consumer: match route.principal {
7997 Principal::Reserved { module_id } => {
7998 SupervisorRouteConsumer::Reserved { module_id }
7999 }
8000 Principal::Direct | Principal::Unverified => {
8001 SupervisorRouteConsumer::Direct {
8002 connection_id: route.goodbye_target.connection_id.get(),
8003 }
8004 }
8005 },
8006 age_ms: Instant::now()
8007 .saturating_duration_since(route.bound_at)
8008 .as_millis()
8009 .try_into()
8010 .unwrap_or(u64::MAX),
8011 draining: route.draining,
8012 drain_reason: route.drain_reason,
8013 })
8014 .collect(),
8015 })
8016 .collect();
8017 let response = ClientControlResponse::SupervisorRoutes { modules };
8018 Ok(vec![control_response_body_frame(
8019 &frame,
8020 &response,
8021 "ClientControlResponse::SupervisorRoutes",
8022 )?])
8023 }
8024
8025 fn running_registration_method(
8026 &self,
8027 module: &str,
8028 ) -> Option<subc_control::RegistrationMethod> {
8029 let registration = self.registry.get_module(module).ok()??;
8030 let connection = registration.connection_id;
8031 let channel = self.supervisor.registration_spawn(connection).is_some();
8032 let nonce = if channel {
8033 None
8034 } else {
8035 self.hello_launch_nonces
8036 .lock()
8037 .ok()?
8038 .nonce(connection)
8039 .map(str::to_owned)
8040 };
8041 if !channel && nonce.is_none() {
8042 return None;
8043 }
8044 self.supervisor
8045 .registration_snapshot_is_current_live(
8046 &self.registry,
8047 module,
8048 connection,
8049 nonce.as_deref(),
8050 )
8051 .then_some(if channel {
8052 subc_control::RegistrationMethod::Channel
8053 } else {
8054 subc_control::RegistrationMethod::Nonce
8055 })
8056 }
8057
8058 async fn handle_supervisor_provenance(
8059 &self,
8060 ctx: &RouteCtx,
8061 frame: Frame,
8062 module_id: Option<String>,
8063 ) -> Result<Vec<Frame>, RouterError> {
8064 let targeted = module_id.is_some();
8065 let selected = if let Some(module_id) = module_id {
8066 let Some(module) = self.supervisor.get(&module_id) else {
8067 return Ok(vec![control_error_frame(
8068 &frame,
8069 "unknown_module",
8070 format!("module_id '{module_id}' is not supervised"),
8071 )?]);
8072 };
8073 vec![module]
8074 } else {
8075 self.supervisor.list()
8076 };
8077
8078 let target = targeted.then(|| selected[0].module_id());
8079 let declarations = if let Some(owner) = &self.config_runtime {
8080 owner
8081 .on_blocking(|owner| owner.registration_declarations())
8082 .await
8083 .ok()
8084 } else {
8085 None
8086 };
8087 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
8088 let mut modules = Vec::with_capacity(selected.len());
8089 let mut extensions = Vec::with_capacity(selected.len());
8090 for module in &selected {
8091 let (status, observed_image) = module
8092 .status_and_running_image_agreement()
8093 .await
8094 .map_err(|err| {
8095 RouterError::backend(
8096 0,
8097 frame.header.corr,
8098 format!("failed to read supervisor status: {err}"),
8099 )
8100 })?;
8101 let module_declared = self
8102 .registry
8103 .get_module(&status.module_id)
8104 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
8105 .and_then(|registration| registration.manifest.provenance)
8106 .map(|build| ModuleDeclaredProvenance::Reported { build })
8107 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
8108 #[cfg(test)]
8109 let running_image = match &self.provenance_probe_override {
8110 Some(result) => result.clone(),
8111 None => observed_image,
8112 };
8113 #[cfg(not(test))]
8114 let running_image = observed_image;
8115 let registration = self.running_registration_method(&status.module_id);
8116 let mut extension = subc_control::SupervisorModuleProvenanceExt::new()
8117 .with_registration(registration);
8118 extension.registration_declaration = declarations
8119 .as_ref()
8120 .and_then(|entries| entries.get(&status.module_id))
8121 .cloned();
8122 if status.process_alive && status.registration_declaration.is_some() {
8123 extension.registration_declaration = status.registration_declaration.clone();
8124 }
8125 extensions.push(extension);
8126 modules.push(SupervisorModuleProvenance {
8127 module_id: status.module_id,
8128 module_declared,
8129 daemon_observed: SupervisorObservedProcess {
8130 launch_nonce_source: status.launch_nonce_source,
8131 pid: status.pid,
8132 spawned_at_ms: status.spawned_at_ms,
8133 spawned_from: status.spawned_from,
8134 running_image,
8135 },
8136 });
8137 }
8138 let daemon = SupervisorDaemonProvenance {
8139 daemon_build: self.daemon_provenance.build.clone(),
8140 daemon_observed: DaemonObservedProcess {
8141 pid: self.daemon_provenance.pid,
8142 started_at_ms: self
8143 .daemon_provenance
8144 .start_clock
8145 .map(|clock| clock.started_at_ms())
8146 .or(self.daemon_provenance.started_at_ms),
8147 running_image: self
8148 .daemon_provenance
8149 .probe
8150 .observe(
8151 self.daemon_provenance.pid,
8152 self.daemon_provenance.executable_path.as_deref(),
8153 self.daemon_provenance.executable_identity.clone(),
8154 self.daemon_provenance.process_start_time,
8155 )
8156 .await,
8157 },
8158 };
8159 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
8160 let mut response = serde_json::to_value(response)
8161 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
8162 for (entry, extension) in response["modules"]
8163 .as_array_mut()
8164 .expect("provenance modules are an array")
8165 .iter_mut()
8166 .zip(extensions)
8167 {
8168 extension.write_to(entry).map_err(|error| {
8169 RouterError::backend(0, frame.header.corr, error.to_string())
8170 })?;
8171 }
8172 Ok(vec![control_response_body_frame(
8173 &frame,
8174 &response,
8175 "ClientControlResponse::SupervisorProvenance",
8176 )?])
8177 })
8178 .await
8179 }
8180
8181 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
8182 self.refresh_capability_requirements();
8183 let publication = self
8184 .config_runtime
8185 .as_ref()
8186 .and_then(|owner| owner.polled_snapshot().ok());
8187 let generation = self
8188 .registry
8189 .generation()
8190 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
8191 let modules = self
8192 .supervisor
8193 .list()
8194 .into_iter()
8195 .map(|module| {
8196 let status = module.status_for_control("health").map_err(|err| {
8197 RouterError::backend(
8198 0,
8199 frame.header.corr,
8200 format!("failed to read supervisor health: {err}"),
8201 )
8202 })?;
8203 let module_id = status.module_id;
8204 let capability_detail = self
8205 .capability_evaluator
8206 .required_problem_detail(&module_id);
8207 let declaration_warning = self.registry.get_module(&module_id).ok().flatten()
8208 .filter(|registration| registration.manifest.registration_channel == Some(false))
8209 .filter(|_| publication.as_ref().and_then(|view| view.registration_declaration(&module_id))
8210 .as_deref().map(|next| if status.process_alive { status.registration_declaration.as_deref().unwrap_or(next) } else { next }) == Some("declared"))
8211 .map(|_| format!("{module_id}: HELLO registration_channel is false despite the approved channel declaration"));
8212 let capability_detail = append_capability_problem_detail(capability_detail, declaration_warning);
8213 Ok(SupervisorHealthEntry {
8214 module_id,
8215 status: status.health.status,
8216 detail: append_capability_problem_detail(
8217 status.health.detail,
8218 capability_detail,
8219 ),
8220 metrics: status.health.metrics,
8221 consecutive_failures: status.health.consecutive_failures,
8222 late_answer_count: status.health.late_answer_count,
8223 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
8224 last_action: status.health.last_action,
8225 last_action_ms: status.health.last_action_ms,
8226 last_probe_ms: status.health.last_probe_ms,
8227 })
8228 })
8229 .collect::<Result<Vec<_>, RouterError>>()?;
8230 let response = ClientControlResponse::SupervisorHealth {
8231 generation,
8232 modules,
8233 };
8234 Ok(vec![control_response_body_frame(
8235 &frame,
8236 &response,
8237 "ClientControlResponse::SupervisorHealth",
8238 )?])
8239 }
8240
8241 async fn handle_supervisor_restart(
8242 &self,
8243 frame: Frame,
8244 module_id: String,
8245 drain_timeout_ms: Option<u64>,
8246 requester: Principal,
8247 ) -> Result<Vec<Frame>, RouterError> {
8248 let operation_lock = self.supervisor.operation_lock();
8249 let _operation_guard = operation_lock.lock().await;
8250 let Some(module) = self.supervisor.get(&module_id) else {
8251 return Ok(vec![control_error_frame(
8252 &frame,
8253 "unknown_module",
8254 format!("module_id '{module_id}' is not supervised"),
8255 )?]);
8256 };
8257
8258 self.route_outages.mark_operator_action(&module_id);
8259 if let Err(err) = module
8260 .restart_requested(drain_timeout_ms, Some(requester))
8261 .await
8262 {
8263 self.route_outages
8264 .operator_action_ended_unrefused(&module_id);
8265 let (code, message) = match err {
8266 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8267 crate::supervise::SuperviseError::Disabled { .. } => {
8268 ("module_disabled", err.to_string())
8269 }
8270 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8271 ("swap_in_progress", err.to_string())
8272 }
8273 _ => (
8274 "target_unavailable",
8275 format!("failed to restart module_id '{module_id}': {err}"),
8276 ),
8277 };
8278 return Ok(vec![control_error_frame(&frame, code, message)?]);
8279 }
8280
8281 let response = ClientControlResponse::SupervisorAck {
8282 module_id,
8283 applied: true,
8284 };
8285 Ok(vec![control_response_body_frame(
8286 &frame,
8287 &response,
8288 "ClientControlResponse::SupervisorAck",
8289 )?])
8290 }
8291
8292 async fn handle_supervisor_swap(
8296 &self,
8297 frame: Frame,
8298 module_id: String,
8299 ready_timeout_ms: Option<u64>,
8300 requester: Principal,
8301 ) -> Result<Vec<Frame>, RouterError> {
8302 let module = {
8309 let operation_lock = self.supervisor.operation_lock();
8310 let _operation_guard = operation_lock.lock().await;
8311 self.supervisor.get(&module_id)
8312 };
8313 let Some(module) = module else {
8314 return Ok(vec![control_error_frame(
8315 &frame,
8316 "unknown_module",
8317 format!("module_id '{module_id}' is not supervised"),
8318 )?]);
8319 };
8320
8321 self.route_outages.mark_operator_action(&module_id);
8322 if let Err(err) = module
8323 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
8324 .await
8325 {
8326 self.route_outages
8327 .operator_action_ended_unrefused(&module_id);
8328 use crate::supervise::SuperviseError;
8329 let message = err.to_string();
8330 let error = match err {
8331 SuperviseError::ConfigHeld { .. } => ErrorBody::new("config_gated", message),
8332 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
8333 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
8334 code: "swap_refused".to_string(),
8335 message,
8336 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
8337 },
8338 SuperviseError::SwapFailed {
8339 arm,
8340 candidate_exit,
8341 ..
8342 } => ErrorBody {
8343 code: "swap_failed".to_string(),
8344 message,
8345 detail: Some(serde_json::json!({
8346 "arm": arm.as_str(),
8347 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
8348 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
8349 })),
8350 },
8351 _ => ErrorBody::new(
8352 "target_unavailable",
8353 format!("failed to swap module_id '{module_id}': {message}"),
8354 ),
8355 };
8356 return Ok(vec![control_error_body_frame(&frame, error)?]);
8357 }
8358 self.route_outages
8362 .operator_action_ended_unrefused(&module_id);
8363
8364 let response = ClientControlResponse::SupervisorAck {
8365 module_id,
8366 applied: true,
8367 };
8368 Ok(vec![control_response_body_frame(
8369 &frame,
8370 &response,
8371 "ClientControlResponse::SupervisorAck",
8372 )?])
8373 }
8374
8375 async fn handle_supervisor_reload(
8376 &self,
8377 frame: Frame,
8378 module_id: String,
8379 requester: Principal,
8380 ) -> Result<Vec<Frame>, RouterError> {
8381 let operation_lock = self.supervisor.operation_lock();
8382 let _operation_guard = operation_lock.lock().await;
8383 let Some(module) = self.supervisor.get(&module_id) else {
8384 return Ok(vec![control_error_frame(
8385 &frame,
8386 "unknown_module",
8387 format!("module_id '{module_id}' is not supervised"),
8388 )?]);
8389 };
8390
8391 self.route_outages.mark_operator_action(&module_id);
8392 if let Err(err) = module.reload_requested(Some(requester)).await {
8393 self.route_outages
8394 .operator_action_ended_unrefused(&module_id);
8395 let (code, message) = match err {
8396 _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8397 crate::supervise::SuperviseError::Disabled { .. } => {
8398 ("module_disabled", err.to_string())
8399 }
8400 crate::supervise::SuperviseError::SwapInProgress { .. } => {
8401 ("swap_in_progress", err.to_string())
8402 }
8403 _ => (
8404 "reload_failed",
8405 format!("failed to reload module_id '{module_id}': {err}"),
8406 ),
8407 };
8408 return Ok(vec![control_error_frame(&frame, code, message)?]);
8409 }
8410
8411 let response = ClientControlResponse::SupervisorAck {
8412 module_id,
8413 applied: true,
8414 };
8415 Ok(vec![control_response_body_frame(
8416 &frame,
8417 &response,
8418 "ClientControlResponse::SupervisorAck",
8419 )?])
8420 }
8421
8422 async fn handle_supervisor_rescan(
8426 &self,
8427 frame: Frame,
8428 mode: RescanMode,
8429 requester: Principal,
8430 ) -> Result<Vec<Frame>, RouterError> {
8431 let preview = mode.previews();
8432 let mut approved_preview = None;
8433 if matches!(mode, RescanMode::Preview) {
8434 if let Some(owner) = &self.config_runtime {
8435 match owner
8436 .on_blocking(|owner| {
8437 owner.observe_file_proposal()?;
8438 owner.approved_daemon_configuration()
8439 })
8440 .await
8441 {
8442 Ok(config) => approved_preview = config,
8443 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8444 }
8445 }
8446 }
8447 let Some(context) = self.rescan.clone() else {
8448 return Ok(vec![control_error_frame(
8449 &frame,
8450 "rescan_unavailable",
8451 "the daemon was not started with a reloadable config path".to_string(),
8452 )?]);
8453 };
8454
8455 let operation_lock = self.supervisor.operation_lock();
8456 let _operation_guard = operation_lock.lock().await;
8457 let loaded = match self.load_rescan_config(&mode, &context) {
8461 Ok(config) => config,
8462 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
8463 };
8464 let Some((config, activity_config)) = loaded else {
8472 return Ok(vec![control_error_frame(
8473 &frame,
8474 "invalid_daemon_config",
8475 format!(
8476 "daemon config not found at {}; refusing to rescan (an absent config would \
8477 retire every supervised module)",
8478 context.config_path.display()
8479 ),
8480 )?]);
8481 };
8482 let (
8483 configured_port,
8484 storage_config,
8485 admission_facts_carrier_module_id,
8486 admission_facts_targets,
8487 scope_authority_owners,
8488 federated_scope_owner,
8489 privileged_operator_prompt,
8490 modules,
8491 reserved_capabilities,
8492 ) = (
8493 config.port,
8494 config.storage,
8495 config.admission_facts_carrier_module_id,
8496 config.admission_facts_targets,
8497 config.scope_authority_owners,
8498 config.federated_scope_owner,
8499 config.privileged_operator_prompt,
8500 config.modules,
8501 config.reserved_capabilities,
8502 );
8503
8504 let mut restart_required = Vec::new();
8513 for section in RestartRequiredSection::ALL {
8514 let changed = match section {
8515 RestartRequiredSection::Port => configured_port != context.configured_port,
8516 RestartRequiredSection::Storage => storage_config != context.storage_config,
8517 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
8518 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
8519 }
8520 RestartRequiredSection::AdmissionFactsTargets => {
8521 admission_facts_targets != context.admission_facts_targets
8522 }
8523 RestartRequiredSection::ScopeAuthorityOwners => {
8524 scope_authority_owners != context.scope_authority_owners
8525 }
8526 RestartRequiredSection::FederatedScopeOwner => {
8527 federated_scope_owner != context.federated_scope_owner
8528 }
8529 RestartRequiredSection::PrivilegedOperatorPrompt => {
8530 privileged_operator_prompt != context.privileged_operator_prompt
8531 }
8532 };
8533 if changed {
8534 restart_required.push(section.label().to_string());
8535 }
8536 }
8537 if !restart_required.is_empty() {
8538 warn!(
8539 config_path = %context.config_path.display(),
8540 sections = %restart_required.join(", "),
8541 "daemon config changed outside the modules section; restart the daemon to apply those changes"
8542 );
8543 }
8544
8545 for configured in &modules {
8546 if let Err(err) = validate_spec(&configured.module_spec()) {
8547 return Ok(vec![control_error_frame(
8548 &frame,
8549 "invalid_daemon_config",
8550 format!("supervisor rescan rejected daemon config: {err}"),
8551 )?]);
8552 }
8553 }
8554
8555 let configured_capabilities = modules
8556 .iter()
8557 .map(|module| (module.module_id.clone(), module.enabled))
8558 .collect::<Vec<_>>();
8559 let preview_capability_warnings = if preview {
8560 let (_, registrations) = self.runtime_capability_snapshot()?;
8561 let current_modules = self
8562 .supervisor
8563 .list()
8564 .into_iter()
8565 .map(|module| module.module_id().to_string())
8566 .collect::<BTreeSet<_>>();
8567 let resulting_modules = configured_capabilities.clone();
8568 let removed = current_modules
8569 .into_iter()
8570 .filter(|module_id| {
8571 !resulting_modules
8572 .iter()
8573 .any(|(configured_id, _)| configured_id == module_id)
8574 })
8575 .collect::<Vec<_>>();
8576 let disabled = match &mode {
8579 RescanMode::Candidate(_) => resulting_modules
8580 .iter()
8581 .filter(|(_, enabled)| !enabled)
8582 .map(|(module_id, _)| module_id.clone())
8583 .collect::<Vec<_>>(),
8584 _ => Vec::new(),
8585 };
8586 self.capability_evaluator.preview_warnings(
8587 resulting_modules,
8588 &removed,
8589 &disabled,
8590 ®istrations,
8591 )
8592 } else {
8593 Vec::new()
8594 };
8595 let reconciled = match (&mode, approved_preview.as_ref()) {
8596 (RescanMode::Preview, Some(approved)) => Ok((
8597 config_gate::preview_approved_daemon(approved, &modules),
8598 Vec::new(),
8599 )),
8600 (RescanMode::Candidate(candidate), _) => self
8601 .preview_candidate(&context, modules, candidate)
8602 .map(|result| (result, Vec::new())),
8603 _ => {
8604 self.reconcile_supervised_modules(
8605 &context.supervisor,
8606 modules,
8607 preview,
8608 requester.clone(),
8609 )
8610 .await
8611 }
8612 };
8613 let (result, changed) = match reconciled {
8614 Ok(result) => result,
8615 Err(message) => {
8616 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
8617 }
8618 };
8619 if !preview {
8620 if let Some(ledger) = &self.activity {
8621 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
8622 ledger.config_changed(&result.added, &result.removed, &changed, requester);
8623 }
8624 if let Err(code) = self.configure_activity(activity_config).await {
8625 warn!(
8626 code,
8627 "activity retention could not be applied during rescan"
8628 );
8629 }
8630 self.capability_evaluator
8631 .configure(configured_capabilities, reserved_capabilities);
8632 self.capability_evaluator.wake_deadline_loop();
8633 self.refresh_capability_requirements();
8634 }
8635 let mut result = result;
8636 result.restart_required = restart_required;
8637 result.capability_warnings = preview_capability_warnings;
8638 let response = ClientControlResponse::SupervisorRescan { result };
8639 Ok(vec![control_response_body_frame(
8640 &frame,
8641 &response,
8642 "ClientControlResponse::SupervisorRescan",
8643 )?])
8644 }
8645
8646 async fn handle_supervisor_release_reserved(
8647 &self,
8648 frame: Frame,
8649 module_id: String,
8650 ) -> Result<Vec<Frame>, RouterError> {
8651 let Some(context) = self.rescan.clone() else {
8652 return Ok(vec![control_error_frame(
8653 &frame,
8654 "release_unavailable",
8655 "reserved-id release requires a daemon started with a reloadable config path",
8656 )?]);
8657 };
8658 let operation_lock = self.supervisor.operation_lock();
8659 let _operation_guard = operation_lock.lock().await;
8660 let approved = if let Some(owner) = &self.config_runtime {
8661 match owner
8664 .on_blocking(|owner| {
8665 owner.observe_file_proposal()?;
8666 owner.approved_daemon_configuration()
8667 })
8668 .await
8669 {
8670 Ok(config) => config,
8671 Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8672 }
8673 } else {
8674 None
8675 };
8676 let loaded = if let Some(approved) = approved {
8677 Ok(Some(approved))
8678 } else {
8679 crate::daemon_config::load(&context.config_path)
8680 };
8681 let loaded = match loaded {
8682 Ok(Some(config)) => config,
8683 Ok(None) => {
8684 return Ok(vec![control_error_frame(
8685 &frame,
8686 "invalid_daemon_config",
8687 format!(
8688 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
8689 context.config_path.display()
8690 ),
8691 )?])
8692 }
8693 Err(err) => {
8694 return Ok(vec![control_error_frame(
8695 &frame,
8696 "invalid_daemon_config",
8697 format!("unable to verify reserved-id release against daemon config: {err}"),
8698 )?])
8699 }
8700 };
8701 if loaded
8702 .modules
8703 .iter()
8704 .any(|configured| configured.module_id == module_id)
8705 {
8706 return Ok(vec![control_error_frame(
8707 &frame,
8708 "reserved_module_configured",
8709 format!(
8710 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
8711 ),
8712 )?]);
8713 }
8714 if !self.supervisor.release_retained_reserved_gate(&module_id) {
8715 return Ok(vec![control_error_frame(
8716 &frame,
8717 "reserved_gate_not_retained",
8718 format!(
8719 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
8720 ),
8721 )?]);
8722 }
8723
8724 let response = ClientControlResponse::SupervisorAck {
8725 module_id,
8726 applied: true,
8727 };
8728 Ok(vec![control_response_body_frame(
8729 &frame,
8730 &response,
8731 "ClientControlResponse::SupervisorAck",
8732 )?])
8733 }
8734
8735 fn plan_rescan(
8739 &self,
8740 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8741 ) -> Result<RescanPlan, String> {
8742 let mut current = BTreeMap::new();
8743 for module in self.supervisor.list() {
8744 let (spec, health) = module.configuration().map_err(|err| {
8745 format!(
8746 "failed to read configuration for module_id '{}': {err}",
8747 module.module_id()
8748 )
8749 })?;
8750 let enabled = module
8751 .status()
8752 .map_err(|err| {
8753 format!(
8754 "failed to read status for module_id '{}': {err}",
8755 module.module_id()
8756 )
8757 })?
8758 .enabled;
8759 current.insert(
8760 module.module_id().to_string(),
8761 (module, spec, health, enabled),
8762 );
8763 }
8764 let configured = configured_modules
8765 .into_iter()
8766 .map(|module| (module.module_id.clone(), module))
8767 .collect::<BTreeMap<_, _>>();
8768
8769 let added = configured
8770 .keys()
8771 .filter(|module_id| !current.contains_key(*module_id))
8772 .cloned()
8773 .collect::<Vec<_>>();
8774 let removed = current
8775 .keys()
8776 .filter(|module_id| !configured.contains_key(*module_id))
8777 .cloned()
8778 .collect::<Vec<_>>();
8779 let mut changed_pending_reload = Vec::new();
8780 let mut changed_pending = Vec::new();
8781 let mut configuration_changes = BTreeSet::new();
8782 let mut enabled_changes = BTreeSet::new();
8783 let mut unchanged = 0_u32;
8784 let publication = self
8785 .config_runtime
8786 .as_ref()
8787 .and_then(|owner| owner.polled_snapshot().ok());
8788
8789 for (module_id, configured_module) in &configured {
8790 let Some((module, current_spec, current_health, current_enabled)) =
8791 current.get(module_id)
8792 else {
8793 continue;
8794 };
8795 let launch_changed = *current_spec != configured_module.module_spec();
8799 let configuration_changed =
8800 launch_changed || *current_health != configured_module.health;
8801 let pending = self.pending_module_fields(
8802 module,
8803 Some(configured_module),
8804 configuration_changed,
8805 Self::registration_channel_in_force(
8806 publication
8807 .as_ref()
8808 .and_then(|view| view.registration_declaration(module_id))
8809 .as_deref(),
8810 publication
8811 .as_ref()
8812 .and_then(|view| match &view.approved_daemon {
8813 crate::config_gate::runtime::ApprovedDaemonView::Approved {
8814 config,
8815 ..
8816 } => config
8817 .modules
8818 .iter()
8819 .find(|entry| entry.module_id == *module_id)
8820 .map(|entry| entry.registration_channel),
8821 _ => None,
8822 }),
8823 ),
8824 )?;
8825 let has_pending = !pending.is_empty();
8826 if has_pending {
8827 changed_pending.push(subc_control::SupervisorPendingModule {
8828 module_id: module_id.clone(),
8829 fields: pending
8830 .into_iter()
8831 .map(|mut field| {
8832 field.value = None;
8833 field
8834 })
8835 .collect(),
8836 });
8837 }
8838 let enabled_changed = *current_enabled != configured_module.enabled
8841 || self
8842 .capability_evaluator
8843 .configured_enabled(module_id)
8844 .is_some_and(|enabled| enabled != configured_module.enabled);
8845 if configuration_changed {
8846 configuration_changes.insert(module_id.clone());
8847 }
8848 if launch_changed {
8849 changed_pending_reload.push(module_id.clone());
8850 }
8851 if enabled_changed {
8852 enabled_changes.insert(module_id.clone());
8853 }
8854 if !configuration_changed && !enabled_changed && !has_pending {
8855 unchanged = unchanged.saturating_add(1);
8856 }
8857 }
8858 Ok(RescanPlan {
8859 current,
8860 configured,
8861 added,
8862 removed,
8863 changed_pending_reload,
8864 changed_pending,
8865 configuration_changes,
8866 enabled_changes,
8867 unchanged,
8868 })
8869 }
8870
8871 async fn reconcile_supervised_modules(
8880 &self,
8881 supervisor: &Supervisor,
8882 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8883 preview: bool,
8884 requester: Principal,
8885 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
8886 let plan = self.plan_rescan(configured_modules)?;
8887
8888 if preview {
8894 return Ok((plan.preview_result(), Vec::new()));
8895 }
8896 let changed = plan
8897 .configuration_changes
8898 .union(&plan.enabled_changes)
8899 .cloned()
8900 .collect::<Vec<_>>();
8901 let RescanPlan {
8902 current,
8903 configured,
8904 added,
8905 removed,
8906 changed_pending_reload,
8907 changed_pending,
8908 configuration_changes,
8909 enabled_changes,
8910 unchanged,
8911 } = plan;
8912 if !(added.is_empty()
8917 && removed.is_empty()
8918 && configuration_changes.is_empty()
8919 && changed_pending.is_empty()
8920 && enabled_changes.is_empty())
8921 {
8922 if let Some(context) = &self.rescan {
8923 context
8924 .generation
8925 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8926 }
8927 }
8928
8929 for module_id in &removed {
8930 let module = ¤t
8931 .get(module_id)
8932 .expect("removed module came from current supervisor state")
8933 .0;
8934 module
8935 .retire_requested(Some(requester.clone()))
8936 .await
8937 .map_err(|err| {
8938 format!("failed to retire module_id '{module_id}' during rescan: {err}")
8939 })?;
8940 self.supervisor.record_rescan_removal(module_id);
8974 self.supervisor.retire(module_id);
8975 self.route_outages.forget(module_id);
8976 }
8977
8978 for module_id in configured.keys() {
8979 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
8980 continue;
8981 };
8982 let configured_module = configured
8983 .get(module_id)
8984 .expect("configured module id came from configured map");
8985 if configuration_changes.contains(module_id) {
8986 module
8987 .update_configuration(
8988 configured_module.module_spec(),
8989 configured_module.health.clone(),
8990 configured_module.drain_timeout_ms,
8991 )
8992 .await
8993 .map_err(|err| {
8994 format!(
8995 "failed to update module_id '{module_id}' configuration during rescan: {err}"
8996 )
8997 })?;
8998 }
8999 if *current_enabled != configured_module.enabled {
9002 self.route_outages.mark_operator_action(module_id);
9005 module
9006 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
9007 .await
9008 .map_err(|err| {
9009 self.route_outages.operator_action_ended_unrefused(module_id);
9010 format!(
9011 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
9012 configured_module.enabled
9013 )
9014 })?;
9015 }
9016 }
9017
9018 for module_id in &added {
9019 let configured_module = configured
9020 .get(module_id)
9021 .expect("added module id came from configured map");
9022 supervisor
9023 .supervise_configured_requested_async(
9024 configured_module.module_spec(),
9025 configured_module.enabled,
9026 configured_module.health.clone(),
9027 configured_module.drain_timeout_ms,
9028 configured_module.restart,
9029 configured_module.register_timeout,
9030 Some(requester.clone()),
9031 )
9032 .await
9033 .map_err(|err| {
9034 format!("failed to add module_id '{module_id}' during rescan: {err}")
9035 })?;
9036 }
9037
9038 if let Some(context) = &self.rescan {
9039 let mut entries = context
9040 .module_entries
9041 .lock()
9042 .unwrap_or_else(|e| e.into_inner());
9043 entries.record(&configured, &added, &removed);
9044 }
9045
9046 Ok((
9047 SupervisorRescanResult {
9048 added,
9049 removed,
9050 changed_pending_reload,
9051 changed_pending,
9052 enabled_changes: enabled_changes.iter().cloned().collect(),
9053 unchanged,
9054 preview: false,
9055 restart_required: Vec::new(),
9058 capability_warnings: Vec::new(),
9059 module_actions: Vec::new(),
9060 earlier_pending: Vec::new(),
9061 expect: None,
9062 },
9063 changed,
9064 ))
9065 }
9066
9067 async fn handle_supervisor_set_enabled(
9068 &self,
9069 frame: Frame,
9070 module_id: String,
9071 enabled: bool,
9072 requester: Principal,
9073 ) -> Result<Vec<Frame>, RouterError> {
9074 let operation_lock = self.supervisor.operation_lock();
9075 let _operation_guard = operation_lock.lock().await;
9076 let Some(module) = self.supervisor.get(&module_id) else {
9077 return Ok(vec![control_error_frame(
9078 &frame,
9079 "unknown_module",
9080 format!("module_id '{module_id}' is not supervised"),
9081 )?]);
9082 };
9083
9084 self.route_outages.mark_operator_action(&module_id);
9087 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
9088 Ok(applied) => applied,
9089 Err(err) => {
9090 self.route_outages
9091 .operator_action_ended_unrefused(&module_id);
9092 return Ok(vec![control_error_frame(
9093 &frame,
9094 if err.is_config_gate_error() {
9095 "config_gated"
9096 } else {
9097 "target_unavailable"
9098 },
9099 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
9100 )?]);
9101 }
9102 };
9103 if !applied {
9104 self.route_outages
9107 .operator_action_ended_unrefused(&module_id);
9108 }
9109
9110 self.capability_evaluator.wake_deadline_loop();
9111 self.refresh_capability_requirements();
9112 let response = ClientControlResponse::SupervisorAck { module_id, applied };
9113 Ok(vec![control_response_body_frame(
9114 &frame,
9115 &response,
9116 "ClientControlResponse::SupervisorAck",
9117 )?])
9118 }
9119
9120 async fn handle_supervisor_health_probe(
9121 &self,
9122 ctx: &RouteCtx,
9123 frame: Frame,
9124 module_id: String,
9125 ) -> Result<Vec<Frame>, RouterError> {
9126 self.refresh_capability_requirements();
9127 let Some(registration) = self
9128 .registry
9129 .get_module(&module_id)
9130 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
9131 else {
9132 return Ok(vec![control_error_frame(
9133 &frame,
9134 "unknown_module",
9135 format!("module_id '{module_id}' is not registered"),
9136 )?]);
9137 };
9138
9139 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
9152 {
9153 return Ok(vec![control_error_frame(
9154 &frame,
9155 "health_not_advertised",
9156 format!("module_id '{module_id}' did not advertise health.check"),
9157 )?]);
9158 }
9159
9160 self.observe_control_wait(
9161 ctx,
9162 Some(®istration.manifest.module_id),
9163 ops::SUPERVISOR_HEALTH_PROBE,
9164 async {
9165 let deadline = Instant::now() + self.health_probe_timeout;
9166 let pending = match self.forwarding.begin_module_control_rpc_for(
9167 &module_id,
9168 MODULE_CONTROL_OP_HEALTH_CHECK,
9169 deadline,
9170 ) {
9171 Ok(pending) => pending,
9172 Err(err) => {
9173 return Ok(vec![control_error_frame(
9174 &frame,
9175 forwarding_error_code(&err),
9176 err.to_string(),
9177 )?])
9178 }
9179 };
9180
9181 let PendingModuleControlRpc {
9182 endpoint,
9183 module_sink,
9184 negotiated_ver,
9185 corr: probe_corr,
9186 receiver,
9187 } = pending;
9188 let mut guard =
9189 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
9190 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
9191 .map_err(|err| {
9192 RouterError::backend(
9193 0,
9194 frame.header.corr,
9195 format!("failed to encode health.check request: {err}"),
9196 )
9197 })?;
9198 let probe_frame = Frame::build_with_version(
9199 negotiated_ver,
9200 FrameType::Request,
9201 control_flags(),
9202 0,
9203 0,
9204 probe_corr,
9205 probe_body,
9206 )
9207 .map_err(RouterError::FrameBuild)?;
9208
9209 match timeout_at(deadline, module_sink.send(probe_frame)).await {
9212 Ok(Ok(())) => {}
9213 Ok(Err(err)) => {
9214 return Ok(vec![control_error_frame(
9215 &frame,
9216 "target_unavailable",
9217 err.to_string(),
9218 )?]);
9219 }
9220 Err(_) => {
9221 return Ok(vec![control_error_frame(
9222 &frame,
9223 "module_timeout",
9224 format!(
9225 "module_id '{module_id}' did not answer health.check within {:?}",
9226 self.health_probe_timeout
9227 ),
9228 )?]);
9229 }
9230 }
9231
9232 match timeout_at(deadline, receiver).await {
9233 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => {
9234 Ok(vec![control_error_frame(
9235 &frame,
9236 "invalid_control_body",
9237 "health.check returned an unexpected config response",
9238 )?])
9239 }
9240 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
9241 guard.disarm();
9242 let Some(report) = response.health_report() else {
9243 return Ok(vec![control_error_frame(
9244 &frame,
9245 "invalid_control_body",
9246 "health.check RPC returned a non-health response",
9247 )?]);
9248 };
9249 let HealthReport {
9254 status,
9255 detail,
9256 metrics,
9257 } = report;
9258 let capability_detail = self
9259 .capability_evaluator
9260 .required_problem_detail(&module_id);
9261 let response = ClientControlResponse::SupervisorHealthProbe {
9262 module_id,
9263 status,
9264 detail: append_capability_problem_detail(detail, capability_detail),
9265 metrics,
9266 };
9267 Ok(vec![control_response_body_frame(
9268 &frame,
9269 &response,
9270 "ClientControlResponse::SupervisorHealthProbe",
9271 )?])
9272 }
9273 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
9274 guard.disarm();
9275 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
9280 }
9281 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
9282 guard.disarm();
9283 Ok(vec![control_error_frame(
9284 &frame,
9285 "target_unavailable",
9286 message,
9287 )?])
9288 }
9289 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
9290 guard.disarm();
9291 Ok(vec![control_error_frame(
9292 &frame,
9293 "invalid_control_body",
9294 message,
9295 )?])
9296 }
9297 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
9298 guard.disarm();
9299 Ok(vec![control_error_frame(
9300 &frame,
9301 "invalid_control_body",
9302 format!("expected module-control op '{expected}', got '{actual}'"),
9303 )?])
9304 }
9305 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
9306 guard.disarm();
9307 Ok(vec![control_error_frame(
9308 &frame,
9309 "module_timeout",
9310 format!(
9311 "module_id '{module_id}' answered health.check after {:?}",
9312 self.health_probe_timeout
9313 ),
9314 )?])
9315 }
9316 Ok(Err(_)) => Ok(vec![control_error_frame(
9317 &frame,
9318 "target_unavailable",
9319 "health.check waiter was canceled before the module responded",
9320 )?]),
9321 Err(_) => Ok(vec![control_error_frame(
9322 &frame,
9323 "module_timeout",
9324 format!(
9325 "module_id '{module_id}' did not answer health.check within {:?}",
9326 self.health_probe_timeout
9327 ),
9328 )?]),
9329 }
9330 },
9331 )
9332 .await
9333 }
9334
9335 fn supervisor_status(
9336 &self,
9337 module_id: &str,
9338 corr: u64,
9339 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
9340 self.supervisor
9341 .get(module_id)
9342 .map(|module| {
9343 let warming = module.is_warming_for_control("status").map_err(|err| {
9344 RouterError::backend(
9345 0,
9346 corr,
9347 format!(
9348 "failed to read supervisor warming state for module_id '{module_id}': {err}"
9349 ),
9350 )
9351 })?;
9352 module.status_for_control("status").map_err(|err| {
9353 RouterError::backend(
9354 0,
9355 corr,
9356 format!(
9357 "failed to read supervisor status for module_id '{module_id}': {err}"
9358 ),
9359 )
9360 }).map(|status| (status, warming))
9361 })
9362 .transpose()
9363 }
9364
9365 fn guard_module_control_op(
9366 &self,
9367 frame: &Frame,
9368 module_id: &str,
9369 op: &str,
9370 ) -> Result<Option<Frame>, RouterError> {
9371 if self.module_grants_op(module_id, op, frame.header.corr)? {
9372 return Ok(None);
9373 }
9374
9375 Ok(Some(control_error_frame(
9376 frame,
9377 "op_not_allowed",
9378 format!("module_id '{module_id}' did not grant control op '{op}'"),
9379 )?))
9380 }
9381
9382 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
9383 let Some(registration) = self
9384 .registry
9385 .get_module(module_id)
9386 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
9387 else {
9388 return Ok(false);
9389 };
9390 Ok(module_registration_grants_op(®istration.control_ops, op))
9391 }
9392
9393 fn handle_status_update(
9394 &self,
9395 endpoint: ModuleEndpointId,
9396 frame: Frame,
9397 ) -> Result<Vec<Frame>, RouterError> {
9398 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
9399 Ok(update) => update,
9400 Err(err) => {
9401 if is_known_module_push_op(&frame.body) {
9406 return Ok(vec![control_error_frame(
9407 &frame,
9408 "invalid_control_body",
9409 format!("malformed module control push body: {err}"),
9410 )?]);
9411 }
9412 return Ok(Vec::new());
9413 }
9414 };
9415
9416 match update {
9417 ModuleControlPush::RouteStatus {
9418 route_channel,
9419 route_epoch,
9420 status,
9421 } => {
9422 self.forwarding
9423 .cache_status(endpoint, route_channel, route_epoch, status)
9424 .map_err(RouterError::Forwarding)?;
9425 }
9426 }
9427 Ok(Vec::new())
9428 }
9429
9430 fn handle_route_poll(
9431 &self,
9432 ctx: &RouteCtx,
9433 frame: Frame,
9434 route_channel: u16,
9435 route_epoch: u32,
9436 kind: PollKind,
9437 ) -> Result<Vec<Frame>, RouterError> {
9438 let snapshot = self
9439 .forwarding
9440 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
9441 .map_err(RouterError::Forwarding)?;
9442 let response = match (kind, snapshot) {
9443 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
9444 ClientControlResponse::RoutePoll {
9445 route_channel,
9446 route_epoch,
9447 status,
9448 live: None,
9449 }
9450 }
9451 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9452 route_channel,
9453 route_epoch,
9454 status: None,
9455 live: None,
9456 },
9457 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
9458 let live = self
9484 .process_liveness
9485 .as_ref()
9486 .and_then(|source| source.process_live(&module_id))
9487 .unwrap_or(true);
9488 ClientControlResponse::RoutePoll {
9489 route_channel,
9490 route_epoch,
9491 status: None,
9492 live: Some(live),
9493 }
9494 }
9495 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9496 route_channel,
9497 route_epoch,
9498 status: None,
9499 live: Some(false),
9500 },
9501 };
9502
9503 Ok(vec![control_response_body_frame(
9504 &frame,
9505 &response,
9506 "ClientControlResponse::RoutePoll",
9507 )?])
9508 }
9509
9510 pub(crate) fn observe_module_control_completion(
9511 &self,
9512 completion: ModuleControlRpcCompletion,
9513 ) -> bool {
9514 match completion {
9515 ModuleControlRpcCompletion::Unknown => false,
9516 ModuleControlRpcCompletion::Settled => true,
9517 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
9518 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
9519 info!(
9520 module_id = %module_id,
9521 latency_ms,
9522 "late health.check answer proves the module is alive"
9523 );
9524 match self
9525 .supervisor
9526 .record_late_health_answer(&module_id, latency_ms)
9527 {
9528 Ok(true) => {}
9529 Ok(false) => debug!(
9530 module_id = %module_id,
9531 latency_ms,
9532 "late health.check answer has no active supervisor snapshot"
9533 ),
9534 Err(err) => warn!(
9535 module_id = %module_id,
9536 latency_ms,
9537 error = %err,
9538 "failed to record late health.check answer"
9539 ),
9540 }
9541 true
9542 }
9543 }
9544 }
9545
9546 fn refuse_to_end_module_connection_for_a_client(
9567 &self,
9568 module_connection_id: ConnectionId,
9569 corr: u64,
9570 err: ForwardingError,
9571 ) -> Result<(), RouterError> {
9572 if let ForwardingError::ConnectionClosing { connection_id } = err {
9573 if connection_id != module_connection_id {
9574 warn!(
9575 module_connection_id = module_connection_id.get(),
9576 client_connection_id = connection_id.get(),
9577 corr,
9578 "dropping a route.bind response for a closing client; the module connection keeps serving"
9579 );
9580 return Ok(());
9581 }
9582 }
9583 Err(RouterError::Forwarding(err))
9584 }
9585
9586 fn handle_module_relay_response(
9587 &self,
9588 connection_id: ConnectionId,
9589 frame: Frame,
9590 ) -> Result<Vec<Frame>, RouterError> {
9591 let mut secondary_error = None;
9592 let outcome = match frame.header.ty {
9593 FrameType::Response => match module_control_response_probe(&frame.body) {
9594 Ok(probe) if probe.op == "route.bind" => {
9595 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9596 Ok(ModuleControlResponse::RouteBindAck {}) => {
9597 RouteBindRelayOutcome::Accepted
9598 }
9599 Ok(other) => {
9600 let message =
9601 format!("route.bind response carried unexpected body: {other:?}");
9602 secondary_error = Some(control_error_frame(
9603 &frame,
9604 "invalid_control_body",
9605 message.clone(),
9606 )?);
9607 RouteBindRelayOutcome::ModuleGone(message)
9608 }
9609 Err(err) => {
9610 let message = format!("malformed route.bind response body: {err}");
9611 secondary_error = Some(control_error_frame(
9612 &frame,
9613 "invalid_control_body",
9614 message.clone(),
9615 )?);
9616 RouteBindRelayOutcome::ModuleGone(message)
9617 }
9618 }
9619 }
9620 Ok(probe) => {
9621 let outcome = if matches!(probe.op.as_str(), "config.review" | "consent/v1") {
9622 ModuleControlRpcOutcome::RawResponse(frame.body.clone())
9623 } else {
9624 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9625 Ok(response) => ModuleControlRpcOutcome::Response(response),
9626 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9627 "malformed {} response body: {err}",
9628 probe.op
9629 )),
9630 }
9631 };
9632 let completion = self
9633 .forwarding
9634 .complete_module_control_rpc(
9635 connection_id,
9636 frame.header.corr,
9637 Some(&probe.op),
9638 outcome,
9639 )
9640 .map_err(RouterError::Forwarding)?;
9641 if !self.observe_module_control_completion(completion) {
9642 debug!(
9643 connection_id = connection_id.get(),
9644 corr = frame.header.corr,
9645 op = %probe.op,
9646 "dropping late or unknown module-control RPC response"
9647 );
9648 }
9649 return Ok(Vec::new());
9650 }
9651 Err(err) => {
9652 if let Some(expected_op) = self
9653 .forwarding
9654 .pending_module_control_op(connection_id, frame.header.corr)
9655 .map_err(RouterError::Forwarding)?
9656 {
9657 let completion = self
9658 .forwarding
9659 .complete_module_control_rpc(
9660 connection_id,
9661 frame.header.corr,
9662 None,
9663 ModuleControlRpcOutcome::MalformedResponse(format!(
9664 "malformed {expected_op} response body: {err}"
9665 )),
9666 )
9667 .map_err(RouterError::Forwarding)?;
9668 if !self.observe_module_control_completion(completion) {
9669 debug!(
9670 connection_id = connection_id.get(),
9671 corr = frame.header.corr,
9672 "dropping late malformed module-control RPC response"
9673 );
9674 }
9675 return Ok(Vec::new());
9676 }
9677 let message = format!("malformed route.bind response body: {err}");
9678 secondary_error = Some(control_error_frame(
9679 &frame,
9680 "invalid_control_body",
9681 message.clone(),
9682 )?);
9683 RouteBindRelayOutcome::ModuleGone(message)
9684 }
9685 },
9686 FrameType::Error => {
9687 if self
9688 .forwarding
9689 .pending_module_control_op(connection_id, frame.header.corr)
9690 .map_err(RouterError::Forwarding)?
9691 .is_some()
9692 {
9693 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
9694 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
9695 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9696 "malformed module-control ERROR body: {err}"
9697 )),
9698 };
9699 let completion = self
9700 .forwarding
9701 .complete_module_control_rpc(
9702 connection_id,
9703 frame.header.corr,
9704 None,
9705 outcome,
9706 )
9707 .map_err(RouterError::Forwarding)?;
9708 if !self.observe_module_control_completion(completion) {
9709 debug!(
9710 connection_id = connection_id.get(),
9711 corr = frame.header.corr,
9712 "dropping late or unknown module-control RPC error"
9713 );
9714 }
9715 return Ok(Vec::new());
9716 }
9717 match serde_json::from_slice::<ErrorBody>(&frame.body) {
9718 Ok(body) => RouteBindRelayOutcome::Rejected(body),
9719 Err(err) => {
9720 let message = format!("malformed route.bind ERROR body: {err}");
9721 secondary_error = Some(control_error_frame(
9722 &frame,
9723 "invalid_control_body",
9724 message.clone(),
9725 )?);
9726 RouteBindRelayOutcome::ModuleGone(message)
9727 }
9728 }
9729 }
9730 ty => {
9731 return Ok(vec![control_error_frame(
9732 &frame,
9733 "unsupported_control_frame",
9734 format!("unsupported module channel-0 frame {ty:?}"),
9735 )?])
9736 }
9737 };
9738
9739 let settled =
9740 self.forwarding
9741 .complete_pending_relay(connection_id, frame.header.corr, outcome);
9742 let completion = match settled {
9743 Ok(completion) => completion,
9744 Err(err) => {
9745 self.refuse_to_end_module_connection_for_a_client(
9746 connection_id,
9747 frame.header.corr,
9748 err,
9749 )?;
9750 return Ok(secondary_error.into_iter().collect());
9751 }
9752 };
9753 if let Some(target) = completion.abandoned.as_ref() {
9754 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
9755 }
9756 if !completion.settled {
9757 debug!(
9758 connection_id = connection_id.get(),
9759 corr = frame.header.corr,
9760 frame_type = ?frame.header.ty,
9761 "dropping late or unknown route.bind relay response"
9762 );
9763 }
9764 Ok(secondary_error.into_iter().collect())
9765 }
9766
9767 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
9768 debug!(connection_id = connection_id.get(), "handling GOODBYE");
9769 self.cleanup_connection_with_end_reason(
9773 connection_id,
9774 RegistrationEndReason::ExplicitGoodbye,
9775 )
9776 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
9777 Ok(Vec::new())
9778 }
9779}
9780
9781impl Default for ControlHandler {
9782 fn default() -> Self {
9783 Self::new(Arc::new(Registry::default()))
9784 }
9785}
9786
9787impl crate::supervise::SwapPromotionObserver for ControlHandler {
9788 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
9789 self.apply_registration_capabilities(registration);
9790 }
9791}
9792
9793fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
9794 CapabilityRequirementStatus {
9795 consumer: status.consumer,
9796 capability: status.capability,
9797 need: match status.need {
9798 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
9799 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
9800 },
9801 verdict: status.verdict.as_str().to_string(),
9802 episode_seq: status.episode_seq,
9803 config_satisfiable: status.config_satisfiable,
9804 runtime_available: status.runtime_available,
9805 detail: status.detail,
9806 }
9807}
9808
9809fn append_capability_problem_detail(
9810 detail: Option<String>,
9811 capability_detail: Option<String>,
9812) -> Option<String> {
9813 match (detail, capability_detail) {
9814 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
9815 (Some(detail), None) => Some(detail),
9816 (None, Some(capability_detail)) => Some(capability_detail),
9817 (None, None) => None,
9818 }
9819}
9820
9821fn subc_ops() -> Vec<String> {
9822 SUBC_CONTROL_OPS
9823 .iter()
9824 .map(|op| (*op).to_string())
9825 .collect()
9826}
9827
9828fn module_subc_ops() -> Vec<String> {
9829 SUBC_CONTROL_OPS
9830 .iter()
9831 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
9832 .map(|op| (*op).to_string())
9833 .collect()
9834}
9835
9836#[cfg(test)]
9837fn module_baseline_control_ops() -> Vec<String> {
9838 MODULE_BASELINE_CONTROL_OPS
9839 .iter()
9840 .map(|op| (*op).to_string())
9841 .collect()
9842}
9843
9844fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
9845 let mut seen = HashSet::new();
9846 let mut effective = Vec::new();
9847 for op in MODULE_BASELINE_CONTROL_OPS {
9848 if seen.insert((*op).to_string()) {
9849 effective.push((*op).to_string());
9850 }
9851 }
9852 for op in declared.unwrap_or_default() {
9853 if seen.insert(op.clone()) {
9854 effective.push(op);
9855 }
9856 }
9857 effective
9858}
9859
9860fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
9861 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
9862}
9863
9864fn target_module_id(target: &RouteTarget) -> &str {
9865 match target {
9866 RouteTarget::ToolProvider { module_id }
9867 | RouteTarget::ManagementSurface { module_id }
9868 | RouteTarget::InternalService { module_id, .. } => module_id,
9869 }
9870}
9871
9872fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
9873 roles.iter().any(|role| match (target, role) {
9874 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
9875 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
9876 (
9877 RouteTarget::InternalService { service_id, .. },
9878 ProviderRole::InternalService {
9879 service_id: provided,
9880 ..
9881 },
9882 ) => service_id == provided,
9883 _ => false,
9884 })
9885}
9886
9887fn is_routable_role(role: &ProviderRole) -> bool {
9888 matches!(
9889 role,
9890 ProviderRole::ToolProvider { .. }
9891 | ProviderRole::ManagementSurface { .. }
9892 | ProviderRole::InternalService { .. }
9893 )
9894}
9895
9896#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9897enum ControlRequestBodyError {
9898 UnknownOp,
9899 InvalidBody,
9900}
9901
9902#[derive(Debug, Deserialize)]
9903struct ControlOpProbe {
9904 op: String,
9905}
9906
9907fn module_control_response_probe(body: &[u8]) -> Result<ControlOpProbe, serde_json::Error> {
9908 serde_json::from_slice(body).or_else(|original| {
9909 #[derive(Deserialize)]
9910 struct ConsentProbe {
9911 protocol: String,
9912 }
9913 match serde_json::from_slice::<ConsentProbe>(body) {
9914 Ok(probe) if probe.protocol == subc_protocol::consent::CONSENT_V1 => {
9915 Ok(ControlOpProbe { op: probe.protocol })
9916 }
9917 _ => Err(original),
9918 }
9919 })
9920}
9921
9922const MODULE_PUSH_OPS: &[&str] = &["route.status"];
9925
9926fn is_known_module_push_op(body: &[u8]) -> bool {
9927 serde_json::from_slice::<ControlOpProbe>(body)
9928 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
9929 .unwrap_or(false)
9930}
9931
9932fn is_known_module_request_op(body: &[u8]) -> bool {
9933 serde_json::from_slice::<ControlOpProbe>(body)
9934 .map(|probe| is_module_to_subc_op(&probe.op))
9935 .unwrap_or(false)
9936}
9937
9938fn is_module_to_subc_op(op: &str) -> bool {
9939 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
9940}
9941
9942fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
9943 debug!(
9944 op = %op,
9945 connection_id = connection_id.get(),
9946 corr,
9947 "control dispatch"
9948 );
9949}
9950
9951fn log_slow_control_dispatch(
9952 dispatch_started_at: Option<StdInstant>,
9953 op: &'static str,
9954 connection_id: ConnectionId,
9955 corr: u64,
9956 route_open_timings: Option<&RouteOpenTimings>,
9957) {
9958 let Some(dispatch_started_at) = dispatch_started_at else {
9959 return;
9960 };
9961 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
9963 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
9964 if let Some(timings) = route_open_timings {
9965 warn!(
9966 op = %op,
9967 connection_id = connection_id.get(),
9968 corr,
9969 elapsed_ms = elapsed.as_millis() as u64,
9970 module_id = ?timings.module_id,
9971 principal = %timings.principal,
9972 admission_ms = timings.admission.as_millis() as u64,
9973 project_root_ms = timings.project_root.as_millis() as u64,
9974 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
9975 bind_send_ms = timings.bind_send.as_millis() as u64,
9976 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
9977 "slow control dispatch"
9978 );
9979 return;
9980 }
9981 warn!(
9982 op = %op,
9983 connection_id = connection_id.get(),
9984 corr,
9985 elapsed_ms = elapsed.as_millis() as u64,
9986 "slow control dispatch"
9987 );
9988 }
9989}
9990
9991#[cfg(test)]
9993type TestCallerLookup =
9994 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
9995
9996fn privileged_operation(request: &impl PrivilegedRequest) -> Option<PrivilegedOperation<'_>> {
9998 request.privileged_operation()
9999}
10000
10001trait PrivilegedRequest {
10002 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>>;
10003}
10004
10005impl PrivilegedRequest for ClientControlRequest {
10006 fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>> {
10007 let request = self;
10008 match request {
10009 ClientControlRequest::SupervisorRestart { module_id, .. } => {
10010 Some(PrivilegedOperation::Restart { module_id })
10011 }
10012 ClientControlRequest::SupervisorSwap { module_id, .. } => {
10013 Some(PrivilegedOperation::Swap { module_id })
10014 }
10015 ClientControlRequest::SupervisorReload { module_id } => {
10016 Some(PrivilegedOperation::Reload { module_id })
10017 }
10018 ClientControlRequest::SupervisorRescan { preview } => {
10019 if *preview {
10020 None
10021 } else {
10022 Some(PrivilegedOperation::Rescan)
10023 }
10024 }
10025 ClientControlRequest::SupervisorRescanChecked {
10030 preview,
10031 candidate,
10032 expect,
10033 } => (!*preview && candidate.is_none() && expect.is_some())
10034 .then_some(PrivilegedOperation::Rescan),
10035 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
10036 Some(PrivilegedOperation::SetEnabled {
10037 module_id,
10038 enabled: *enabled,
10039 })
10040 }
10041 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
10042 Some(PrivilegedOperation::ReleaseReserved { module_id })
10043 }
10044 ClientControlRequest::ServerDescribe {}
10045 | ClientControlRequest::DaemonShutdownIntent { .. }
10048 | ClientControlRequest::CatalogList { .. }
10049 | ClientControlRequest::RouteOpen { .. }
10050 | ClientControlRequest::RoutePoll { .. }
10051 | ClientControlRequest::SupervisorList {}
10052 | ClientControlRequest::SupervisorHealth {}
10053 | ClientControlRequest::SupervisorHealthProbe { .. }
10054 | ClientControlRequest::SupervisorRoutes { .. }
10055 | ClientControlRequest::SupervisorProvenance { .. }
10056 | ClientControlRequest::SupervisorStderrTail { .. }
10057 | ClientControlRequest::SupervisorTerminals { .. }
10058 | ClientControlRequest::SupervisorSpawnSnapshot {}
10059 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
10060 }
10061 }
10062}
10063
10064fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
10065 match request {
10066 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
10067 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
10068 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
10069 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
10070 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
10071 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
10072 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
10073 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
10074 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
10075 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
10076 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
10077 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
10078 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
10079 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
10080 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
10081 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
10082 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
10083 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
10084 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
10085 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
10086 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
10087 }
10088}
10089
10090struct RouteOpenCaller {
10098 principal: String,
10102 scope: String,
10105 harness: String,
10108}
10109
10110impl RouteOpenCaller {
10111 fn new(
10112 principal: Option<&Principal>,
10113 scope: Option<&ScopeSelector>,
10114 identity: &BindIdentity,
10115 ) -> Self {
10116 Self {
10117 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
10118 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
10119 harness: bounded_log_text(&identity.harness),
10120 }
10121 }
10122
10123 fn unknown() -> Self {
10126 Self {
10127 principal: "unknown".to_string(),
10128 scope: "unknown".to_string(),
10129 harness: "unknown".to_string(),
10130 }
10131 }
10132}
10133
10134const MAX_LOGGED_TEXT_CHARS: usize = 64;
10137
10138fn bounded_log_text(value: &str) -> String {
10142 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
10143 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
10144 bounded.push('…');
10145 }
10146 format!("{bounded:?}")
10147}
10148
10149fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
10155 use subc_protocol::scope::ScopeKind;
10156 let kind = match stamp.kind {
10157 ScopeKind::Head => "head",
10158 ScopeKind::Worker => "worker",
10159 ScopeKind::Ephemeral => "ephemeral",
10160 ScopeKind::Federated => "federated",
10161 _ => "unknown",
10162 };
10163 format!(
10164 "{kind}/{}/{}@{}",
10165 principal_label(&stamp.owner),
10166 bounded_log_text(&stamp.scope_ref),
10167 stamp.scope_epoch
10168 )
10169}
10170
10171fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
10178 let epoch = selector
10179 .scope_epoch
10180 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
10181 format!(
10182 "requested/{}/{}@{epoch}",
10183 bounded_log_text(&principal_label(&selector.owner)),
10184 bounded_log_text(&selector.scope_ref),
10185 )
10186}
10187
10188fn principal_label(principal: &Principal) -> String {
10189 match principal {
10190 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
10191 Principal::Direct => "direct".to_string(),
10192 other => format!("{other:?}"),
10193 }
10194}
10195
10196fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
10197 match request {
10198 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
10199 ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
10200 MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
10201 }
10202 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
10203 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
10204 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
10205 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
10206 ModuleControlRequestFromModule::RouteAuthorize { .. } => {
10207 subc_protocol::scope::ROUTE_AUTHORIZE_OP
10208 }
10209 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
10210 _ => "unsupported",
10211 }
10212}
10213
10214fn parse_client_control_request(
10215 body: &[u8],
10216) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
10217 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
10218 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10219 Ok(probe)
10220 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
10221 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
10222 {
10223 ControlRequestBodyError::InvalidBody
10224 }
10225 Ok(_) => ControlRequestBodyError::UnknownOp,
10226 Err(_) => ControlRequestBodyError::InvalidBody,
10227 };
10228 (err, classification)
10229 })
10230}
10231
10232fn parse_module_control_request_from_module(
10233 body: &[u8],
10234) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
10235 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
10236 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10237 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
10238 Ok(_) => ControlRequestBodyError::UnknownOp,
10239 Err(_) => ControlRequestBodyError::InvalidBody,
10240 };
10241 (err, classification)
10242 })
10243}
10244
10245#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
10246enum ProviderRoleKind {
10247 ToolProvider,
10248 PipelineStage,
10249 ManagementSurface,
10250 InternalService,
10251}
10252
10253fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
10254 match role {
10255 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
10256 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
10257 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
10258 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
10259 }
10260}
10261
10262fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
10263 roles.iter().map(provider_role_kind).collect()
10264}
10265
10266const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
10269
10270#[derive(Debug, Default, PartialEq, Eq)]
10272struct ScopeOutcomeCounts {
10273 created: usize,
10274 replaced: usize,
10275 updated: usize,
10276 unchanged: usize,
10277 refused: usize,
10278}
10279
10280impl ScopeOutcomeCounts {
10281 fn of(results: &[ScopeRecordResult]) -> Self {
10282 let mut counts = Self::default();
10283 for result in results {
10284 let slot = match result.outcome {
10285 ScopeRecordOutcome::Created => &mut counts.created,
10286 ScopeRecordOutcome::Replaced => &mut counts.replaced,
10287 ScopeRecordOutcome::Updated => &mut counts.updated,
10288 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
10289 ScopeRecordOutcome::Refused => &mut counts.refused,
10290 };
10291 *slot += 1;
10292 }
10293 counts
10294 }
10295}
10296
10297#[cfg(test)]
10298mod scope_outcome_count_tests {
10299 use super::*;
10300
10301 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
10302 ScopeRecordResult {
10303 scope_ref: "r".to_string(),
10304 scope_epoch: 1,
10305 outcome,
10306 code: None,
10307 message: None,
10308 version: None,
10309 parent_state: None,
10310 }
10311 }
10312
10313 #[test]
10316 fn every_outcome_is_counted_in_its_own_field() {
10317 let results = [
10318 result(ScopeRecordOutcome::Created),
10319 result(ScopeRecordOutcome::Created),
10320 result(ScopeRecordOutcome::Replaced),
10321 result(ScopeRecordOutcome::Updated),
10322 result(ScopeRecordOutcome::Unchanged),
10323 result(ScopeRecordOutcome::Refused),
10324 result(ScopeRecordOutcome::Refused),
10325 result(ScopeRecordOutcome::Refused),
10326 ];
10327 assert_eq!(
10328 ScopeOutcomeCounts::of(&results),
10329 ScopeOutcomeCounts {
10330 created: 2,
10331 replaced: 1,
10332 updated: 1,
10333 unchanged: 1,
10334 refused: 3,
10335 }
10336 );
10337 }
10338}
10339
10340fn capability_census_trigger(
10344 old: Option<&CapabilityDeclarations>,
10345 new: Option<&CapabilityDeclarations>,
10346) -> bool {
10347 let old_provides = old
10348 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
10349 .unwrap_or_default();
10350 let old_denies = old
10351 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
10352 .unwrap_or_default();
10353 let new = new.cloned().unwrap_or(CapabilityDeclarations {
10354 provides: Vec::new(),
10355 requires: Vec::new(),
10356 must_never_reach: Vec::new(),
10357 });
10358
10359 new.provides
10360 .iter()
10361 .any(|capability| !old_provides.contains(capability))
10362 || new
10363 .must_never_reach
10364 .iter()
10365 .any(|capability| !old_denies.contains(capability))
10366}
10367
10368fn catalog_update_frozen_field_message(
10369 registered: &ModuleManifest,
10370 provides: &[ProviderRole],
10371) -> Option<String> {
10372 let old_has_provides = !registered.provides.is_empty();
10373 let new_has_provides = !provides.is_empty();
10374 if old_has_provides != new_has_provides {
10375 return Some(format!(
10376 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
10377 registered.module_id
10378 ));
10379 }
10380
10381 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
10382 return Some(format!(
10383 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
10384 registered.module_id
10385 ));
10386 }
10387
10388 let registered_concurrency = manifest_concurrency(registered);
10389 let mut candidate = registered.clone();
10390 candidate.provides = provides.to_vec();
10391 let candidate_concurrency = manifest_concurrency(&candidate);
10392 if candidate_concurrency != registered_concurrency {
10393 return Some(format!(
10394 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
10395 registered.module_id, registered_concurrency, candidate_concurrency
10396 ));
10397 }
10398
10399 None
10402}
10403
10404fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
10405 manifest.provides.iter().any(is_routable_role)
10406}
10407
10408fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
10414 manifest
10415 .provides
10416 .iter()
10417 .find_map(|provider| match provider {
10418 ProviderRole::ToolProvider { concurrency, .. }
10419 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
10420 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
10421 })
10422 .unwrap_or(Concurrency::ModuleManaged)
10423}
10424
10425fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
10432 let has_management_surface = manifest
10433 .provides
10434 .iter()
10435 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
10436 if !has_management_surface {
10437 return false;
10438 }
10439 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
10440 return false;
10441 };
10442 let Some(provides) = raw
10443 .get("manifest")
10444 .and_then(|manifest| manifest.get("provides"))
10445 .and_then(serde_json::Value::as_array)
10446 else {
10447 return false;
10448 };
10449 provides.iter().any(|role| {
10454 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
10455 && role.get("concurrency").is_none()
10456 })
10457}
10458
10459fn negotiate_version(peer_version: u8) -> Result<u8, String> {
10460 if peer_version != PROTOCOL_VERSION {
10461 return Err(format!(
10462 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
10463 ));
10464 }
10465 Ok(PROTOCOL_VERSION)
10466}
10467
10468fn pong(frame: &Frame) -> Result<Frame, RouterError> {
10469 Frame::build_with_version(
10470 response_version(frame),
10471 FrameType::Pong,
10472 frame.header.flags,
10473 0,
10474 0,
10475 frame.header.corr,
10476 Vec::new(),
10477 )
10478 .map_err(RouterError::FrameBuild)
10479}
10480
10481pub(crate) fn control_error_frame(
10482 frame: &Frame,
10483 code: &'static str,
10484 message: impl Into<String>,
10485) -> Result<Frame, RouterError> {
10486 control_error_body_frame(
10487 frame,
10488 ErrorBody {
10489 code: code.to_string(),
10490 message: message.into(),
10491 detail: None,
10492 },
10493 )
10494}
10495
10496pub(crate) fn control_error_body_frame(
10504 frame: &Frame,
10505 error: ErrorBody,
10506) -> Result<Frame, RouterError> {
10507 control_error_frame_with_flags(frame, error, daemon_error_flags())
10508}
10509
10510fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
10516 control_error_frame_with_flags(frame, error, control_flags())
10517}
10518
10519fn control_error_frame_with_flags(
10520 frame: &Frame,
10521 error: ErrorBody,
10522 flags: Flags,
10523) -> Result<Frame, RouterError> {
10524 let body = serde_json::to_vec(&error).map_err(|err| {
10525 RouterError::backend(
10526 0,
10527 frame.header.corr,
10528 format!("failed to encode control ERROR: {err}"),
10529 )
10530 })?;
10531
10532 Frame::build_with_version(
10533 response_version(frame),
10534 FrameType::Error,
10535 flags,
10536 0,
10537 0,
10538 frame.header.corr,
10539 body,
10540 )
10541 .map_err(RouterError::FrameBuild)
10542}
10543
10544pub(crate) fn control_response_body_frame<T: Serialize>(
10545 frame: &Frame,
10546 reply: &T,
10547 label: &'static str,
10548) -> Result<Frame, RouterError> {
10549 let body = serde_json::to_vec(reply).map_err(|err| {
10550 RouterError::backend(
10551 0,
10552 frame.header.corr,
10553 format!("failed to encode {label}: {err}"),
10554 )
10555 })?;
10556
10557 Frame::build_with_version(
10558 response_version(frame),
10559 FrameType::Response,
10560 control_flags(),
10561 0,
10562 0,
10563 frame.header.corr,
10564 body,
10565 )
10566 .map_err(RouterError::FrameBuild)
10567}
10568
10569struct HelloAuthority {
10602 record: Arc<crate::spawn_authority::SpawnAuthorityRecord>,
10603 requested: ProviderDeclarations,
10604 froze: std::sync::atomic::AtomicBool,
10607}
10608
10609fn forwarding_error_code(err: &ForwardingError) -> &'static str {
10610 match err {
10611 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
10612 ForwardingError::NoModuleConnection => "target_unavailable",
10613 ForwardingError::ModuleReloading { .. } => "module_reloading",
10614 ForwardingError::ClientRouteChannelExhausted { .. }
10615 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
10616 ForwardingError::StaleModuleEndpoint
10617 | ForwardingError::UnknownReservation { .. }
10618 | ForwardingError::ConnectionClosing { .. }
10619 | ForwardingError::ClientEgressClosed { .. }
10620 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
10621 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
10624 ForwardingError::RelayCorrelationExhausted
10625 | ForwardingError::RouteOpenBuild(_)
10626 | ForwardingError::Poisoned => "forwarding_error",
10627 }
10628}
10629
10630fn response_version(frame: &Frame) -> u8 {
10631 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
10632 frame.header.ver
10633 } else {
10634 PROTOCOL_VERSION
10635 }
10636}
10637
10638fn control_flags() -> Flags {
10639 Flags::new(false, Priority::Passive, false)
10640}
10641
10642fn send_goodbye_target_best_effort(
10647 counters: &DaemonCounters,
10648 target: &GoodbyeTarget,
10649 context: &'static str,
10650) {
10651 let Ok(frame) = Frame::build_with_version(
10652 target.negotiated_ver,
10653 FrameType::Goodbye,
10654 control_flags(),
10655 target.channel,
10656 target.epoch,
10657 0,
10658 Vec::new(),
10659 ) else {
10660 return;
10661 };
10662 crate::forwarding::send_module_route_goodbye(
10663 counters,
10664 &target.sink,
10665 frame,
10666 target.module_id.as_deref(),
10667 context,
10668 );
10669}
10670
10671fn route_close_reason_for_opener(
10672 reason: RouteCloseReason,
10673 accepts_close_reasons: Option<&[String]>,
10674) -> RouteCloseReason {
10675 if reason == RouteCloseReason::AuthorityChanged
10676 && !accepts_close_reasons
10677 .is_some_and(|reasons| reasons.iter().any(|reason| reason == "authority_changed"))
10678 {
10679 RouteCloseReason::Reload
10682 } else {
10683 reason
10684 }
10685}
10686
10687pub(crate) fn send_route_control_pushes(
10688 forwarding: &ForwardingTable,
10689 routes: Vec<EndpointRoute>,
10690 push: ClientControlPush,
10691) {
10692 let reason = match &push {
10693 ClientControlPush::RouteClosing { reason, .. }
10694 | ClientControlPush::RouteClosed { reason, .. } => *reason,
10695 };
10696 let mut targets: Vec<(GoodbyeTarget, RouteCloseReason, Vec<u16>)> = Vec::new();
10697 for route in routes {
10698 let recipient_reason =
10699 route_close_reason_for_opener(reason, route.accepts_close_reasons.as_deref());
10700 let target = route.goodbye_target;
10701 if let Some((existing, _, channels)) = targets.iter_mut().find(|(existing, reason, _)| {
10702 existing.connection_id == target.connection_id && *reason == recipient_reason
10703 }) {
10704 debug_assert_eq!(
10705 existing.negotiated_ver, target.negotiated_ver,
10706 "one connection cannot negotiate multiple frame versions"
10707 );
10708 if !channels.contains(&target.channel) {
10709 channels.push(target.channel);
10710 }
10711 continue;
10712 }
10713 let channel = target.channel;
10714 targets.push((target, recipient_reason, vec![channel]));
10715 }
10716 for (target, recipient_reason, mut channels) in targets {
10717 channels.sort_unstable();
10718 let mut push = push.clone();
10719 match &mut push {
10720 ClientControlPush::RouteClosing {
10721 channels: covered,
10722 reason,
10723 ..
10724 }
10725 | ClientControlPush::RouteClosed {
10726 channels: covered,
10727 reason,
10728 ..
10729 } => {
10730 *covered = channels;
10731 *reason = recipient_reason;
10732 }
10733 }
10734 let body = match serde_json::to_vec(&push) {
10735 Ok(body) => body,
10736 Err(err) => {
10737 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
10738 continue;
10739 }
10740 };
10741 let frame = match Frame::build_with_version(
10742 target.negotiated_ver,
10743 FrameType::Push,
10744 control_flags(),
10745 0,
10746 0,
10747 0,
10748 body.clone(),
10749 ) {
10750 Ok(frame) => frame,
10751 Err(err) => {
10752 warn!(
10753 route_channel = target.channel,
10754 error = %err,
10755 "failed to build route lifecycle control PUSH frame"
10756 );
10757 continue;
10758 }
10759 };
10760 if let Err(err) = target.sink.try_send(frame) {
10761 if target.close_on_delivery_failure() {
10762 warn!(
10763 target_connection_id = target.connection_id.get(),
10764 route_channel = target.channel,
10765 error = %err,
10766 "route lifecycle control PUSH was not delivered to client; closing target connection"
10767 );
10768 let _ = forwarding.escalate_client_delivery_failure(
10769 target.connection_id,
10770 target.channel,
10771 target.epoch,
10772 CloseReason::new(
10773 "route_lifecycle_push_delivery_failed",
10774 format!(
10775 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
10776 target.channel
10777 ),
10778 ),
10779 crate::forwarding::UndeliveredFrame {
10780 module_id: target.module_id.as_deref(),
10781 sink: &target.sink,
10782 },
10783 );
10784 }
10785 }
10786 }
10787}
10788
10789#[cfg(test)]
10790mod tests {
10791 include!("control/agent_commands_tests.rs");
10792 include!("control/resource_revision_tests.rs");
10793 include!("control/route_open_recovery_tests.rs");
10794 #[tokio::test]
10795 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
10796 let registry = Arc::new(Registry::default());
10797 let handler = ControlHandler::new(registry);
10798 assert_eq!(
10799 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10800 None
10801 );
10802 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
10803 hello_via_sink(
10804 &handler,
10805 &ctx,
10806 &mut rx,
10807 hello_frame("daemon", PROTOCOL_VERSION, 1),
10808 )
10809 .await;
10810 assert_eq!(
10811 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10812 Some("daemon".into())
10813 );
10814 assert_eq!(
10815 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
10816 None
10817 );
10818 assert_eq!(
10819 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
10820 None,
10821 "a registered module is not necessarily supervised"
10822 );
10823 assert_eq!(
10824 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
10825 None,
10826 "aggregate provenance has no single target"
10827 );
10828 assert_eq!(
10829 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
10830 None
10831 );
10832 }
10833
10834 mod peer_manifest {
10835 include!("peer_manifest_tests.rs");
10836 }
10837 mod catalog_declarations {
10838 include!("control/catalog_tests.rs");
10839 }
10840 mod activity_read {
10841 include!("activity_read_tests.rs");
10842 }
10843
10844 mod lifecycle_activity {
10845 include!("control_lifecycle_tests.rs");
10846 }
10847 #[cfg(unix)]
10848 #[tokio::test]
10849 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
10850 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
10851 let path = dir.join("subc.jsonc");
10852 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
10853 "program":"/bin/sleep","args":["60"],"protocol":"none",
10854 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
10855 }}}).to_string()).unwrap();
10856 let mut configured = crate::daemon_config::load(&path)
10857 .unwrap()
10858 .unwrap()
10859 .modules
10860 .pop()
10861 .unwrap();
10862 let registry = std::sync::Arc::new(crate::Registry::default());
10863 let handle = crate::SupervisorHandle::new();
10864 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
10865 .with_handle(handle.clone());
10866 let module = supervisor
10867 .supervise_configured_with_health(
10868 configured.module_spec(),
10869 true,
10870 configured.health.clone(),
10871 None,
10872 configured.restart,
10873 )
10874 .unwrap();
10875 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
10876 let before = module.status().unwrap().pid;
10877 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
10878 configured.health.cadence = std::time::Duration::from_secs(3600);
10879 let health_only = handler
10880 .reconcile_supervised_modules(
10881 &supervisor,
10882 vec![configured.clone()],
10883 false,
10884 Principal::Direct,
10885 )
10886 .await
10887 .unwrap();
10888 assert!(
10889 health_only.0.changed_pending_reload.is_empty(),
10890 "health policy is already applied live"
10891 );
10892 assert_eq!(module.status().unwrap().pid, before);
10893 assert_eq!(
10894 module.configuration().unwrap().1.http,
10895 configured.health.http
10896 );
10897 configured.args = vec!["61".into()];
10898 let launch = handler
10899 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
10900 .await
10901 .unwrap();
10902 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
10903 assert_eq!(
10904 module.status().unwrap().pid,
10905 before,
10906 "a launch edit is stored until reload"
10907 );
10908 module.drain().await.unwrap();
10909 }
10910 use cortexkit_test_support::ScratchDir;
10911 use std::{
10912 collections::BTreeMap,
10913 fmt,
10914 path::PathBuf,
10915 sync::{Arc, Mutex},
10916 time::Duration,
10917 };
10918
10919 use serde_json::{json, Value};
10920 use subc_protocol::{
10921 manifest::{
10922 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
10923 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
10924 },
10925 session::HealthStatus,
10926 FrameType,
10927 };
10928
10929 use super::*;
10930 use crate::{
10931 forwarding::{DataRoute, DataRouteState},
10932 registry::ChannelState,
10933 router::FrameSink,
10934 stderr_tail::DEFAULT_MAX_LINE_BYTES,
10935 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
10936 RouteCtx, Router,
10937 };
10938 use tokio::{
10939 sync::mpsc,
10940 time::{sleep, Instant},
10941 };
10942 use tracing::{
10943 field::{Field, Visit},
10944 Event, Subscriber,
10945 };
10946 use tracing_subscriber::{layer::Context, prelude::*, Layer};
10947
10948 fn fake_aft_stub_path() -> PathBuf {
10967 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
10968 assert!(
10969 path.exists(),
10970 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
10971 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
10972 path.display()
10973 );
10974 path
10975 }
10976
10977 fn client_retries(code: &str) -> bool {
10982 subc_protocol::error_codes::is_retryable_route_open(code)
10983 }
10984
10985 #[test]
10996 fn retryability_of_forwarding_codes_matches_the_failure() {
10997 let transient = [
11000 ForwardingError::NoModuleConnection,
11001 ForwardingError::ModuleReloading {
11002 module_id: "m".into(),
11003 },
11004 ForwardingError::StaleModuleEndpoint,
11005 ForwardingError::UnknownReservation {
11006 client_channel: 1,
11007 module_channel: 1,
11008 },
11009 ForwardingError::ConnectionClosing {
11010 connection_id: ConnectionId::new(1),
11011 },
11012 ForwardingError::ClientEgressClosed {
11013 connection_id: ConnectionId::new(1),
11014 },
11015 ForwardingError::ModuleEgressUnavailable {
11016 connection_id: ConnectionId::new(1),
11017 },
11018 ];
11019 for err in transient {
11020 let code = forwarding_error_code(&err);
11021 assert!(
11022 client_retries(code),
11023 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
11024 );
11025 }
11026
11027 let permanent = [
11032 ForwardingError::ConnectionRoleConflict {
11033 connection_id: ConnectionId::new(1),
11034 },
11035 ForwardingError::ClientRouteChannelExhausted {
11036 connection_id: ConnectionId::new(1),
11037 },
11038 ForwardingError::ModuleRouteChannelExhausted {
11039 endpoint: ModuleEndpointId {
11040 connection_id: ConnectionId::new(1),
11041 generation: 1,
11042 },
11043 },
11044 ForwardingError::RelayCorrelationExhausted,
11045 ForwardingError::RouteOpenBuild("x".into()),
11046 ForwardingError::Poisoned,
11047 ];
11048 for err in permanent {
11049 let code = forwarding_error_code(&err);
11050 assert!(
11051 !client_retries(code),
11052 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
11053 );
11054 }
11055 }
11056
11057 #[tokio::test]
11085 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
11086 let handler = ControlHandler::default();
11087 let frame =
11088 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
11089
11090 let stamped = handler
11094 .route_open_principal(ConnectionId::new(1), &frame, None)
11095 .unwrap()
11096 .unwrap();
11097 assert_eq!(
11098 stamped,
11099 Principal::Direct,
11100 "a caller that proved nothing must not be stamped as a supervised module"
11101 );
11102
11103 let forged = handler
11108 .route_open_principal(
11109 ConnectionId::new(1),
11110 &frame,
11111 Some(ConsumerIdentity {
11112 module_id: "aft".to_string(),
11113 launch_nonce: "not-a-real-nonce".to_string(),
11114 }),
11115 )
11116 .unwrap();
11117 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
11118 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
11119 }
11120
11121 #[test]
11138 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
11139 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
11140 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
11141 let ClientControlRequest::RouteOpen {
11142 consumer_identity, ..
11143 } = parsed
11144 else {
11145 panic!("route.open body must parse as RouteOpen");
11146 };
11147 assert_eq!(
11148 consumer_identity,
11149 Some(ConsumerIdentity {
11150 module_id: "aft".to_string(),
11151 launch_nonce: "n".to_string(),
11152 }),
11153 "the wire field name must reach the value route_open_principal reads"
11154 );
11155 }
11156
11157 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
11158 ModuleManifest::builder(module_id, "0.1.0")
11159 .protocol_ver(protocol_ver)
11160 .provides(vec![ProviderRole::ToolProvider {
11161 tools: vec![Tool {
11162 name: "read".to_string(),
11163 description: None,
11164 execution_mode: ExecutionMode::Pure,
11165 schema: json!({"type": "object"}),
11166 }],
11167 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
11168 concurrency: Concurrency::ModuleManaged,
11169 emits_push: true,
11170 sub_supervises: true,
11171 }])
11172 .build()
11173 }
11174
11175 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
11176 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
11177 }
11178
11179 fn hello_frame_with_control_ops(
11180 module_id: &str,
11181 protocol_ver: u8,
11182 corr: u64,
11183 control_ops: Option<Vec<String>>,
11184 ) -> Frame {
11185 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
11186 }
11187
11188 fn hello_frame_with_nonce(
11189 module_id: &str,
11190 protocol_ver: u8,
11191 corr: u64,
11192 launch_nonce: Option<&str>,
11193 ) -> Frame {
11194 hello_frame_full(
11195 module_id,
11196 protocol_ver,
11197 corr,
11198 None,
11199 launch_nonce.map(ToOwned::to_owned),
11200 )
11201 }
11202
11203 fn hello_frame_full(
11204 module_id: &str,
11205 protocol_ver: u8,
11206 corr: u64,
11207 control_ops: Option<Vec<String>>,
11208 launch_nonce: Option<String>,
11209 ) -> Frame {
11210 let body = serde_json::to_vec(&ModuleHelloBody {
11211 manifest: manifest(module_id, protocol_ver),
11212 protocol_ver,
11213 control_ops,
11214 launch_nonce,
11215 })
11216 .unwrap();
11217 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11218 }
11219
11220 fn hello_frame_with_rich_ui(
11221 module_id: &str,
11222 protocol_ver: u8,
11223 corr: u64,
11224 launch_nonce: Option<&str>,
11225 rich_ui: Value,
11226 ) -> Frame {
11227 let mut body = serde_json::to_value(ModuleHelloBody {
11228 manifest: manifest(module_id, protocol_ver),
11229 protocol_ver,
11230 control_ops: None,
11231 launch_nonce: launch_nonce.map(str::to_string),
11232 })
11233 .unwrap();
11234 body["manifest"]["rich_ui"] = rich_ui;
11235 Frame::build(
11236 FrameType::Hello,
11237 control_flags(),
11238 0,
11239 0,
11240 corr,
11241 serde_json::to_vec(&body).unwrap(),
11242 )
11243 .unwrap()
11244 }
11245
11246 fn non_routable_hello_frame_with_control_ops(
11247 module_id: &str,
11248 corr: u64,
11249 control_ops: Option<Vec<String>>,
11250 ) -> Frame {
11251 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11252 manifest.provides.clear();
11253 let body = serde_json::to_vec(&ModuleHelloBody {
11254 manifest,
11255 protocol_ver: PROTOCOL_VERSION,
11256 control_ops,
11257 launch_nonce: None,
11258 })
11259 .unwrap();
11260 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11261 }
11262
11263 fn capability_grammar_hello_frame(
11264 capabilities: Value,
11265 runtime_computed: Option<Value>,
11266 corr: u64,
11267 ) -> Frame {
11268 let mut body = serde_json::to_value(ModuleHelloBody {
11269 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
11270 protocol_ver: PROTOCOL_VERSION,
11271 control_ops: None,
11272 launch_nonce: None,
11273 })
11274 .expect("HELLO body serializes");
11275 body["manifest"]["capabilities"] = capabilities;
11276 if let Some(runtime_computed) = runtime_computed {
11277 body["runtime_computed"] = runtime_computed;
11278 }
11279 Frame::build(
11280 FrameType::Hello,
11281 control_flags(),
11282 0,
11283 0,
11284 corr,
11285 serde_json::to_vec(&body).expect("HELLO body reserializes"),
11286 )
11287 .expect("HELLO frame builds")
11288 }
11289
11290 fn channel_request(channel: u16, corr: u64) -> Frame {
11291 Frame::build(
11292 FrameType::Request,
11293 Flags::new(true, Priority::Interactive, false),
11294 channel,
11295 0,
11296 corr,
11297 b"opaque".to_vec(),
11298 )
11299 .unwrap()
11300 }
11301
11302 fn route_ctx(
11303 connection_id: ConnectionId,
11304 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11305 let (tx, rx) = mpsc::channel(8);
11306 (
11307 RouteCtx {
11308 connection_id,
11309 egress: FrameSink::new(tx),
11310 },
11311 rx,
11312 )
11313 }
11314
11315 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
11316 serde_json::from_slice(&frame.body).unwrap()
11317 }
11318
11319 async fn hello_via_sink(
11324 handler: &ControlHandler,
11325 ctx: &RouteCtx,
11326 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11327 hello: Frame,
11328 ) -> Frame {
11329 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
11330 assert!(
11331 replies.is_empty(),
11332 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
11333 );
11334 let ack = rx
11335 .try_recv()
11336 .expect("HELLO_ACK is queued on the module sink")
11337 .frame;
11338 assert_eq!(ack.header.ty, FrameType::HelloAck);
11339 ack
11340 }
11341
11342 fn parse_error(frame: &Frame) -> Value {
11343 serde_json::from_slice(&frame.body).unwrap()
11344 }
11345
11346 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
11347 serde_json::from_slice(&frame.body).unwrap()
11348 }
11349
11350 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
11351 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
11352 route_channel,
11353 route_epoch: 0,
11354 kind,
11355 })
11356 .unwrap();
11357 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11358 }
11359
11360 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
11361 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
11362 module_id: module_id.to_string(),
11363 })
11364 .unwrap();
11365 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11366 }
11367
11368 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
11369 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
11370 }
11371
11372 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
11374 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
11375 let mut frame = route_open_frame(corr, module_id, root);
11376 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
11377 body["identity"]["project_root"] = serde_json::json!(path.as_path());
11378 frame.body = serde_json::to_vec(&body).unwrap();
11379 frame
11380 }
11381
11382 fn route_open_frame_with_consumer_capabilities(
11383 corr: u64,
11384 module_id: &str,
11385 _project_root: ScratchDir,
11386 consumer_capabilities: Option<Vec<String>>,
11387 ) -> Frame {
11388 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11389 busy_refusals: None,
11390 target: RouteTarget::ToolProvider {
11391 module_id: module_id.to_string(),
11392 },
11393 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
11394 consumer_identity: None,
11395 consumer_capabilities,
11396 role_versions: None,
11397 admission_facts: None,
11398 scope: None,
11399 })
11400 .unwrap();
11401 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11402 }
11403
11404 fn route_open_frame_with_role_versions(
11405 corr: u64,
11406 module_id: &str,
11407 _project_root: ScratchDir,
11408 role_versions: Option<BTreeMap<String, String>>,
11409 ) -> Frame {
11410 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11411 busy_refusals: None,
11412 target: RouteTarget::ToolProvider {
11413 module_id: module_id.to_string(),
11414 },
11415 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11416 consumer_identity: None,
11417 consumer_capabilities: None,
11418 role_versions,
11419 admission_facts: None,
11420 scope: None,
11421 })
11422 .unwrap();
11423 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11424 }
11425
11426 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
11427 entries
11428 .iter()
11429 .map(|(role, version)| (role.to_string(), version.to_string()))
11430 .collect()
11431 }
11432
11433 fn route_open_frame_with_admission_facts(
11434 corr: u64,
11435 module_id: &str,
11436 _project_root: ScratchDir,
11437 consumer_identity: Option<subc_control::ConsumerIdentity>,
11438 facts: Option<Value>,
11439 ) -> Frame {
11440 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11441 busy_refusals: None,
11442 target: RouteTarget::ToolProvider {
11443 module_id: module_id.to_string(),
11444 },
11445 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11446 consumer_identity,
11447 consumer_capabilities: None,
11448 role_versions: None,
11449 admission_facts: facts,
11450 scope: None,
11451 })
11452 .unwrap();
11453 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11454 }
11455
11456 #[derive(Clone, Default)]
11457 struct EventCapture {
11458 events: Arc<Mutex<Vec<CapturedEvent>>>,
11459 }
11460
11461 #[derive(Clone, Debug)]
11462 struct CapturedEvent {
11463 target: String,
11464 level: tracing::Level,
11465 fields: BTreeMap<String, String>,
11466 }
11467
11468 impl EventCapture {
11469 fn events(&self) -> Vec<CapturedEvent> {
11470 self.events.lock().unwrap().clone()
11471 }
11472 }
11473
11474 impl<S> Layer<S> for EventCapture
11475 where
11476 S: Subscriber,
11477 {
11478 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
11479 let mut visitor = EventFieldVisitor::default();
11480 event.record(&mut visitor);
11481 self.events.lock().unwrap().push(CapturedEvent {
11482 target: event.metadata().target().to_string(),
11483 level: *event.metadata().level(),
11484 fields: visitor.fields,
11485 });
11486 }
11487 }
11488
11489 #[derive(Default)]
11490 struct EventFieldVisitor {
11491 fields: BTreeMap<String, String>,
11492 }
11493
11494 impl Visit for EventFieldVisitor {
11495 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
11496 self.fields
11497 .insert(field.name().to_string(), format!("{value:?}"));
11498 }
11499 }
11500
11501 fn health_response(corr: u64, status: HealthStatus) -> Frame {
11502 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
11503 status,
11504 detail: Some("warming".to_string()),
11505 metrics: Some(json!({"queue_depth": 3})),
11506 })
11507 .unwrap();
11508 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11509 }
11510
11511 fn route_bind_ack(corr: u64) -> Frame {
11512 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
11513 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11514 }
11515
11516 fn unique_project_root(label: &str) -> ScratchDir {
11517 ScratchDir::new(label)
11518 }
11519
11520 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
11521 match parse_route_poll(frame) {
11522 ClientControlResponse::RoutePoll {
11523 status: None,
11524 live: Some(live),
11525 ..
11526 } => assert_eq!(live, expected_live),
11527 other => panic!("unexpected route.poll response: {other:?}"),
11528 }
11529 }
11530
11531 fn bind_liveness_route(
11532 registry: &Registry,
11533 forwarding: &ForwardingTable,
11534 module_id: &str,
11535 ) -> (RouteCtx, u16, u32) {
11536 let module_connection = ConnectionId::new(101);
11537 let client_connection = ConnectionId::new(202);
11538 let registration = registry
11539 .register_with_control_ops(
11540 manifest(module_id, PROTOCOL_VERSION),
11541 PROTOCOL_VERSION,
11542 module_connection,
11543 module_baseline_control_ops(),
11544 )
11545 .unwrap();
11546 let (module_tx, _module_rx) = mpsc::channel(8);
11547 let endpoint = forwarding
11548 .register_module_connection(
11549 module_connection,
11550 module_id.to_string(),
11551 PROTOCOL_VERSION,
11552 manifest_concurrency(®istration.manifest),
11553 FrameSink::new(module_tx),
11554 )
11555 .unwrap();
11556 let (client_ctx, _client_rx) = route_ctx(client_connection);
11557 let pending = forwarding
11558 .begin_route_bind_relay_for_test(
11559 client_connection,
11560 client_ctx.egress.clone(),
11561 1,
11562 module_id,
11563 )
11564 .unwrap();
11565 assert_eq!(pending.endpoint, endpoint);
11566 let route_channel = pending.client_channel;
11567 let route_epoch = pending.client_epoch;
11568 forwarding
11569 .complete_pending_relay(
11570 module_connection,
11571 pending.corr,
11572 RouteBindRelayOutcome::Accepted,
11573 )
11574 .unwrap();
11575 (client_ctx, route_channel, route_epoch)
11576 }
11577
11578 struct FakeProcessLiveness {
11579 live: Option<bool>,
11580 }
11581
11582 impl ModuleProcessLiveness for FakeProcessLiveness {
11583 fn process_live(&self, _module_id: &str) -> Option<bool> {
11584 self.live
11585 }
11586 }
11587
11588 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11589 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
11590 {
11591 let registry = Arc::new(Registry::default());
11592 let supervisor_handle = SupervisorHandle::new();
11593 let supervisor = Supervisor::new_for_test(
11594 Arc::clone(®istry),
11595 RestartPolicy::new(1, Duration::from_millis(10)),
11596 )
11597 .with_handle(supervisor_handle.clone());
11598 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
11599 let module = supervisor
11600 .spawn(ModuleSpec {
11601 script: None,
11602 carry_containment: false,
11603 module_id: "stderr-tail-wire".to_string(),
11604 program: fake_aft_stub_path(),
11605 args: Vec::new(),
11606 env: vec![
11607 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
11608 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
11609 ],
11610 reserved: false,
11611 reserved_prefixes: Vec::new(),
11612 protocol: ModuleProtocol::Subc,
11613 overlap: Default::default(),
11614 })
11615 .unwrap();
11616
11617 let deadline = Instant::now() + Duration::from_secs(5);
11618 loop {
11619 let tail = module.stderr_tail(None, None);
11620 if tail
11621 .entries
11622 .iter()
11623 .any(|entry| matches!(entry, TailEntry::ProcessStart))
11624 && tail.entries.iter().any(|entry| {
11625 matches!(
11626 entry,
11627 TailEntry::Line {
11628 truncated: true,
11629 ..
11630 }
11631 )
11632 })
11633 {
11634 break;
11635 }
11636 assert!(
11637 Instant::now() < deadline,
11638 "module did not produce a truncated line and restart boundary: {tail:?}"
11639 );
11640 sleep(Duration::from_millis(10)).await;
11641 }
11642
11643 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11644 let request = ClientControlRequest::SupervisorStderrTail {
11645 module_id: "stderr-tail-wire".to_string(),
11646 max_lines: None,
11647 max_bytes: None,
11648 };
11649 let frame = Frame::build(
11650 FrameType::Request,
11651 control_flags(),
11652 0,
11653 0,
11654 1,
11655 serde_json::to_vec(&request).unwrap(),
11656 )
11657 .unwrap();
11658 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11659 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11660 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
11661 serde_json::from_slice(&responses[0].body).unwrap()
11662 else {
11663 panic!("expected supervisor.stderr_tail response");
11664 };
11665
11666 assert!(
11667 tail.entries
11668 .iter()
11669 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
11670 "the control response lost the restart boundary"
11671 );
11672 let Some(StderrTailEntry::Line {
11673 text,
11674 truncated,
11675 at_ms,
11676 }) = tail.entries.iter().find(|entry| {
11677 matches!(
11678 entry,
11679 StderrTailEntry::Line {
11680 truncated: true,
11681 ..
11682 }
11683 )
11684 })
11685 else {
11686 panic!("the control response lost the truncated line");
11687 };
11688 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
11689 assert!(*truncated);
11690 assert!(
11691 at_ms.is_some(),
11692 "the control response lost the line's capture time"
11693 );
11694 }
11695
11696 #[tokio::test(flavor = "current_thread")]
11701 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
11702 let dir = ScratchDir::new("terminals-off-worker");
11703 let journal_path = dir.join("terminals.jsonl");
11704 let registry = Arc::new(Registry::default());
11705 let supervisor_handle = SupervisorHandle::new();
11706 let supervisor =
11707 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11708 .with_handle(supervisor_handle.clone())
11709 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
11710 let module = supervisor
11711 .spawn(ModuleSpec {
11712 script: None,
11713 carry_containment: false,
11714 module_id: "terminal-off-worker".to_string(),
11715 program: fake_aft_stub_path(),
11716 args: Vec::new(),
11717 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11718 reserved: false,
11719 reserved_prefixes: Vec::new(),
11720 protocol: ModuleProtocol::Subc,
11721 overlap: Default::default(),
11722 })
11723 .unwrap();
11724 let deadline = Instant::now() + Duration::from_secs(5);
11725 while module.terminal_history().entries.len() != 2 {
11726 assert!(Instant::now() < deadline, "module did not record two exits");
11727 sleep(Duration::from_millis(10)).await;
11728 }
11729
11730 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
11731 let handler =
11732 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
11733 let frame = Frame::build(
11734 FrameType::Request,
11735 control_flags(),
11736 0,
11737 0,
11738 1,
11739 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
11740 module_id: "terminal-off-worker".to_string(),
11741 })
11742 .unwrap(),
11743 )
11744 .unwrap();
11745 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11746 let spawned_at = std::time::Instant::now();
11747 let read = tokio::spawn({
11748 let handler = Arc::clone(&handler);
11749 async move { handler.handle_control_frame(&ctx, frame).await }
11750 });
11751 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
11754 .await
11755 .unwrap()
11756 .expect("the history read reached its pause");
11757 let elapsed = spawned_at.elapsed();
11758 assert!(
11759 elapsed < Duration::from_secs(2) && !read.is_finished(),
11760 "this task could not run while the history read was paused \
11761 (resumed after {elapsed:?}, read finished: {})",
11762 read.is_finished()
11763 );
11764
11765 drop(release);
11766 let responses = read.await.unwrap().unwrap();
11767 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11768 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
11769 panic!("expected supervisor.terminals response");
11770 };
11771 assert_eq!(terminals.entries.len(), 2);
11772 assert_eq!(terminals.journal_skipped_lines, 0);
11773 assert_eq!(terminals.journal_read_errors, 0);
11774 }
11775
11776 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11777 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
11778 let registry = Arc::new(Registry::default());
11779 let supervisor_handle = SupervisorHandle::new();
11780 let supervisor =
11781 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
11782 .with_handle(supervisor_handle.clone());
11783 let module = supervisor
11784 .spawn(ModuleSpec {
11785 script: None,
11786 carry_containment: false,
11787 module_id: "terminal-golden".to_string(),
11788 program: fake_aft_stub_path(),
11789 args: Vec::new(),
11790 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11791 reserved: false,
11792 reserved_prefixes: Vec::new(),
11793 protocol: ModuleProtocol::Subc,
11794 overlap: Default::default(),
11795 })
11796 .unwrap();
11797
11798 let deadline = Instant::now() + Duration::from_secs(5);
11799 while module.terminal_history().entries.len() != 2 {
11800 assert!(
11801 Instant::now() < deadline,
11802 "module did not retain two terminal exits: {:?}",
11803 module.terminal_history()
11804 );
11805 sleep(Duration::from_millis(10)).await;
11806 }
11807
11808 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11809 let request = ClientControlRequest::SupervisorTerminals {
11810 module_id: "terminal-golden".to_string(),
11811 };
11812 let frame = Frame::build(
11813 FrameType::Request,
11814 control_flags(),
11815 0,
11816 0,
11817 1,
11818 serde_json::to_vec(&request).unwrap(),
11819 )
11820 .unwrap();
11821 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11822 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11823 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11824 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
11825 panic!("expected supervisor.terminals response");
11826 };
11827 assert_eq!(terminals.entries.len(), 2);
11828 assert_eq!(terminals.dropped, 0);
11829
11830 let mut rendered = serde_json::to_value(response).unwrap();
11831 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
11834 for (index, entry) in rendered["entries"]
11835 .as_array_mut()
11836 .expect("terminal response entries array")
11837 .iter_mut()
11838 .enumerate()
11839 {
11840 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
11841 }
11842
11843 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11844 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
11845 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
11846 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11847 std::fs::write(&golden_path, &serialized).unwrap();
11848 }
11849 let expected: Value =
11850 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
11851 assert_eq!(rendered, expected);
11852 }
11853
11854 #[test]
11855 fn hello_registers_manifest_and_returns_ack() {
11856 let registry = Arc::new(Registry::default());
11857 let handler = ControlHandler::new(Arc::clone(®istry));
11858 let conn = ConnectionId::new(1);
11859
11860 let responses = handler
11861 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
11862 .unwrap();
11863
11864 assert_eq!(responses.len(), 1);
11865 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11866 assert_eq!(responses[0].header.channel, 0);
11867 assert_eq!(responses[0].header.corr, 7);
11868 let ack = parse_ack(&responses[0]);
11869 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11870 assert!(ack
11871 .subc_capabilities
11872 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
11873 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
11874 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
11875 assert!(ack
11876 .subc_ops
11877 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
11878 assert!(ack
11879 .subc_ops
11880 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
11881 assert!(ack
11882 .subc_ops
11883 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
11884
11885 let registration = registry.get_module("aft").unwrap().unwrap();
11886 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
11887 assert_eq!(registration.state, ChannelState::Active);
11888 assert_eq!(registration.connection_id, conn);
11889 assert_eq!(registration.control_ops, module_baseline_control_ops());
11890 assert!(registration.manifest.display_name.is_none());
11891 }
11892
11893 #[test]
11894 fn hello_display_name_refuses_invalid_manifest_by_field() {
11895 let registry = Arc::new(Registry::default());
11896 let handler = ControlHandler::new(Arc::clone(®istry));
11897 for (index, invalid) in [
11898 json!(""),
11899 json!("a".repeat(41)),
11900 json!(" Routing"),
11901 json!("a\nb"),
11902 json!("a\u{202e}b"),
11903 json!(7),
11904 ]
11905 .into_iter()
11906 .enumerate()
11907 {
11908 let module_id = format!("bad-display-name-{index}");
11909 let mut body = serde_json::to_value(ModuleHelloBody {
11910 manifest: manifest(&module_id, PROTOCOL_VERSION),
11911 protocol_ver: PROTOCOL_VERSION,
11912 control_ops: None,
11913 launch_nonce: None,
11914 })
11915 .unwrap();
11916 body["manifest"]["display_name"] = invalid;
11917 let hello = Frame::build(
11918 FrameType::Hello,
11919 control_flags(),
11920 0,
11921 0,
11922 7,
11923 serde_json::to_vec(&body).unwrap(),
11924 )
11925 .unwrap();
11926 let response = handler
11927 .handle_control(ConnectionId::new(100 + index as u64), hello)
11928 .unwrap();
11929 assert_eq!(response[0].header.ty, FrameType::Error);
11930 let error = parse_error(&response[0]);
11931 assert_eq!(error["code"], "invalid_manifest");
11932 assert!(error["message"].as_str().unwrap().contains("display_name"));
11933 assert!(registry.get_module(&module_id).unwrap().is_none());
11934 }
11935 }
11936
11937 #[test]
11938 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
11939 let invalid_identifiers = [
11940 ("case_change", "credentials-Provider/v1"),
11941 ("leading_zero", "credentials-provider/v01"),
11942 ("trailing_hyphen", "credentials-provider-/v1"),
11943 ("consecutive_hyphens", "credentials--provider/v1"),
11944 ("uppercase", "Credentials-provider/v1"),
11945 ("missing_v", "credentials-provider/1"),
11946 ("whitespace", "credentials provider/v1"),
11947 ("zero_version", "credentials-provider/v0"),
11948 ("out_of_range_version", "credentials-provider/v4294967296"),
11949 (
11950 "overlength_name",
11951 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
11952 ),
11953 ];
11954 let mut cases = invalid_identifiers
11955 .into_iter()
11956 .map(|(name, identifier)| {
11957 (
11958 format!("identifier_{name}"),
11959 "capabilities.provides[0]".to_string(),
11960 identifier.to_string(),
11961 json!({ "provides": [identifier] }),
11962 None,
11963 )
11964 })
11965 .collect::<Vec<_>>();
11966 cases.extend([
11967 (
11968 "unknown_need".to_string(),
11969 "capabilities.requires[0].need".to_string(),
11970 "deferred".to_string(),
11971 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
11972 None,
11973 ),
11974 (
11975 "duplicate_provides".to_string(),
11976 "capabilities.provides[1]".to_string(),
11977 "credentials-provider/v1".to_string(),
11978 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
11979 None,
11980 ),
11981 (
11982 "duplicate_must_never_reach".to_string(),
11983 "capabilities.must_never_reach[1]".to_string(),
11984 "credentials-provider/v1".to_string(),
11985 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
11986 None,
11987 ),
11988 (
11989 "duplicate_requires_same_need".to_string(),
11990 "capabilities.requires[1]".to_string(),
11991 "credentials-provider/v1".to_string(),
11992 json!({ "requires": [
11993 { "capability": "credentials-provider/v1", "need": "required" },
11994 { "capability": "credentials-provider/v1", "need": "required" }
11995 ] }),
11996 None,
11997 ),
11998 (
11999 "duplicate_requires_conflicting_need".to_string(),
12000 "capabilities.requires[1]".to_string(),
12001 "credentials-provider/v1".to_string(),
12002 json!({ "requires": [
12003 { "capability": "credentials-provider/v1", "need": "required" },
12004 { "capability": "credentials-provider/v1", "need": "optional" }
12005 ] }),
12006 None,
12007 ),
12008 (
12009 "capabilities_root_pointer".to_string(),
12010 "runtime_computed[0]".to_string(),
12011 "/capabilities".to_string(),
12012 json!({}),
12013 Some(json!(["/capabilities"])),
12014 ),
12015 (
12016 "capabilities_descendant_pointer".to_string(),
12017 "runtime_computed[0]".to_string(),
12018 "/capabilities/provides".to_string(),
12019 json!({}),
12020 Some(json!(["/capabilities/provides"])),
12021 ),
12022 (
12023 "malformed_pointer_without_leading_slash".to_string(),
12024 "runtime_computed[0]".to_string(),
12025 "capabilities".to_string(),
12026 json!({}),
12027 Some(json!(["capabilities"])),
12028 ),
12029 (
12030 "malformed_pointer_escape".to_string(),
12031 "runtime_computed[0]".to_string(),
12032 "/roles/~2/tools".to_string(),
12033 json!({}),
12034 Some(json!(["/roles/~2/tools"])),
12035 ),
12036 (
12037 "unknown_capabilities_field".to_string(),
12038 "capabilities.future".to_string(),
12039 "<array>".to_string(),
12040 json!({ "future": [] }),
12041 None,
12042 ),
12043 ]);
12044
12045 for (index, (name, field, value, capabilities, runtime_computed)) in
12046 cases.into_iter().enumerate()
12047 {
12048 let registry = Arc::new(Registry::default());
12049 let handler = ControlHandler::new(Arc::clone(®istry));
12050 let response = handler
12051 .handle_control(
12052 ConnectionId::new((index + 1) as u64),
12053 capability_grammar_hello_frame(
12054 capabilities,
12055 runtime_computed,
12056 index as u64 + 1,
12057 ),
12058 )
12059 .expect("invalid HELLO returns a refusal");
12060
12061 assert_eq!(response.len(), 1, "{name} must emit one refusal");
12062 let error = parse_error(&response[0]);
12063 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
12064 let message = error["message"]
12065 .as_str()
12066 .expect("error message is a string");
12067 assert!(
12068 message.contains(&field),
12069 "{name}: field missing from {message}"
12070 );
12071 assert!(
12072 message.contains(&value),
12073 "{name}: value missing from {message}"
12074 );
12075 assert_eq!(
12076 registry
12077 .active_registration_count()
12078 .expect("registry reads"),
12079 0,
12080 "{name}: refused HELLO must not create a catalog entry"
12081 );
12082 }
12083 }
12084
12085 #[test]
12086 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
12087 let registry = Arc::new(Registry::default());
12088 let handler = ControlHandler::new(Arc::clone(®istry));
12089 let capabilities = json!({
12090 "provides": ["credentials-provider/v1"],
12091 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
12092 "must_never_reach": ["federation-transport/v1"]
12093 });
12094 let response = handler
12095 .handle_control(
12096 ConnectionId::new(99),
12097 capability_grammar_hello_frame(
12098 capabilities.clone(),
12099 Some(json!(["/roles/0/tools"])),
12100 99,
12101 ),
12102 )
12103 .expect("valid HELLO registers");
12104 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12105
12106 let request = Frame::build(
12107 FrameType::Request,
12108 control_flags(),
12109 0,
12110 0,
12111 100,
12112 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12113 .expect("catalog request serializes"),
12114 )
12115 .expect("catalog request frame builds");
12116 let response = handler
12117 .handle_catalog_list(request, None)
12118 .expect("catalog list succeeds");
12119 let ClientControlResponse::CatalogList { modules, .. } =
12120 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
12121 else {
12122 panic!("catalog request must return catalog.list");
12123 };
12124 assert_eq!(modules.len(), 1);
12125 assert_eq!(
12126 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
12127 capabilities
12128 );
12129 }
12130
12131 #[test]
12132 fn catalog_list_mirrors_management_operation_description() {
12133 let registry = Arc::new(Registry::default());
12134 let handler = ControlHandler::new(Arc::clone(®istry));
12135 let description = "List managed records and return their identifiers and metadata.";
12136 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
12137 manifest.provides = vec![ProviderRole::ManagementSurface {
12138 operations: vec![ManagementOperation {
12139 name: "records.list".to_string(),
12140 kind: ManagementOperationKind::Query,
12141 description: Some(description.to_string()),
12142 }],
12143 config_schema: json!({"type": "object"}),
12144 observability: vec![ObservabilitySurface {
12145 name: "records.stats".to_string(),
12146 kind: ObservabilityKind::Snapshot,
12147 }],
12148 identity_scope: vec![IdentityScope::Project],
12149 concurrency: Concurrency::ModuleManaged,
12150 }];
12151 registry
12152 .register_with_control_ops(
12153 manifest,
12154 PROTOCOL_VERSION,
12155 ConnectionId::new(99),
12156 Vec::new(),
12157 )
12158 .expect("described management manifest registers");
12159
12160 let request = Frame::build(
12161 FrameType::Request,
12162 control_flags(),
12163 0,
12164 0,
12165 100,
12166 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12167 .expect("catalog request serializes"),
12168 )
12169 .expect("catalog request frame builds");
12170 let response = handler
12171 .handle_catalog_list(request, None)
12172 .expect("catalog list succeeds");
12173 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
12174 assert_eq!(
12175 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
12176 "catalog.list must preserve the declared operation description verbatim"
12177 );
12178 }
12179
12180 #[test]
12181 fn catalog_list_mirrors_registered_operation_delivery() {
12182 let registry = Arc::new(Registry::default());
12183 let handler = ControlHandler::new(Arc::clone(®istry));
12184 let delivery =
12185 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
12186 let mut body = serde_json::to_value(ModuleHelloBody {
12187 manifest: manifest("watch-management", PROTOCOL_VERSION),
12188 protocol_ver: PROTOCOL_VERSION,
12189 control_ops: None,
12190 launch_nonce: None,
12191 })
12192 .unwrap();
12193 body["manifest"]["provides"] = json!([{
12194 "role": "management_surface",
12195 "operations": [
12196 {"name": "status", "kind": "query"},
12197 {"name": "future", "kind": "query"},
12198 {"name": "reset", "kind": "mutate"}
12199 ],
12200 "config_schema": {}, "observability": [], "identity_scope": []
12201 }]);
12202 body["manifest"]["operation_delivery"] = delivery.clone();
12203 let hello = Frame::build(
12204 FrameType::Hello,
12205 control_flags(),
12206 0,
12207 0,
12208 99,
12209 serde_json::to_vec(&body).unwrap(),
12210 )
12211 .unwrap();
12212 let response = handler
12213 .handle_control(ConnectionId::new(99), hello)
12214 .unwrap();
12215 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12216
12217 let (_, registrations) = registry.list_modules().unwrap();
12218 let registered = ®istrations[0].manifest;
12219 assert!(registered.is_watch_operation("status"));
12220 assert!(!registered.is_watch_operation("future"));
12221 assert!(!registered.is_watch_operation("reset"));
12222 assert!(!registered.is_watch_operation("missing"));
12223
12224 let request = Frame::build(
12225 FrameType::Request,
12226 control_flags(),
12227 0,
12228 0,
12229 100,
12230 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12231 )
12232 .unwrap();
12233 let response = handler.handle_catalog_list(request, None).unwrap();
12234 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
12235 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
12236 let ClientControlResponse::CatalogList { modules, .. } =
12237 serde_json::from_slice(&response[0].body).unwrap()
12238 else {
12239 panic!("expected catalog.list");
12240 };
12241 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
12242 }
12243
12244 #[test]
12245 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
12246 let registry = Arc::new(Registry::default());
12247 let supervisor = SupervisorHandle::new();
12248 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12249 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12250 let descriptor = json!({
12251 "schema": "ckios.richui.manifest/1",
12252 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12253 "bytes": 4096,
12254 "chunks": 4
12255 });
12256 let connection = ConnectionId::new(201);
12257 let hello = handler
12258 .handle_control(
12259 connection,
12260 hello_frame_with_rich_ui(
12261 "vault",
12262 PROTOCOL_VERSION,
12263 201,
12264 Some("the-real-nonce"),
12265 descriptor.clone(),
12266 ),
12267 )
12268 .unwrap();
12269 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12270
12271 let request =
12272 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
12273 let provides = registry
12274 .get_module_by_connection(connection)
12275 .unwrap()
12276 .unwrap()
12277 .manifest
12278 .provides;
12279 handler
12280 .handle_catalog_update(connection, request, provides, None, Some(false))
12281 .unwrap();
12282
12283 let request = Frame::build(
12284 FrameType::Request,
12285 control_flags(),
12286 0,
12287 0,
12288 203,
12289 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12290 )
12291 .unwrap();
12292 let response = handler.handle_catalog_list(request, None).unwrap();
12293 let ClientControlResponse::CatalogList { modules, .. } =
12294 serde_json::from_slice(&response[0].body).unwrap()
12295 else {
12296 panic!("expected catalog.list");
12297 };
12298 assert_eq!(
12299 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
12300 descriptor
12301 );
12302 assert!(modules[0].nonce_verified);
12303 }
12304
12305 fn hello_with_role_declarations(declarations: Value) -> Frame {
12306 let mut body = serde_json::to_value(ModuleHelloBody {
12307 manifest: manifest("role-test", PROTOCOL_VERSION),
12308 protocol_ver: PROTOCOL_VERSION,
12309 control_ops: None,
12310 launch_nonce: None,
12311 })
12312 .unwrap();
12313 body["manifest"]["role_declarations"] = declarations;
12314 Frame::build(
12315 FrameType::Hello,
12316 control_flags(),
12317 0,
12318 0,
12319 301,
12320 serde_json::to_vec(&body).unwrap(),
12321 )
12322 .unwrap()
12323 }
12324
12325 fn update_role_declarations_frame(declarations: Value) -> Frame {
12326 Frame::build(
12327 FrameType::Request,
12328 control_flags(),
12329 0,
12330 0,
12331 303,
12332 serde_json::to_vec(&json!({
12333 "op": "catalog.update_role_declarations",
12334 "role_declarations": declarations,
12335 }))
12336 .unwrap(),
12337 )
12338 .unwrap()
12339 }
12340
12341 #[tokio::test]
12342 async fn role_declarations_update_replaces_then_clears_catalog() {
12343 let registry = Arc::new(Registry::default());
12344 let handler = ControlHandler::new(Arc::clone(®istry));
12345 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12346 let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
12347 hello_via_sink(
12348 &handler,
12349 &ctx,
12350 &mut rx,
12351 hello_with_role_declarations(initial.clone()),
12352 )
12353 .await;
12354 assert_eq!(
12355 role_declarations_catalog(&handler).0["role_declarations"],
12356 initial
12357 );
12358 let before = registry.get_module("role-test").unwrap().unwrap();
12359 for declarations in [
12360 json!({"media-provider/v1": {"engines": ["new"]}}),
12361 json!({}),
12362 ] {
12363 let response = handler
12364 .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
12365 .await
12366 .unwrap();
12367 assert_eq!(response[0].header.ty, FrameType::Response);
12368 assert_eq!(
12369 serde_json::from_slice::<Value>(&response[0].body).unwrap(),
12370 json!({"op": "catalog.update_role_declarations"})
12371 );
12372 let (raw, entry) = role_declarations_catalog(&handler);
12373 assert_eq!(
12374 serde_json::to_value(entry.role_declarations).unwrap(),
12375 declarations
12376 );
12377 if declarations.as_object().unwrap().is_empty() {
12378 assert!(raw.get("role_declarations").is_none());
12379 } else {
12380 assert_eq!(raw["role_declarations"], declarations);
12381 }
12382 let after = registry.get_module("role-test").unwrap().unwrap();
12383 let mut expected = before.manifest.clone();
12384 expected.role_declarations = serde_json::from_value(declarations).unwrap();
12385 assert_eq!(after.manifest, expected);
12386 assert_eq!(after.ready, before.ready);
12387 }
12388 }
12389
12390 #[tokio::test]
12391 async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
12392 let registry = Arc::new(Registry::default());
12393 let handler = ControlHandler::new(registry);
12394 let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12395 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12396 hello_via_sink(
12397 &handler,
12398 &ctx,
12399 &mut rx,
12400 hello_with_role_declarations(initial.clone()),
12401 )
12402 .await;
12403 for (invalid, reason) in [
12404 (json!(null), "role_declarations must be an object"),
12405 (json!([]), "role_declarations must be an object"),
12406 (
12407 json!({"bad key": {}}),
12408 "role_declarations key must use capability-name grammar",
12409 ),
12410 (
12411 json!({"media-provider/v1": []}),
12412 "role_declarations values must be objects",
12413 ),
12414 (
12415 json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
12416 "role_declarations exceeds 16384 compact JSON bytes",
12417 ),
12418 ] {
12419 let response = handler
12420 .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
12421 .await
12422 .unwrap();
12423 let error = parse_error(&response[0]);
12424 assert_eq!(error["code"], "invalid_role_declarations");
12425 assert_eq!(error["message"], reason);
12426 assert_eq!(
12427 role_declarations_catalog(&handler).0["role_declarations"],
12428 initial
12429 );
12430 }
12431 }
12432
12433 #[tokio::test]
12434 async fn role_declarations_update_is_owned_by_the_registered_connection() {
12435 let registry = Arc::new(Registry::default());
12436 let handler = ControlHandler::new(Arc::clone(®istry));
12437 let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
12438 let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12439 hello_via_sink(
12440 &handler,
12441 &owner,
12442 &mut owner_rx,
12443 hello_with_role_declarations(initial.clone()),
12444 )
12445 .await;
12446 let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
12447 let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
12448 let response = handler
12449 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12450 .await
12451 .unwrap();
12452 assert_eq!(parse_error(&response[0])["code"], "not_registered");
12453 hello_via_sink(
12454 &handler,
12455 &other,
12456 &mut other_rx,
12457 hello_frame("other", PROTOCOL_VERSION, 304),
12458 )
12459 .await;
12460 let response = handler
12461 .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12462 .await
12463 .unwrap();
12464 assert_eq!(response[0].header.ty, FrameType::Response);
12465 assert_eq!(
12466 serde_json::to_value(
12467 registry
12468 .get_module("role-test")
12469 .unwrap()
12470 .unwrap()
12471 .manifest
12472 .role_declarations
12473 )
12474 .unwrap(),
12475 initial
12476 );
12477 assert_eq!(
12478 serde_json::to_value(
12479 registry
12480 .get_module("other")
12481 .unwrap()
12482 .unwrap()
12483 .manifest
12484 .role_declarations
12485 )
12486 .unwrap(),
12487 replacement
12488 );
12489 }
12490
12491 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
12492 let request = Frame::build(
12493 FrameType::Request,
12494 control_flags(),
12495 0,
12496 0,
12497 302,
12498 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12499 )
12500 .unwrap();
12501 let response = handler.handle_catalog_list(request, None).unwrap();
12502 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
12503 let ClientControlResponse::CatalogList { mut modules, .. } =
12504 serde_json::from_slice(&response[0].body).unwrap()
12505 else {
12506 panic!("expected catalog.list");
12507 };
12508 (raw["modules"][0].clone(), modules.remove(0))
12509 }
12510
12511 #[test]
12512 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
12513 let registry = Arc::new(Registry::default());
12514 let handler = ControlHandler::new(Arc::clone(®istry));
12515 let declarations = json!({
12516 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
12517 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
12518 });
12519 let connection = ConnectionId::new(301);
12520 let response = handler
12521 .handle_control(
12522 connection,
12523 hello_with_role_declarations(declarations.clone()),
12524 )
12525 .unwrap();
12526 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12527 let registered = registry
12528 .get_module_by_connection(connection)
12529 .unwrap()
12530 .unwrap()
12531 .manifest;
12532 assert_eq!(
12536 serde_json::to_value(®istered.role_declarations).unwrap(),
12537 declarations
12538 );
12539 let (raw, entry) = role_declarations_catalog(&handler);
12540 assert_eq!(raw["role_declarations"], declarations);
12541 assert_eq!(entry.role_declarations, registered.role_declarations);
12542
12543 let request =
12544 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
12545 handler
12546 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
12547 .unwrap();
12548 let (raw, entry) = role_declarations_catalog(&handler);
12549 assert_eq!(raw["role_declarations"], declarations);
12550 assert_eq!(entry.role_declarations, registered.role_declarations);
12551 }
12552
12553 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
12554 let registry = Arc::new(Registry::default());
12555 let handler = ControlHandler::new(Arc::clone(®istry));
12556 let capture = EventCapture::default();
12557 let _subscriber =
12558 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12559 let response = handler
12560 .handle_control(
12561 ConnectionId::new(301),
12562 hello_with_role_declarations(declarations),
12563 )
12564 .unwrap();
12565 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12566 assert!(registry
12567 .get_module_by_connection(ConnectionId::new(301))
12568 .unwrap()
12569 .unwrap()
12570 .manifest
12571 .role_declarations
12572 .is_empty());
12573 let (raw, entry) = role_declarations_catalog(&handler);
12574 assert!(raw.get("role_declarations").is_none());
12575 assert!(entry.role_declarations.is_empty());
12576 let warnings: Vec<_> = capture
12577 .events()
12578 .into_iter()
12579 .filter(|event| {
12580 event.fields.get("message").map(String::as_str)
12581 == Some("dropping whole role_declarations map from HELLO")
12582 })
12583 .collect();
12584 assert_eq!(warnings.len(), 1);
12585 assert_eq!(warnings[0].level, tracing::Level::WARN);
12586 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
12587 assert_eq!(warnings[0].fields["reason"], reason);
12588 }
12589
12590 #[test]
12591 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
12592 for value in [
12593 json!(null),
12594 json!(false),
12595 json!(0),
12596 json!("opaque"),
12597 json!([]),
12598 ] {
12599 assert_hello_drops_role_declarations_with_warning(
12600 json!({"routing/v1": {}, "broken/v1": value}),
12601 "role_declarations values must be objects",
12602 );
12603 }
12604 assert_hello_drops_role_declarations_with_warning(
12605 json!([]),
12606 "role_declarations must be an object",
12607 );
12608 }
12609
12610 #[test]
12611 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
12612 assert_hello_drops_role_declarations_with_warning(
12613 json!({"routing/v1": {}, "bad key": {}}),
12614 "role_declarations key must use capability-name grammar",
12615 );
12616 }
12617
12618 #[test]
12619 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
12620 assert_hello_drops_role_declarations_with_warning(
12621 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
12622 "role_declarations exceeds 16384 compact JSON bytes",
12623 );
12624 }
12625
12626 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
12627 let mut body = serde_json::to_value(ModuleHelloBody {
12628 manifest: manifest("flow-ops", PROTOCOL_VERSION),
12629 protocol_ver: PROTOCOL_VERSION,
12630 control_ops: None,
12631 launch_nonce: None,
12632 })
12633 .unwrap();
12634 body["manifest"]["provides"] = json!([{
12635 "role": "management_surface",
12636 "operations": [
12637 {"name": "diagnostics.findings", "kind": "query"},
12638 {"name": "reset", "kind": "mutate"}
12639 ],
12640 "config_schema": {}, "observability": [], "identity_scope": []
12641 }]);
12642 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
12643 if let Some(flow_operations) = flow_operations {
12644 body["manifest"]["flow_operations"] = flow_operations;
12645 }
12646 Frame::build(
12647 FrameType::Hello,
12648 control_flags(),
12649 0,
12650 0,
12651 401,
12652 serde_json::to_vec(&body).unwrap(),
12653 )
12654 .unwrap()
12655 }
12656
12657 #[test]
12658 fn catalog_list_mirrors_flow_operations_unchanged() {
12659 for (capability, flow_operations) in [
12662 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
12663 ("call-modes/v1", json!(["diagnostics.findings"])),
12664 ("flow-scopes/v1", json!([])),
12665 ] {
12666 let registry = Arc::new(Registry::default());
12667 let handler = ControlHandler::new(Arc::clone(®istry));
12668 let connection = ConnectionId::new(401);
12669 let response = handler
12670 .handle_control(
12671 connection,
12672 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
12673 )
12674 .unwrap();
12675 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12676 let registered = registry
12677 .get_module_by_connection(connection)
12678 .unwrap()
12679 .unwrap()
12680 .manifest;
12681 assert_eq!(
12682 serde_json::to_value(®istered.flow_operations).unwrap(),
12683 flow_operations
12684 );
12685 let (raw, entry) = role_declarations_catalog(&handler);
12686 assert_eq!(raw["flow_operations"], flow_operations);
12687 assert_eq!(entry.flow_operations, registered.flow_operations);
12688 }
12689
12690 let registry = Arc::new(Registry::default());
12691 let handler = ControlHandler::new(Arc::clone(®istry));
12692 let response = handler
12693 .handle_control(
12694 ConnectionId::new(401),
12695 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
12696 )
12697 .unwrap();
12698 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12699 let (raw, entry) = role_declarations_catalog(&handler);
12700 assert!(raw.get("flow_operations").is_none());
12701 assert_eq!(entry.flow_operations, None);
12702 }
12703
12704 #[test]
12705 fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
12706 let oversized: serde_json::Map<String, Value> = (0..257)
12707 .map(|n| (format!("tool:t{n}"), json!(true)))
12708 .collect();
12709 for (capabilities, map, valid) in [
12710 (
12711 json!(["scope-federated/v1"]),
12712 json!({"tool:read":true,"management:reset":true}),
12713 true,
12714 ),
12715 (json!(["scope-federated/v1"]), json!({}), true),
12716 (json!([]), json!({"tool:read":true}), false),
12717 (
12718 json!(["scope-federated/v1"]),
12719 json!({"tool:valid":true,"tool:bad":false}),
12720 false,
12721 ),
12722 (
12723 json!(["scope-federated/v1"]),
12724 json!({"tool:valid":true,"management:missing":true}),
12725 false,
12726 ),
12727 (
12728 json!(["scope-federated/v1"]),
12729 Value::Object(oversized),
12730 false,
12731 ),
12732 ] {
12733 let capture = EventCapture::default();
12734 let _subscriber = tracing::subscriber::set_default(
12735 tracing_subscriber::registry().with(capture.clone()),
12736 );
12737 let registry = Arc::new(Registry::default());
12738 let handler = ControlHandler::new(Arc::clone(®istry));
12739 let hello = hello_with_flow_operations(capabilities, None);
12740 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12741 body["manifest"]["operation_deferral"] = map.clone();
12742 let hello = Frame::build(
12743 FrameType::Hello,
12744 control_flags(),
12745 0,
12746 0,
12747 401,
12748 serde_json::to_vec(&body).unwrap(),
12749 )
12750 .unwrap();
12751 let response = handler
12752 .handle_control(ConnectionId::new(401), hello)
12753 .unwrap();
12754 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12755 let (raw, entry) = role_declarations_catalog(&handler);
12756 if valid {
12757 assert_eq!(raw["operation_deferral"], map);
12758 assert_eq!(
12759 serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
12760 map
12761 );
12762 } else {
12763 assert!(entry.operation_deferral.is_none());
12764 assert!(raw.get("operation_deferral").is_none());
12765 assert_eq!(
12766 capture
12767 .events()
12768 .iter()
12769 .filter(|event| event.fields.get("message").map(String::as_str)
12770 == Some("dropping operation_deferral from HELLO"))
12771 .count(),
12772 1
12773 );
12774 }
12775 }
12776 }
12777
12778 fn assert_hello_drops_flow_operations_with_warning(
12779 capabilities: Value,
12780 flow_operations: Value,
12781 reason: &str,
12782 ) {
12783 let registry = Arc::new(Registry::default());
12784 let handler = ControlHandler::new(Arc::clone(®istry));
12785 let capture = EventCapture::default();
12786 let _subscriber =
12787 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12788 let response = handler
12789 .handle_control(
12790 ConnectionId::new(401),
12791 hello_with_flow_operations(capabilities, Some(flow_operations)),
12792 )
12793 .unwrap();
12794 assert_eq!(response[0].header.ty, FrameType::HelloAck);
12795 let registered = registry
12796 .get_module_by_connection(ConnectionId::new(401))
12797 .unwrap()
12798 .expect("the module still registers")
12799 .manifest;
12800 assert_eq!(registered.module_id, "flow-ops");
12801 assert_eq!(registered.flow_operations, None);
12802 let (raw, entry) = role_declarations_catalog(&handler);
12803 assert_eq!(raw["module_id"], "flow-ops");
12804 assert!(raw.get("flow_operations").is_none());
12805 assert_eq!(entry.flow_operations, None);
12806 let warnings: Vec<_> = capture
12807 .events()
12808 .into_iter()
12809 .filter(|event| {
12810 event.fields.get("message").map(String::as_str)
12811 == Some("dropping flow_operations from HELLO")
12812 })
12813 .collect();
12814 assert_eq!(warnings.len(), 1);
12815 assert_eq!(warnings[0].level, tracing::Level::WARN);
12816 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
12817 assert_eq!(warnings[0].fields["reason"], reason);
12818 }
12819
12820 #[test]
12821 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
12822 assert_hello_drops_flow_operations_with_warning(
12823 json!(["flow-scopes/v1"]),
12824 json!(["diagnostics.findings", "missing"]),
12825 "flow_operations names 'missing', which is not a declared management operation",
12826 );
12827 }
12828
12829 #[test]
12830 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
12831 assert_hello_drops_flow_operations_with_warning(
12832 json!([]),
12833 json!(["diagnostics.findings"]),
12834 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12835 );
12836 assert_hello_drops_flow_operations_with_warning(
12838 json!(["agent-run-scopes/v1"]),
12839 json!([]),
12840 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12841 );
12842 }
12843
12844 #[test]
12845 fn hello_malformed_flow_operations_is_dropped_with_warning() {
12846 assert_hello_drops_flow_operations_with_warning(
12847 json!(["flow-scopes/v1"]),
12848 json!(["reset", "reset"]),
12849 "flow_operations names 'reset' more than once",
12850 );
12851 assert_hello_drops_flow_operations_with_warning(
12852 json!(["flow-scopes/v1"]),
12853 json!(vec!["reset"; 257]),
12854 "flow_operations lists 257 names, more than 256",
12855 );
12856 assert_hello_drops_flow_operations_with_warning(
12857 json!(["flow-scopes/v1"]),
12858 json!("reset"),
12859 "flow_operations must be an array of operation names",
12860 );
12861 }
12862
12863 #[test]
12864 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
12865 let registry = Arc::new(Registry::default());
12866 let handler = ControlHandler::new(registry.clone());
12867 let descriptor = json!({
12868 "schema": "ckios.richui.manifest/1",
12869 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12870 "bytes": 128,
12871 "chunks": 2
12872 });
12873 let malformed = [
12874 ("schema", json!("")),
12875 ("schema", json!("a".repeat(65))),
12876 ("schema", json!("Bad schema")),
12877 (
12878 "manifest_digest",
12879 json!(format!("sha256:{}", "A".repeat(64))),
12880 ),
12881 ("manifest_digest", json!("sha256:abc")),
12882 ("bytes", json!(0)),
12883 ("bytes", json!(16_777_217)),
12884 ("bytes", json!("128")),
12885 ("chunks", json!(0)),
12886 ("chunks", json!(1025)),
12887 ("chunks", json!("2")),
12888 ];
12889 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
12890 let mut invalid = descriptor.clone();
12891 invalid[field] = invalid_value;
12892 let response = handler
12893 .handle_control(
12894 ConnectionId::new(210 + index as u64),
12895 hello_frame_with_rich_ui(
12896 &format!("bad-rich-ui-{index}"),
12897 PROTOCOL_VERSION,
12898 210 + index as u64,
12899 None,
12900 invalid,
12901 ),
12902 )
12903 .unwrap();
12904 assert_eq!(response[0].header.ty, FrameType::Error);
12905 let error = parse_error(&response[0]);
12906 assert_eq!(error["code"], "invalid_manifest");
12907 assert!(error["message"]
12908 .as_str()
12909 .unwrap()
12910 .contains(&format!("rich_ui.{field}")));
12911 assert!(registry
12912 .get_module(&format!("bad-rich-ui-{index}"))
12913 .unwrap()
12914 .is_none());
12915 }
12916 }
12917
12918 #[test]
12919 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
12920 let registry = Arc::new(Registry::default());
12921 let handler = ControlHandler::new(Arc::clone(®istry));
12922 let connection = ConnectionId::new(219);
12923 registry
12924 .register_with_control_ops(
12925 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
12926 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
12927 "bad schema",
12928 format!("sha256:{}", "a".repeat(64)),
12929 128,
12930 2,
12931 )))
12932 .build(),
12933 PROTOCOL_VERSION,
12934 connection,
12935 Vec::new(),
12936 )
12937 .unwrap();
12938 let request =
12939 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
12940 let response = handler
12941 .handle_catalog_update(connection, request, Vec::new(), None, None)
12942 .unwrap();
12943 assert_eq!(response[0].header.ty, FrameType::Error);
12944 let error = parse_error(&response[0]);
12945 assert_eq!(error["code"], "invalid_manifest");
12946 assert!(error["message"]
12947 .as_str()
12948 .unwrap()
12949 .contains("rich_ui.schema"));
12950 }
12951
12952 #[test]
12953 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
12954 let registry = Arc::new(Registry::default());
12955 let supervisor = SupervisorHandle::new();
12956 supervisor.open_swap("provider", "candidate-nonce".to_string());
12957 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12958 let connection = ConnectionId::new(218);
12959 let hello = handler
12960 .handle_control(
12961 connection,
12962 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
12963 )
12964 .unwrap();
12965 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12966 assert!(registry.list_modules().unwrap().1.is_empty());
12967
12968 registry.promote_candidate("provider").unwrap().unwrap();
12969 let request = Frame::build(
12970 FrameType::Request,
12971 control_flags(),
12972 0,
12973 0,
12974 219,
12975 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12976 )
12977 .unwrap();
12978 let response = handler.handle_catalog_list(request, None).unwrap();
12979 let ClientControlResponse::CatalogList { modules, .. } =
12980 serde_json::from_slice(&response[0].body).unwrap()
12981 else {
12982 panic!("expected catalog.list");
12983 };
12984 assert_eq!(modules.len(), 1);
12985 assert!(modules[0].nonce_verified);
12986 }
12987
12988 #[test]
12989 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
12990 for provenance_json in [
12991 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
12992 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
12993 ] {
12994 let registry = Arc::new(Registry::default());
12995 let handler = ControlHandler::new(registry);
12996 let hello = Frame::build(
12997 FrameType::Hello,
12998 control_flags(),
12999 0,
13000 0,
13001 220,
13002 serde_json::to_vec(&ModuleHelloBody {
13003 manifest: ModuleManifest::builder("built", "0.1.0")
13004 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
13005 .build(),
13006 protocol_ver: PROTOCOL_VERSION,
13007 control_ops: None,
13008 launch_nonce: None,
13009 })
13010 .unwrap(),
13011 )
13012 .unwrap();
13013 assert_eq!(
13014 handler
13015 .handle_control(ConnectionId::new(220), hello)
13016 .unwrap()[0]
13017 .header
13018 .ty,
13019 FrameType::HelloAck
13020 );
13021 let request = Frame::build(
13022 FrameType::Request,
13023 control_flags(),
13024 0,
13025 0,
13026 221,
13027 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13028 )
13029 .unwrap();
13030 let response = handler.handle_catalog_list(request, None).unwrap();
13031 let ClientControlResponse::CatalogList { modules, .. } =
13032 serde_json::from_slice(&response[0].body).unwrap()
13033 else {
13034 panic!("expected catalog.list");
13035 };
13036 assert_eq!(modules.len(), 1);
13037 assert_eq!(
13038 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13039 format!(
13040 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
13041 )
13042 .as_bytes()
13043 );
13044 }
13045 }
13046
13047 #[test]
13048 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
13049 let registry = Arc::new(Registry::default());
13050 let handler = ControlHandler::new(Arc::clone(®istry));
13051 let hello = Frame::build(
13052 FrameType::Hello,
13053 control_flags(),
13054 0,
13055 0,
13056 220,
13057 serde_json::to_vec(&ModuleHelloBody {
13058 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
13059 protocol_ver: PROTOCOL_VERSION,
13060 control_ops: None,
13061 launch_nonce: Some("unverified".to_string()),
13062 })
13063 .unwrap(),
13064 )
13065 .unwrap();
13066 assert_eq!(
13067 handler
13068 .handle_control(ConnectionId::new(220), hello)
13069 .unwrap()[0]
13070 .header
13071 .ty,
13072 FrameType::HelloAck
13073 );
13074 let request = Frame::build(
13075 FrameType::Request,
13076 control_flags(),
13077 0,
13078 0,
13079 221,
13080 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13081 )
13082 .unwrap();
13083 let response = handler.handle_catalog_list(request, None).unwrap();
13084 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
13085 assert!(wire["modules"][0].get("provenance").is_none());
13086 let ClientControlResponse::CatalogList { modules, .. } =
13087 serde_json::from_slice(&response[0].body).unwrap()
13088 else {
13089 panic!("expected catalog.list");
13090 };
13091 assert_eq!(
13092 serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13093 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
13094 );
13095 assert!(!modules[0].nonce_verified);
13096 assert!(modules[0].provenance.is_none());
13097 }
13098
13099 fn catalog_legacy_projection(mut entry: CatalogEntry) -> CatalogEntry {
13101 assert_eq!(entry.first_party, Some(false));
13102 entry.incarnation = None;
13103 entry.spawn = None;
13104 entry.first_party = None;
13105 entry.origin = None;
13106 entry.requested_declarations_digest = None;
13107 entry.approved_declarations_digest = None;
13108 entry
13109 }
13110
13111 #[test]
13112 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
13113 let registry = Arc::new(Registry::default());
13114 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13115 [("vault".to_string(), true), ("squatter".to_string(), true)],
13116 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13117 );
13118 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
13119 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13120 provides: vec!["credentials-provider/v1".to_string()],
13121 requires: Vec::new(),
13122 must_never_reach: Vec::new(),
13123 });
13124 let frame = Frame::build(
13125 FrameType::Hello,
13126 control_flags(),
13127 0,
13128 0,
13129 77,
13130 serde_json::to_vec(&ModuleHelloBody {
13131 manifest: squatter,
13132 protocol_ver: PROTOCOL_VERSION,
13133 control_ops: None,
13134 launch_nonce: None,
13135 })
13136 .expect("HELLO serializes"),
13137 )
13138 .expect("HELLO frame builds");
13139 let response = handler
13140 .handle_control(ConnectionId::new(77), frame)
13141 .expect("reserved claim receives a typed refusal");
13142 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13143 assert_eq!(
13144 registry
13145 .active_registration_count()
13146 .expect("registry reads"),
13147 0,
13148 "a reserved capability refusal must not leave a catalog entry"
13149 );
13150 }
13151
13152 #[test]
13153 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
13154 for settlement in ["timeout", "inconclusive", "drop"] {
13155 let breakers = RouteBindBreakers::default();
13156 let RouteBindAdmission::Admitted {
13157 guard: mut old,
13158 probe: false,
13159 } = breakers.admit("prov")
13160 else {
13161 panic!("ordinary relay admitted")
13162 };
13163 let RouteBindAdmission::Admitted {
13164 guard: mut opener, ..
13165 } = breakers.admit("prov")
13166 else {
13167 panic!("second relay admitted")
13168 };
13169 assert!(
13170 !opener
13171 .record_timeout(1, Duration::ZERO)
13172 .unwrap()
13173 .reopened_after_probe
13174 );
13175 let RouteBindAdmission::Admitted {
13176 guard: mut probe,
13177 probe: true,
13178 } = breakers.admit("prov")
13179 else {
13180 panic!("one cooldown probe admitted")
13181 };
13182 match settlement {
13183 "timeout" => assert!(
13184 !old.record_timeout(1, Duration::ZERO)
13185 .unwrap()
13186 .reopened_after_probe
13187 ),
13188 "inconclusive" => old.record_inconclusive(),
13189 "drop" => drop(old),
13190 _ => unreachable!(),
13191 }
13192 assert!(
13193 matches!(
13194 breakers.admit("prov"),
13195 RouteBindAdmission::Refused {
13196 probe_in_flight: true,
13197 ..
13198 }
13199 ),
13200 "{settlement} of a pre-open relay cannot release the real probe"
13201 );
13202 assert!(
13203 probe
13204 .record_timeout(1, Duration::ZERO)
13205 .unwrap()
13206 .reopened_after_probe
13207 );
13208 assert!(matches!(
13209 breakers.admit("prov"),
13210 RouteBindAdmission::Admitted { probe: true, .. }
13211 ));
13212 }
13213 let breakers = RouteBindBreakers::default();
13214 let admit = || match breakers.admit("prov") {
13215 RouteBindAdmission::Admitted { guard, .. } => guard,
13216 _ => panic!("relay admitted"),
13217 };
13218 admit().record_timeout(1, Duration::ZERO);
13219 let mut old_probe = admit();
13220 breakers.reset_for_new_module_connection("prov");
13221 admit().record_timeout(1, Duration::ZERO);
13222 let _new_probe = admit();
13223 old_probe.record_inconclusive();
13224 assert!(matches!(
13225 breakers.admit("prov"),
13226 RouteBindAdmission::Refused {
13227 probe_in_flight: true,
13228 ..
13229 }
13230 ));
13231 }
13232
13233 #[tokio::test]
13234 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
13235 for candidate in [false, true] {
13236 let registry = Arc::new(Registry::default());
13237 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13238 [("vault".to_string(), true), ("squatter".to_string(), true)],
13239 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13240 );
13241 let conn = ConnectionId::new(77);
13242 let (ctx, mut rx) = route_ctx(conn);
13243 let initial = capability_manifest("squatter", &[], &[]);
13244 if candidate {
13245 registry
13246 .register_candidate_with_control_ops(
13247 initial.clone(),
13248 PROTOCOL_VERSION,
13249 conn,
13250 module_baseline_control_ops(),
13251 )
13252 .unwrap();
13253 handler
13254 .forwarding
13255 .register_candidate_module_connection(
13256 conn,
13257 "squatter".to_string(),
13258 PROTOCOL_VERSION,
13259 manifest_concurrency(&initial),
13260 ctx.egress.clone(),
13261 )
13262 .unwrap();
13263 } else {
13264 hello_via_sink(
13265 &handler,
13266 &ctx,
13267 &mut rx,
13268 hello_frame_with_manifest(initial.clone(), 1),
13269 )
13270 .await;
13271 }
13272 let response = handler
13273 .handle_control_frame(
13274 &ctx,
13275 catalog_update_with_capabilities_frame(
13276 2,
13277 capability_manifest("squatter", &["credentials-provider/v1"], &[])
13278 .capabilities
13279 .unwrap(),
13280 ),
13281 )
13282 .await
13283 .unwrap();
13284 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13285 assert_eq!(
13286 registry
13287 .get_module_by_connection(conn)
13288 .unwrap()
13289 .unwrap()
13290 .manifest,
13291 initial
13292 );
13293 }
13294 }
13295
13296 #[test]
13297 #[cfg(not(feature = "dev-desktop-approval"))]
13298 fn server_describe_default_build_advertises_no_config_capability() {
13299 let assert_default = |handler: &ControlHandler| {
13300 let request = Frame::build(
13301 FrameType::Request,
13302 control_flags(),
13303 0,
13304 0,
13305 79,
13306 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13307 )
13308 .unwrap();
13309 let responses = handler.handle_server_describe(request).unwrap();
13310 assert_eq!(responses.len(), 1);
13311 let response: Value = serde_json::from_slice(&responses[0].body).unwrap();
13312 assert_eq!(response["op"], "server.describe");
13313 let capabilities = response["capabilities"]
13314 .as_array()
13315 .expect("capability array");
13316 assert!(
13317 !capabilities.is_empty(),
13318 "exercise an actual describe response, not an empty fixture"
13319 );
13320 assert!(
13321 capabilities
13322 .iter()
13323 .all(|capability| !capability.as_str().unwrap().starts_with("config.")),
13324 "a default build must not announce configuration authority: {capabilities:?}"
13325 );
13326 };
13327 assert_default(&ControlHandler::new(Arc::new(Registry::default())));
13328 #[cfg(unix)]
13329 {
13330 let (_rig, _owner, _supervisor, handler, _spec) =
13334 crate::config_gate::runtime::RuntimeOwner::unactivated_supervisor_fixture_for_test(
13335 );
13336 assert_default(&handler);
13337 }
13338 }
13339
13340 #[test]
13341 fn server_describe_surfaces_required_capability_verdict_fields() {
13342 let registry = Arc::new(Registry::default());
13343 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
13344 [
13345 ("consumer".to_string(), true),
13346 ("provider".to_string(), false),
13347 ],
13348 BTreeMap::new(),
13349 );
13350 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13351 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13352 provides: Vec::new(),
13353 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13354 capability: "credentials-provider/v1".to_string(),
13355 need: subc_protocol::manifest::CapabilityNeed::Required,
13356 }],
13357 must_never_reach: Vec::new(),
13358 });
13359 let hello = Frame::build(
13360 FrameType::Hello,
13361 control_flags(),
13362 0,
13363 0,
13364 78,
13365 serde_json::to_vec(&ModuleHelloBody {
13366 manifest: consumer,
13367 protocol_ver: PROTOCOL_VERSION,
13368 control_ops: None,
13369 launch_nonce: None,
13370 })
13371 .expect("HELLO serializes"),
13372 )
13373 .expect("HELLO frame builds");
13374 handler
13375 .handle_control(ConnectionId::new(78), hello)
13376 .expect("consumer registers");
13377 let describe = Frame::build(
13378 FrameType::Request,
13379 control_flags(),
13380 0,
13381 0,
13382 79,
13383 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
13384 .expect("request serializes"),
13385 )
13386 .expect("describe frame builds");
13387 let response = handler
13388 .handle_server_describe(describe)
13389 .expect("server.describe succeeds");
13390 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
13391 let requirement = &rendered["capability_requirements"][0];
13392 assert_eq!(requirement["consumer"], "consumer");
13393 assert_eq!(requirement["verdict"], "never_provided");
13394 assert_eq!(requirement["episode_seq"], 1);
13395 assert_eq!(requirement["config_satisfiable"], false);
13396 assert_eq!(requirement["runtime_available"], false);
13397 assert!(requirement["detail"]
13398 .as_str()
13399 .expect("detail string")
13400 .contains("credentials-provider/v1"));
13401 }
13402
13403 fn deregister_required_provider(
13409 daemon_shutdown_begun: bool,
13410 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
13411 const CAPABILITY: &str = "project-identity/v1";
13412 let registry = Arc::new(Registry::default());
13413 let daemon_shutdown = DaemonShutdownFlag::default();
13414 let handler = ControlHandler::new(Arc::clone(®istry))
13415 .with_capability_config(
13416 [
13417 ("consumer".to_string(), true),
13418 ("provider".to_string(), true),
13419 ],
13420 BTreeMap::new(),
13421 )
13422 .with_daemon_shutdown_flag(daemon_shutdown.clone());
13423 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13424 consumer.capabilities = Some(CapabilityDeclarations {
13425 provides: Vec::new(),
13426 requires: vec![subc_protocol::manifest::CapabilityRequirement {
13427 capability: CAPABILITY.to_string(),
13428 need: subc_protocol::manifest::CapabilityNeed::Required,
13429 }],
13430 must_never_reach: Vec::new(),
13431 });
13432 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
13433 let provider_conn = ConnectionId::new(182);
13434 for (conn, manifest, corr) in [
13435 (ConnectionId::new(181), consumer, 181),
13436 (provider_conn, provider, 182),
13437 ] {
13438 let hello = handler
13439 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
13440 .expect("module registers");
13441 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13442 }
13443 assert_eq!(
13444 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13445 Some(CapabilityVerdict::Provided),
13446 "the registered provider must satisfy the requirement first"
13447 );
13448 if daemon_shutdown_begun {
13449 daemon_shutdown.set_for_test();
13450 }
13451
13452 let capture = EventCapture::default();
13453 let _subscriber =
13454 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13455 handler
13456 .cleanup_connection(provider_conn)
13457 .expect("provider deregisters");
13458 let events = capture
13459 .events()
13460 .into_iter()
13461 .filter(|event| {
13462 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
13463 })
13464 .collect();
13465 (
13466 events,
13467 handler.capability_evaluator.verdict("consumer", CAPABILITY),
13468 )
13469 }
13470
13471 #[test]
13472 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
13473 let (events, verdict) = deregister_required_provider(true);
13474 assert_eq!(
13475 verdict,
13476 Some(CapabilityVerdict::NeverProvided),
13477 "shutdown must not change the verdict route.open refusals read"
13478 );
13479 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13480 let event = &events[0];
13481 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
13482 assert_eq!(
13483 event.fields.get("cause").map(String::as_str),
13484 Some("\"daemon_shutdown\""),
13485 "{event:?}"
13486 );
13487 assert_eq!(
13488 event.fields.get("verdict").map(String::as_str),
13489 Some("\"never_provided\""),
13490 "{event:?}"
13491 );
13492 }
13493
13494 #[test]
13495 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
13496 let (events, verdict) = deregister_required_provider(false);
13497 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
13498 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13499 let event = &events[0];
13500 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
13501 assert_eq!(event.fields.get("cause"), None, "{event:?}");
13502 assert_eq!(
13503 event.fields.get("verdict").map(String::as_str),
13504 Some("\"never_provided\""),
13505 "{event:?}"
13506 );
13507 }
13508
13509 #[test]
13510 fn catalog_list_omits_capabilities_for_legacy_manifest() {
13511 let registry = Arc::new(Registry::default());
13512 let handler = ControlHandler::new(Arc::clone(®istry));
13513 let hello = handler
13514 .handle_control(
13515 ConnectionId::new(101),
13516 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
13517 )
13518 .expect("legacy HELLO registers");
13519 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13520
13521 let request = Frame::build(
13522 FrameType::Request,
13523 control_flags(),
13524 0,
13525 0,
13526 102,
13527 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
13528 .expect("catalog request serializes"),
13529 )
13530 .expect("catalog request frame builds");
13531 let response = handler
13532 .handle_catalog_list(request, None)
13533 .expect("catalog list succeeds");
13534 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
13535 assert!(
13536 body["modules"][0].get("capabilities").is_none(),
13537 "legacy manifest must retain an absent capabilities field on catalog.list"
13538 );
13539 }
13540
13541 #[test]
13542 fn hello_ack_omits_storage_when_no_storage_config() {
13543 let registry = Arc::new(Registry::default());
13544 let handler = ControlHandler::new(Arc::clone(®istry));
13545 let responses = handler
13546 .handle_control(
13547 ConnectionId::new(1),
13548 hello_frame("aft", PROTOCOL_VERSION, 7),
13549 )
13550 .unwrap();
13551 let ack = parse_ack(&responses[0]);
13552 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
13553 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
13554 }
13555
13556 #[tokio::test]
13557 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
13558 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
13559 let registry = Arc::new(Registry::default());
13560 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
13561 let responses = handler
13562 .handle_control(
13563 ConnectionId::new(1),
13564 hello_frame("aft", PROTOCOL_VERSION, 7),
13565 )
13566 .unwrap();
13567 let ack = parse_ack(&responses[0]);
13568 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
13569
13570 let described = handler
13571 .handle_control_frame(
13572 &route_ctx(ConnectionId::new(2)).0,
13573 Frame::build(
13574 FrameType::Request,
13575 control_flags(),
13576 0,
13577 0,
13578 9,
13579 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13580 )
13581 .unwrap(),
13582 )
13583 .await
13584 .unwrap();
13585 let ClientControlResponse::ServerDescribe { machine_id, .. } =
13586 serde_json::from_slice(&described[0].body).unwrap()
13587 else {
13588 panic!("server.describe answered with another shape");
13589 };
13590 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
13591 }
13592
13593 #[test]
13594 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
13595 let registry = Arc::new(Registry::default());
13598 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
13599 crate::daemon_config::StorageConfig::Sqlite {
13600 data_home: std::path::PathBuf::from("/data"),
13601 },
13602 ));
13603
13604 let responses = handler
13605 .handle_control(
13606 ConnectionId::new(1),
13607 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
13608 )
13609 .unwrap();
13610 let ack = parse_ack(&responses[0]);
13611 assert_eq!(
13612 ack.storage,
13613 Some(serde_json::json!({
13614 "module_id": "alfonso-routing",
13615 "storage_namespace": "default",
13616 "isolation": { "kind": "module" },
13617 "backend": {
13618 "backend": "sqlite",
13619 "path": "/data/cortexkit/alfonso-routing/store.db"
13620 }
13621 })),
13622 "the delivered descriptor is the module's own sqlite store path"
13623 );
13624 }
13625
13626 #[test]
13627 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
13628 let registry = Arc::new(Registry::default());
13629 let handler = ControlHandler::new(Arc::clone(®istry));
13630 let conn = ConnectionId::new(1);
13631 let responses = handler
13632 .handle_control(
13633 conn,
13634 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13635 )
13636 .unwrap();
13637 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
13638 let registration = registry.get_module("aft").unwrap().unwrap();
13639 assert_eq!(registration.control_ops, module_baseline_control_ops());
13640
13641 let frame =
13642 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
13643 assert!(handler
13644 .guard_module_control_op(&frame, "aft", "route.bind")
13645 .unwrap()
13646 .is_none());
13647 let error = handler
13648 .guard_module_control_op(&frame, "aft", "test.synthetic")
13649 .unwrap()
13650 .expect("synthetic ungranted op should be rejected");
13651 assert_eq!(error.header.ty, FrameType::Error);
13652 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
13653 }
13654
13655 #[test]
13656 fn hello_control_ops_some_adds_optional_grants() {
13657 let registry = Arc::new(Registry::default());
13658 let handler = ControlHandler::new(Arc::clone(®istry));
13659 handler
13660 .handle_control(
13661 ConnectionId::new(1),
13662 hello_frame_with_control_ops(
13663 "aft",
13664 PROTOCOL_VERSION,
13665 7,
13666 Some(vec![
13667 "future.synthetic".to_string(),
13668 "route.bind".to_string(),
13669 ]),
13670 ),
13671 )
13672 .unwrap();
13673 let registration = registry.get_module("aft").unwrap().unwrap();
13674 assert_eq!(
13675 registration.control_ops,
13676 vec![
13677 "route.bind".to_string(),
13678 "route.status".to_string(),
13679 "future.synthetic".to_string(),
13680 ]
13681 );
13682 let frame =
13683 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
13684 assert!(handler
13685 .guard_module_control_op(&frame, "aft", "future.synthetic")
13686 .unwrap()
13687 .is_none());
13688 }
13689
13690 #[tokio::test]
13691 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
13692 let registry = Arc::new(Registry::default());
13693 let forwarding = Arc::new(ForwardingTable::default());
13694 let handler =
13695 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13696 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
13697 hello_via_sink(
13698 &handler,
13699 &module_ctx,
13700 &mut module_rx,
13701 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13702 )
13703 .await;
13704
13705 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
13706 let responses = handler
13707 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
13708 .await
13709 .unwrap();
13710 assert_eq!(responses.len(), 1);
13711 assert_eq!(responses[0].header.ty, FrameType::Error);
13712 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
13713 assert!(module_rx.try_recv().is_err());
13714 }
13715
13716 #[tokio::test]
13717 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
13718 let registry = Arc::new(Registry::default());
13719 let forwarding = Arc::new(ForwardingTable::default());
13720 let handler =
13721 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13722 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
13723 hello_via_sink(
13724 &handler,
13725 &module_ctx,
13726 &mut module_rx,
13727 hello_frame_with_control_ops(
13728 "aft",
13729 PROTOCOL_VERSION,
13730 7,
13731 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13732 ),
13733 )
13734 .await;
13735
13736 let project_root = unique_project_root("demux");
13737 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
13738 let route_handler = handler.clone();
13739 let route_task = tokio::spawn(async move {
13740 route_handler
13741 .handle_control_frame(
13742 &route_client_ctx,
13743 route_open_frame(100, "aft", project_root),
13744 )
13745 .await
13746 .unwrap()
13747 });
13748 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13749 .await
13750 .unwrap()
13751 .unwrap();
13752 assert!(matches!(
13753 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
13754 ModuleControlRequest::RouteBind { .. }
13755 ));
13756
13757 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
13758 let health_handler = handler.clone();
13759 let health_task = tokio::spawn(async move {
13760 health_handler
13761 .handle_control_frame(
13762 &health_client_ctx,
13763 supervisor_health_probe_frame(101, "aft"),
13764 )
13765 .await
13766 .unwrap()
13767 });
13768 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13769 .await
13770 .unwrap()
13771 .unwrap();
13772 assert_eq!(
13773 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13774 ModuleControlRequest::HealthCheck {}
13775 );
13776
13777 handler
13778 .handle_control_frame(
13779 &module_ctx,
13780 health_response(health_frame.header.corr, HealthStatus::Degraded),
13781 )
13782 .await
13783 .unwrap();
13784 let health_response = health_task.await.unwrap();
13785 assert_eq!(health_response.len(), 1);
13786 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13787 ClientControlResponse::SupervisorHealthProbe {
13788 module_id,
13789 status,
13790 detail,
13791 metrics,
13792 } => {
13793 assert_eq!(module_id, "aft");
13794 assert_eq!(status, HealthStatus::Degraded);
13795 assert_eq!(detail.as_deref(), Some("warming"));
13796 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
13797 }
13798 other => panic!("unexpected health response: {other:?}"),
13799 }
13800
13801 handler
13802 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13803 .await
13804 .unwrap();
13805 let route_response = route_task.await.unwrap();
13806 assert!(route_response.is_empty());
13807 let published = route_client_rx.recv().await.unwrap();
13808 assert!(matches!(
13809 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13810 ClientControlResponse::RouteOpen { .. }
13811 ));
13812 }
13813
13814 async fn relay_route_open(
13819 handler: &ControlHandler,
13820 client_connection: ConnectionId,
13821 client_egress: &FrameSink,
13822 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13823 corr: u64,
13824 module_id: &str,
13825 project_root_label: &str,
13826 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
13827 let ctx = RouteCtx {
13828 connection_id: client_connection,
13829 egress: client_egress.clone(),
13830 };
13831 let handler = handler.clone();
13832 let project_root = unique_project_root(project_root_label);
13833 let module_id = module_id.to_string();
13834 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
13835 let task = tokio::spawn(async move {
13836 let _guard = tracing::dispatcher::set_default(&dispatch);
13837 handler
13838 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
13839 .await
13840 .unwrap()
13841 });
13842 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13843 .await
13844 .expect("module receives the relayed route.bind")
13845 .expect("module egress is open");
13846 (task, bind.frame)
13847 }
13848
13849 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
13850 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
13851 ModuleControlRequest::RouteBind {
13852 route_channel,
13853 epoch,
13854 ..
13855 } => (route_channel, epoch),
13856 other => panic!("expected a route.bind request, got {other:?}"),
13857 }
13858 }
13859
13860 fn published_route(frame: &Frame) -> (u16, u32) {
13861 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
13862 ClientControlResponse::RouteOpen {
13863 route_channel,
13864 route_epoch,
13865 ..
13866 } => (route_channel, route_epoch),
13867 other => panic!("expected a route.open response, got {other:?}"),
13868 }
13869 }
13870
13871 #[tokio::test]
13891 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
13892 let registry = Arc::new(Registry::default());
13893 let forwarding = Arc::new(ForwardingTable::default());
13894 let handler =
13895 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13896
13897 let module_connection = ConnectionId::new(30);
13898 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13899 hello_via_sink(
13900 &handler,
13901 &module_ctx,
13902 &mut module_rx,
13903 hello_frame("aft", PROTOCOL_VERSION, 7),
13904 )
13905 .await;
13906
13907 let dying_client = ConnectionId::new(31);
13908 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
13909
13910 let (first_task, first_bind) = relay_route_open(
13913 &handler,
13914 dying_client,
13915 &dying_ctx.egress,
13916 &mut module_rx,
13917 100,
13918 "aft",
13919 "closing-first",
13920 )
13921 .await;
13922 handler
13923 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
13924 .await
13925 .unwrap();
13926 assert!(first_task.await.unwrap().is_empty());
13927 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
13928
13929 let (second_task, second_bind) = relay_route_open(
13931 &handler,
13932 dying_client,
13933 &dying_ctx.egress,
13934 &mut module_rx,
13935 101,
13936 "aft",
13937 "closing-second",
13938 )
13939 .await;
13940 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
13941
13942 assert!(forwarding
13945 .escalate_client_delivery_failure(
13946 dying_client,
13947 first_channel,
13948 first_epoch,
13949 CloseReason::new(
13950 "module_to_client_delivery_failed",
13951 "client egress refused a module frame",
13952 ),
13953 crate::forwarding::UndeliveredFrame {
13954 module_id: None,
13955 sink: &dying_ctx.egress,
13956 },
13957 )
13958 .unwrap());
13959 assert!(!dying_ctx.egress.is_closed());
13960
13961 let ack = handler
13963 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
13964 .await;
13965 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13966 if module_loop_error.is_some() {
13967 handler.cleanup_connection(module_connection).unwrap();
13971 }
13972 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13976 .await
13977 .ok()
13978 .flatten();
13979
13980 assert!(
13982 registry
13983 .get_module_by_connection(module_connection)
13984 .unwrap()
13985 .is_some(),
13986 "one client's closing connection ended the shared module connection: \
13987 {module_loop_error:?}"
13988 );
13989 let cotenant = ConnectionId::new(32);
13991 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13992 let (cotenant_task, cotenant_bind) = relay_route_open(
13993 &handler,
13994 cotenant,
13995 &cotenant_ctx.egress,
13996 &mut module_rx,
13997 102,
13998 "aft",
13999 "closing-cotenant",
14000 )
14001 .await;
14002 handler
14003 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
14004 .await
14005 .unwrap();
14006 assert!(cotenant_task.await.unwrap().is_empty());
14007 let (cotenant_channel, cotenant_epoch) =
14008 published_route(&cotenant_rx.recv().await.unwrap());
14009 assert!(matches!(
14010 forwarding
14011 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14012 .unwrap(),
14013 DataRoute::Client(DataRouteState::Bound(_))
14014 ));
14015
14016 let goodbye = post_ack_module_frame
14019 .expect("module receives a GOODBYE for the abandoned route channel");
14020 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14021 assert_eq!(goodbye.header.channel, abandoned_channel);
14022 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14023
14024 assert!(dying_rx.try_recv().is_err());
14028 let second_response = second_task.await.unwrap();
14029 assert_eq!(second_response.len(), 1);
14030 assert_eq!(
14031 parse_error(&second_response[0])["code"],
14032 "target_unavailable"
14033 );
14034 }
14035
14036 async fn catalog_snapshot(
14039 handler: &ControlHandler,
14040 client_ctx: &RouteCtx,
14041 corr: u64,
14042 ) -> (u64, Vec<(String, bool)>) {
14043 let body =
14044 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
14045 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
14046 let reply = handler
14047 .handle_control_frame(client_ctx, request)
14048 .await
14049 .unwrap()
14050 .pop()
14051 .unwrap();
14052 let ClientControlResponse::CatalogList {
14053 generation,
14054 modules,
14055 ..
14056 } = serde_json::from_slice(&reply.body).unwrap()
14057 else {
14058 panic!("not a catalog.list reply");
14059 };
14060 (
14061 generation,
14062 modules
14063 .into_iter()
14064 .map(|entry| (entry.module_id, entry.ready))
14065 .collect(),
14066 )
14067 }
14068
14069 fn hello_held_before_forwarding() -> (
14075 ControlHandler,
14076 Arc<std::sync::Barrier>,
14077 std::thread::JoinHandle<Vec<Frame>>,
14078 RouteCtx,
14079 mpsc::Receiver<crate::router::OutboundFrame>,
14080 ) {
14081 let hold = Arc::new(std::sync::Barrier::new(2));
14082 let handler = ControlHandler::with_forwarding(
14083 Arc::new(Registry::default()),
14084 Arc::new(ForwardingTable::default()),
14085 )
14086 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14087 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14088 let hello = {
14089 let handler = handler.clone();
14090 let module_ctx = module_ctx.clone();
14091 std::thread::spawn(move || {
14092 handler
14093 .handle_hello(
14094 module_ctx.connection_id,
14095 Some(module_ctx.egress.clone()),
14096 hello_frame("aft", PROTOCOL_VERSION, 7),
14097 )
14098 .unwrap()
14099 })
14100 };
14101 (handler, hold, hello, module_ctx, module_rx)
14102 }
14103
14104 fn lookup_during_hello_publication<T: Send + 'static>(
14110 registry: &Registry,
14111 hold: &std::sync::Barrier,
14112 lookup: impl FnOnce() -> T + Send + 'static,
14113 ) -> T {
14114 let attempted = registry.observe_next_lock_attempt();
14115 let reader = std::thread::spawn(lookup);
14116 if attempted.recv().unwrap() {
14117 let result = reader.join().unwrap();
14118 hold.wait();
14119 result
14120 } else {
14121 hold.wait();
14122 reader.join().unwrap()
14123 }
14124 }
14125
14126 #[test]
14127 fn catalog_read_after_hello_ack_waits_for_publication() {
14128 let registry = Arc::new(Registry::default());
14129 let hold = Arc::new(std::sync::Barrier::new(2));
14130 let mut handler = ControlHandler::with_forwarding(
14131 Arc::clone(®istry),
14132 Arc::new(ForwardingTable::default()),
14133 );
14134 handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
14135 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14136 let hello = {
14137 let handler = handler.clone();
14138 std::thread::spawn(move || {
14139 handler
14140 .handle_hello(
14141 module_ctx.connection_id,
14142 Some(module_ctx.egress),
14143 hello_frame("aft", PROTOCOL_VERSION, 7),
14144 )
14145 .unwrap()
14146 })
14147 };
14148 hold.wait();
14149 assert_eq!(
14150 module_rx.try_recv().unwrap().frame.header.ty,
14151 FrameType::HelloAck
14152 );
14153 let replies = lookup_during_hello_publication(®istry, &hold, move || {
14154 let request = Frame::build(
14157 FrameType::Request,
14158 control_flags(),
14159 0,
14160 0,
14161 300,
14162 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
14163 )
14164 .unwrap();
14165 handler.handle_catalog_list(request, None).unwrap()
14166 });
14167 assert!(hello.join().unwrap().is_empty());
14168 let ClientControlResponse::CatalogList {
14169 generation,
14170 modules,
14171 ..
14172 } = serde_json::from_slice(&replies[0].body).unwrap()
14173 else {
14174 panic!("not a catalog.list reply");
14175 };
14176 assert_eq!(
14177 modules
14178 .iter()
14179 .map(|entry| entry.module_id.as_str())
14180 .collect::<Vec<_>>(),
14181 vec!["aft"]
14182 );
14183 assert_eq!(generation, 1);
14184 }
14185
14186 #[tokio::test]
14195 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
14196 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
14197 let client = ConnectionId::new(41);
14198 let (client_ctx, mut client_rx) = route_ctx(client);
14199
14200 hold.wait();
14201 assert!(handler.registry.get_module("aft").unwrap().is_some());
14203 assert!(!handler
14204 .forwarding
14205 .has_live_module_connection("aft")
14206 .unwrap());
14207 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
14208 assert!(
14209 listed.is_empty(),
14210 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
14211 );
14212 let refused = handler
14213 .handle_control_frame(
14214 &client_ctx,
14215 route_open_frame(301, "aft", unique_project_root("listed-window")),
14216 )
14217 .await
14218 .unwrap();
14219 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
14220 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
14221 let duplicate = handler
14222 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
14223 .await
14224 .unwrap();
14225 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14226
14227 hold.wait();
14228 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14229 let ack = module_rx.try_recv().unwrap().frame;
14230 assert_eq!(ack.header.ty, FrameType::HelloAck);
14231 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14232 assert_eq!(listed, vec![("aft".to_string(), true)]);
14233
14234 let (open_task, bind) = relay_route_open(
14235 &handler,
14236 client,
14237 &client_ctx.egress,
14238 &mut module_rx,
14239 303,
14240 "aft",
14241 "listed-after",
14242 )
14243 .await;
14244 assert!(handler
14245 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14246 .await
14247 .unwrap()
14248 .is_empty());
14249 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
14250 published_route(&client_rx.try_recv().unwrap().frame);
14251 }
14252
14253 #[tokio::test]
14260 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
14261 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
14262 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14263 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14264
14265 hold.wait();
14266 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
14267 assert!(listed.is_empty());
14268 assert_eq!(
14269 during, before,
14270 "the generation moved while the module was not listed"
14271 );
14272
14273 hold.wait();
14274 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14275 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14276 assert_eq!(listed, vec![("aft".to_string(), true)]);
14277 assert_eq!(
14278 after,
14279 before + 1,
14280 "the module appeared without exactly one generation bump"
14281 );
14282 }
14283
14284 #[tokio::test]
14289 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
14290 let registry = Arc::new(Registry::default());
14291 let handler = ControlHandler::with_forwarding(
14292 Arc::clone(®istry),
14293 Arc::new(ForwardingTable::default()),
14294 );
14295 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14296 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14297 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14298 drop(module_rx);
14299
14300 let replies = handler
14301 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
14302 .await
14303 .unwrap();
14304 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
14305 assert!(registry.get_module("aft").unwrap().is_none());
14306 assert!(registry
14307 .get_module_by_connection(module_ctx.connection_id)
14308 .unwrap()
14309 .is_none());
14310 assert_eq!(
14311 catalog_snapshot(&handler, &client_ctx, 301).await,
14312 (before, Vec::new())
14313 );
14314 }
14315
14316 async fn route_open_held_after_bind_send() -> (
14320 ControlHandler,
14321 Arc<tokio::sync::Barrier>,
14322 RouteCtx,
14323 mpsc::Receiver<crate::router::OutboundFrame>,
14324 ) {
14325 let hold = Arc::new(tokio::sync::Barrier::new(2));
14326 let handler = ControlHandler::with_forwarding(
14327 Arc::new(Registry::default()),
14328 Arc::new(ForwardingTable::default()),
14329 )
14330 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
14331 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14332 hello_via_sink(
14333 &handler,
14334 &module_ctx,
14335 &mut module_rx,
14336 hello_frame("aft", PROTOCOL_VERSION, 7),
14337 )
14338 .await;
14339 (handler, hold, module_ctx, module_rx)
14340 }
14341
14342 #[tokio::test]
14349 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
14350 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14351 let client = ConnectionId::new(41);
14352 let (client_ctx, mut client_rx) = route_ctx(client);
14353
14354 let (open_task, bind) = relay_route_open(
14355 &handler,
14356 client,
14357 &client_ctx.egress,
14358 &mut module_rx,
14359 200,
14360 "aft",
14361 "acked-before-mark",
14362 )
14363 .await;
14364 let (module_channel, module_epoch) = route_bind_channel(&bind);
14365 hold.wait().await;
14367
14368 assert!(handler
14370 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14371 .await
14372 .unwrap()
14373 .is_empty());
14374 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
14375
14376 hold.wait().await;
14377 assert!(
14378 open_task.await.unwrap().is_empty(),
14379 "route.open answers accepted"
14380 );
14381
14382 match module_rx.try_recv() {
14385 Err(mpsc::error::TryRecvError::Empty) => {}
14386 Ok(outbound) => panic!(
14387 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
14388 outbound.frame.header.ty,
14389 outbound.frame.header.channel,
14390 outbound.frame.header.epoch
14391 ),
14392 Err(err) => panic!("module egress closed: {err}"),
14393 }
14394
14395 let router = Router::with_control_handler(Arc::new(handler.clone()));
14398 let request = Frame::build(
14399 FrameType::Request,
14400 Flags::new(true, Priority::Interactive, false),
14401 client_channel,
14402 client_epoch,
14403 201,
14404 b"first request".to_vec(),
14405 )
14406 .unwrap();
14407 router
14408 .route_for_connection(&client_ctx, request)
14409 .await
14410 .unwrap();
14411 let delivered = module_rx.try_recv().unwrap().frame;
14412 assert_eq!(delivered.header.ty, FrameType::Request);
14413 assert_eq!(
14414 (delivered.header.channel, delivered.header.epoch),
14415 (module_channel, module_epoch)
14416 );
14417 assert_eq!(delivered.body, b"first request");
14418 }
14419
14420 #[tokio::test]
14426 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
14427 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14428 let client = ConnectionId::new(42);
14429 let (client_ctx, _client_rx) = route_ctx(client);
14430
14431 let (open_task, bind) = relay_route_open(
14432 &handler,
14433 client,
14434 &client_ctx.egress,
14435 &mut module_rx,
14436 300,
14437 "aft",
14438 "released-before-mark",
14439 )
14440 .await;
14441 let (module_channel, module_epoch) = route_bind_channel(&bind);
14442 hold.wait().await;
14443
14444 handler.cleanup_connection(client).unwrap();
14445 assert!(
14446 module_rx.try_recv().is_err(),
14447 "client teardown must not GOODBYE a bind route.open is still queueing"
14448 );
14449
14450 hold.wait().await;
14451 let response = open_task.await.unwrap();
14452 assert_eq!(response.len(), 1);
14453 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14454
14455 let goodbye = module_rx
14456 .try_recv()
14457 .expect("module is told to drop the binding the daemon abandoned")
14458 .frame;
14459 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14460 assert_eq!(
14461 (goodbye.header.channel, goodbye.header.epoch),
14462 (module_channel, module_epoch)
14463 );
14464 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14465 }
14466
14467 #[tokio::test]
14472 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
14473 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14474 let client = ConnectionId::new(43);
14475 let (client_ctx, client_rx) = route_ctx(client);
14476
14477 let (open_task, bind) = relay_route_open(
14478 &handler,
14479 client,
14480 &client_ctx.egress,
14481 &mut module_rx,
14482 400,
14483 "aft",
14484 "gone-client-before-mark",
14485 )
14486 .await;
14487 let (module_channel, module_epoch) = route_bind_channel(&bind);
14488 hold.wait().await;
14489
14490 drop(client_rx);
14491 assert!(client_ctx.egress.is_closed());
14492 handler
14493 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14494 .await
14495 .unwrap();
14496
14497 hold.wait().await;
14498 let response = open_task.await.unwrap();
14499 assert_eq!(response.len(), 1);
14500 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14501
14502 let goodbye = module_rx
14503 .try_recv()
14504 .expect("module is told to drop the binding it acked")
14505 .frame;
14506 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14507 assert_eq!(
14508 (goodbye.header.channel, goodbye.header.epoch),
14509 (module_channel, module_epoch)
14510 );
14511 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14512 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14513 }
14514
14515 #[test]
14522 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
14523 let handler = ControlHandler::default();
14524 let module_connection = ConnectionId::new(30);
14525 let client_connection = ConnectionId::new(31);
14526
14527 handler
14528 .refuse_to_end_module_connection_for_a_client(
14529 module_connection,
14530 77,
14531 ForwardingError::ConnectionClosing {
14532 connection_id: client_connection,
14533 },
14534 )
14535 .expect("a closing client must never end the module connection");
14536
14537 assert!(matches!(
14538 handler.refuse_to_end_module_connection_for_a_client(
14539 module_connection,
14540 78,
14541 ForwardingError::ConnectionClosing {
14542 connection_id: module_connection,
14543 },
14544 ),
14545 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
14546 connection_id
14547 })) if connection_id == module_connection
14548 ));
14549 assert!(matches!(
14550 handler.refuse_to_end_module_connection_for_a_client(
14551 module_connection,
14552 79,
14553 ForwardingError::Poisoned,
14554 ),
14555 Err(RouterError::Forwarding(ForwardingError::Poisoned))
14556 ));
14557 assert!(matches!(
14558 handler.refuse_to_end_module_connection_for_a_client(
14559 module_connection,
14560 80,
14561 ForwardingError::StaleModuleEndpoint,
14562 ),
14563 Err(RouterError::Forwarding(
14564 ForwardingError::StaleModuleEndpoint
14565 ))
14566 ));
14567 }
14568
14569 #[tokio::test]
14582 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
14583 let registry = Arc::new(Registry::default());
14584 let forwarding = Arc::new(ForwardingTable::default());
14585 let supervisor = SupervisorHandle::new();
14586 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14587 let handler =
14588 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14589 .with_supervisor(supervisor);
14590
14591 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14592 hello_via_sink(
14593 &handler,
14594 &target_ctx,
14595 &mut target_rx,
14596 hello_frame("target", PROTOCOL_VERSION, 1),
14597 )
14598 .await;
14599
14600 let wrong_nonce = handler
14604 .handle_control_frame(
14605 &route_ctx(ConnectionId::new(91)).0,
14606 route_open_frame_with_admission_facts(
14607 20,
14608 "target",
14609 unique_project_root("admission-facts"),
14610 Some(subc_control::ConsumerIdentity {
14611 module_id: "fed".to_string(),
14612 launch_nonce: "not-the-real-nonce".to_string(),
14613 }),
14614 None,
14615 ),
14616 )
14617 .await
14618 .unwrap();
14619 assert_eq!(
14620 parse_error(&wrong_nonce[0])["code"],
14621 "bad_consumer_identity",
14622 "a mismatched launch nonce must be refused, not stamped Reserved"
14623 );
14624
14625 let never_spawned = handler
14629 .handle_control_frame(
14630 &route_ctx(ConnectionId::new(92)).0,
14631 route_open_frame_with_admission_facts(
14632 21,
14633 "target",
14634 unique_project_root("admission-facts"),
14635 Some(subc_control::ConsumerIdentity {
14636 module_id: "never-spawned".to_string(),
14637 launch_nonce: "any-nonce".to_string(),
14638 }),
14639 None,
14640 ),
14641 )
14642 .await
14643 .unwrap();
14644 assert_eq!(
14645 parse_error(&never_spawned[0])["code"],
14646 "bad_consumer_identity",
14647 "an unspawned module_id must be refused rather than accepted for lack of a record"
14648 );
14649 }
14650
14651 #[tokio::test]
14674 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
14675 let registry = Arc::new(Registry::default());
14676 let forwarding = Arc::new(ForwardingTable::default());
14677 let supervisor = SupervisorHandle::new();
14678 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14679 let handler =
14680 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14681 .with_supervisor(supervisor);
14682
14683 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
14684 hello_via_sink(
14685 &handler,
14686 &target_ctx,
14687 &mut target_rx,
14688 hello_frame("target", PROTOCOL_VERSION, 1),
14689 )
14690 .await;
14691
14692 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
14693 let route_handler = handler.clone();
14694 let route_task = tokio::spawn(async move {
14695 route_handler
14696 .handle_control_frame(
14697 &client_ctx,
14698 route_open_frame_with_admission_facts(
14699 30,
14700 "target",
14701 unique_project_root("admission-facts"),
14702 Some(subc_control::ConsumerIdentity {
14703 module_id: "fed".to_string(),
14704 launch_nonce: "fed-nonce".to_string(),
14705 }),
14706 None,
14707 ),
14708 )
14709 .await
14710 .unwrap()
14711 });
14712
14713 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14720 .await
14721 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
14722 .expect("module control channel closed before route.bind");
14723 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14724 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14725 panic!("expected route.bind")
14726 };
14727 assert_eq!(
14728 principal,
14729 Some(Principal::Reserved {
14730 module_id: "fed".to_string()
14731 }),
14732 "a correctly attested consumer must be stamped Reserved for its own id"
14733 );
14734
14735 handler
14736 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14737 .await
14738 .unwrap();
14739 assert!(route_task.await.unwrap().is_empty());
14740 assert!(
14741 matches!(
14742 serde_json::from_slice::<ClientControlResponse>(
14743 &client_rx.recv().await.unwrap().body
14744 )
14745 .unwrap(),
14746 ClientControlResponse::RouteOpen { .. }
14747 ),
14748 "the route must actually open, not merely avoid an error"
14749 );
14750 }
14751
14752 #[tokio::test(start_paused = true)]
14753 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
14754 let registry = Arc::new(Registry::default());
14755 let forwarding = Arc::new(ForwardingTable::default());
14756 let supervisor = SupervisorHandle::new();
14757 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14758 let handler =
14759 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14760 .with_supervisor(supervisor);
14761
14762 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
14763 hello_via_sink(
14764 &handler,
14765 &target_ctx,
14766 &mut target_rx,
14767 hello_frame("target", PROTOCOL_VERSION, 1),
14768 )
14769 .await;
14770
14771 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
14772 let direct_handler = handler.clone();
14773 let direct_open = tokio::spawn(async move {
14774 direct_handler
14775 .handle_control_frame(
14776 &direct_ctx,
14777 route_open_frame(2, "target", unique_project_root("route-census-direct")),
14778 )
14779 .await
14780 .unwrap()
14781 });
14782 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14783 .await
14784 .expect("no direct route.bind within 5s")
14785 .expect("target control channel closed before direct route.bind");
14786 handler
14787 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
14788 .await
14789 .unwrap();
14790 assert!(direct_open.await.unwrap().is_empty());
14791 let _ = direct_rx.recv().await.unwrap();
14792
14793 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
14794 let reserved_handler = handler.clone();
14795 let reserved_open = tokio::spawn(async move {
14796 reserved_handler
14797 .handle_control_frame(
14798 &reserved_ctx,
14799 route_open_frame_with_admission_facts(
14800 3,
14801 "target",
14802 unique_project_root("admission-facts"),
14803 Some(ConsumerIdentity {
14804 module_id: "fed".to_string(),
14805 launch_nonce: "fed-nonce".to_string(),
14806 }),
14807 None,
14808 ),
14809 )
14810 .await
14811 .unwrap()
14812 });
14813 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14814 .await
14815 .expect("no reserved route.bind within 5s")
14816 .expect("target control channel closed before reserved route.bind");
14817 handler
14818 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
14819 .await
14820 .unwrap();
14821 assert!(reserved_open.await.unwrap().is_empty());
14822 let _ = reserved_rx.recv().await.unwrap();
14823
14824 forwarding
14825 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
14826 .unwrap();
14827 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
14828 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
14829 module_id: Some("target".to_string()),
14830 })
14831 .unwrap();
14832 let census_frame =
14833 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
14834 let response = handler
14835 .handle_control_frame(&census_ctx, census_frame)
14836 .await
14837 .unwrap()
14838 .pop()
14839 .unwrap();
14840 let actual: Value = serde_json::from_slice(&response.body).unwrap();
14841 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
14842 assert!(matches!(
14843 decoded,
14844 ClientControlResponse::SupervisorRoutes { .. }
14845 ));
14846 let routes = actual["modules"][0]["routes"].as_array().unwrap();
14847 assert_eq!(routes.len(), 2);
14848 assert!(routes.iter().all(|route| route["draining"] == true));
14849 assert!(
14852 routes.iter().all(|route| route["drain_reason"] == "reload"),
14853 "draining routes must name the drain's reason: {routes:?}"
14854 );
14855 assert!(routes.iter().any(|route| {
14856 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
14857 }));
14858 assert!(routes.iter().any(|route| {
14859 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
14860 }));
14861
14862 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14863 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
14864 if std::env::var_os("UPDATE_GOLDEN").is_some() {
14865 std::fs::write(
14866 &golden_path,
14867 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
14868 )
14869 .unwrap();
14870 }
14871 let expected: Value =
14872 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
14873 assert_eq!(actual, expected);
14874 }
14875
14876 async fn query_live_roots(
14877 handler: &ControlHandler,
14878 module_ctx: &RouteCtx,
14879 ) -> ModuleControlResponseToModule {
14880 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
14881 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
14882 let response = handler
14883 .handle_control_frame(module_ctx, frame)
14884 .await
14885 .unwrap()
14886 .pop()
14887 .unwrap();
14888 serde_json::from_slice(&response.body).unwrap()
14889 }
14890
14891 #[tokio::test(start_paused = true)]
14892 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
14893 let registry = Arc::new(Registry::default());
14894 let forwarding = Arc::new(ForwardingTable::default());
14895 let handler = ControlHandler::with_forwarding(registry, forwarding);
14896 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
14897 hello_via_sink(
14898 &handler,
14899 &target_ctx,
14900 &mut target_rx,
14901 hello_frame("target", PROTOCOL_VERSION, 1),
14902 )
14903 .await;
14904 let root = unique_project_root("live-roots-known");
14905 let path = ProjectRootId::from_path_allowing_missing(root.path())
14906 .unwrap()
14907 .as_path()
14908 .to_path_buf();
14909 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
14910 let open_handler = handler.clone();
14911 let opened = tokio::spawn(async move {
14912 open_handler
14913 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
14914 .await
14915 .unwrap()
14916 });
14917 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14918 .await
14919 .unwrap()
14920 .unwrap();
14921 handler
14922 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
14923 .await
14924 .unwrap();
14925 assert!(opened.await.unwrap().is_empty());
14926 let _ = client_rx.recv().await.unwrap();
14927
14928 let root = unique_project_root("live-roots-pending");
14929 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
14930 .unwrap()
14931 .as_path()
14932 .to_path_buf();
14933 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
14934 let open_handler = handler.clone();
14935 let pending = tokio::spawn(async move {
14936 open_handler
14937 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
14938 .await
14939 .unwrap()
14940 });
14941 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14942 .await
14943 .unwrap()
14944 .unwrap();
14945 let actual = query_live_roots(&handler, &target_ctx).await;
14946 let ModuleControlResponseToModule::LiveRoots {
14947 roots,
14948 unknown_root_bindings,
14949 total_bindings,
14950 } = actual
14951 else {
14952 panic!("expected live roots")
14953 };
14954 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
14955 assert_eq!(unknown_root_bindings, 0);
14956 assert_eq!(
14957 roots.len(),
14958 2,
14959 "root-known arm must retain each canonical root"
14960 );
14961 assert_eq!(
14962 total_bindings,
14963 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14964 );
14965 let counts = roots
14966 .iter()
14967 .map(|root| (root.project_root.clone(), root.bound, root.pending))
14968 .collect::<Vec<_>>();
14969 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
14970 expected.sort_by(|a, b| a.0.cmp(&b.0));
14971 assert_eq!(
14972 counts, expected,
14973 "roots must sort by path and count pending separately"
14974 );
14975 handler
14976 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
14977 .await
14978 .unwrap();
14979 assert!(pending.await.unwrap().is_empty());
14980 }
14981
14982 #[tokio::test(start_paused = true)]
14983 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
14984 let forwarding = Arc::new(ForwardingTable::default());
14985 let handler =
14986 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14987 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
14988 hello_via_sink(
14989 &handler,
14990 &target_ctx,
14991 &mut target_rx,
14992 hello_frame("target", PROTOCOL_VERSION, 1),
14993 )
14994 .await;
14995 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
14996 let pending = forwarding
14997 .begin_route_bind_relay_for_test(
14998 client_ctx.connection_id,
14999 client_ctx.egress.clone(),
15000 2,
15001 "target",
15002 )
15003 .unwrap();
15004 forwarding
15005 .complete_pending_relay(
15006 target_ctx.connection_id,
15007 pending.corr,
15008 RouteBindRelayOutcome::Accepted,
15009 )
15010 .unwrap();
15011 let actual = query_live_roots(&handler, &target_ctx).await;
15012 let ModuleControlResponseToModule::LiveRoots {
15013 roots,
15014 unknown_root_bindings,
15015 total_bindings,
15016 } = actual
15017 else {
15018 panic!("expected live roots")
15019 };
15020 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
15021 assert_eq!(
15022 unknown_root_bindings, 1,
15023 "unknown-root arm must not read as no bindings"
15024 );
15025 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
15026 assert_eq!(
15027 total_bindings,
15028 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15029 );
15030 }
15031
15032 #[tokio::test(start_paused = true)]
15040 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
15041 let forwarding = Arc::new(ForwardingTable::default());
15042 let handler =
15043 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
15044 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
15045 let replies = handler
15046 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
15047 .await
15048 .unwrap();
15049 let queued_by_hello = module_rx.len();
15050
15051 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
15052 let open_handler = handler.clone();
15053 let open = tokio::spawn(async move {
15054 open_handler
15055 .handle_control_frame(
15056 &client_ctx,
15057 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
15058 )
15059 .await
15060 .unwrap()
15061 });
15062 let mut spins = 0;
15064 while module_rx.len() == queued_by_hello {
15065 spins += 1;
15066 assert!(spins < 10_000, "route.open never queued a route.bind");
15067 tokio::task::yield_now().await;
15068 }
15069
15070 for reply in replies {
15072 module_ctx.egress.send(reply).await.unwrap();
15073 }
15074
15075 let first = module_rx.recv().await.unwrap().frame;
15076 assert_eq!(
15077 first.header.ty,
15078 FrameType::HelloAck,
15079 "the first frame a registering module reads must be its HELLO_ACK"
15080 );
15081 assert_eq!(first.header.corr, 7);
15082 let second = module_rx.recv().await.unwrap().frame;
15083 assert_eq!(second.header.ty, FrameType::Request);
15084 assert!(
15085 matches!(
15086 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
15087 ModuleControlRequest::RouteBind { .. }
15088 ),
15089 "the route.bind follows the ack"
15090 );
15091 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
15092
15093 handler
15094 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
15095 .await
15096 .unwrap();
15097 assert!(open.await.unwrap().is_empty());
15098 let _ = client_rx.recv().await.unwrap();
15099 }
15100
15101 #[tokio::test(start_paused = true)]
15102 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
15103 let handler = ControlHandler::with_forwarding(
15104 Arc::new(Registry::default()),
15105 Arc::new(ForwardingTable::default()),
15106 );
15107 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
15108 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
15109 hello_via_sink(
15110 &handler,
15111 &first_ctx,
15112 &mut first_rx,
15113 hello_frame("first", PROTOCOL_VERSION, 1),
15114 )
15115 .await;
15116 hello_via_sink(
15117 &handler,
15118 &second_ctx,
15119 &mut second_rx,
15120 hello_frame("second", PROTOCOL_VERSION, 2),
15121 )
15122 .await;
15123 let root = unique_project_root("second-only");
15124 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
15125 let cloned = handler.clone();
15126 let open = tokio::spawn(async move {
15127 cloned
15128 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
15129 .await
15130 .unwrap()
15131 });
15132 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
15133 .await
15134 .unwrap()
15135 .unwrap();
15136 let first = query_live_roots(&handler, &first_ctx).await;
15137 let second = query_live_roots(&handler, &second_ctx).await;
15138 assert!(
15139 matches!(
15140 first,
15141 ModuleControlResponseToModule::LiveRoots {
15142 total_bindings: 0,
15143 ..
15144 }
15145 ),
15146 "cross-module scope must not expose another module's roots"
15147 );
15148 assert!(
15149 matches!(
15150 second,
15151 ModuleControlResponseToModule::LiveRoots {
15152 total_bindings: 1,
15153 ..
15154 }
15155 ),
15156 "second module must see its pending route"
15157 );
15158 handler
15159 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
15160 .await
15161 .unwrap();
15162 assert!(open.await.unwrap().is_empty());
15163 }
15164
15165 #[tokio::test(start_paused = true)]
15166 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
15167 let handler = ControlHandler::with_forwarding(
15168 Arc::new(Registry::default()),
15169 Arc::new(ForwardingTable::default()),
15170 );
15171 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
15172 hello_via_sink(
15173 &handler,
15174 &target_ctx,
15175 &mut target_rx,
15176 hello_frame("target", PROTOCOL_VERSION, 1),
15177 )
15178 .await;
15179 let actual = query_live_roots(&handler, &target_ctx).await;
15180 let ModuleControlResponseToModule::LiveRoots {
15181 roots,
15182 unknown_root_bindings,
15183 total_bindings,
15184 } = actual
15185 else {
15186 panic!("expected live roots")
15187 };
15188 assert!(roots.is_empty());
15189 assert_eq!(unknown_root_bindings, 0);
15190 assert_eq!(total_bindings, 0);
15191 assert_eq!(
15192 total_bindings,
15193 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15194 );
15195 }
15196
15197 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
15204 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
15205 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
15206 let text = std::fs::read_to_string(&path)
15207 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
15208 let vectors: Vec<(String, Value)> = text
15209 .lines()
15210 .filter(|line| !line.trim().is_empty())
15211 .map(|line| {
15212 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
15213 let id = entry["corpus_id"]
15214 .as_str()
15215 .expect("every vector carries a corpus_id")
15216 .to_string();
15217 (id, entry["package"].clone())
15218 })
15219 .collect();
15220 assert_eq!(
15224 vectors.len(),
15225 3,
15226 "vendored fed corpus changed size; re-sync from subc-federation"
15227 );
15228
15229 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
15244 let richest = vectors
15245 .iter()
15246 .filter_map(|(_, package)| package.as_object())
15247 .map(|object| {
15248 object
15249 .keys()
15250 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
15251 .count()
15252 })
15253 .max()
15254 .unwrap_or(0);
15255 assert!(
15256 richest >= 2,
15257 "vendored corpus no longer carries a package with unmodelled fields, \
15258 so the relay test can no longer distinguish a verbatim relay from a lossy one"
15259 );
15260
15261 vectors
15262 }
15263
15264 #[tokio::test]
15272 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
15273 for (corpus_id, package) in fed_admission_facts_vectors() {
15274 let registry = Arc::new(Registry::default());
15275 let forwarding = Arc::new(ForwardingTable::default());
15276 let supervisor = SupervisorHandle::new();
15277 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15278 let handler =
15279 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15280 .with_supervisor(supervisor)
15281 .with_admission_facts_config(
15282 Some("fed".to_string()),
15283 Some(vec!["target".to_string()]),
15284 );
15285
15286 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
15287 hello_via_sink(
15288 &handler,
15289 &target_ctx,
15290 &mut target_rx,
15291 hello_frame("target", PROTOCOL_VERSION, 1),
15292 )
15293 .await;
15294
15295 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
15296 let route_handler = handler.clone();
15297 let expected = package.clone();
15298 let route_task = tokio::spawn(async move {
15299 route_handler
15300 .handle_control_frame(
15301 &client_ctx,
15302 route_open_frame_with_admission_facts(
15303 20,
15304 "target",
15305 unique_project_root("admission-facts"),
15306 Some(subc_control::ConsumerIdentity {
15307 module_id: "fed".to_string(),
15308 launch_nonce: "fed-nonce".to_string(),
15309 }),
15310 Some(package),
15311 ),
15312 )
15313 .await
15314 .unwrap()
15315 });
15316
15317 let bind_frame = target_rx.recv().await.unwrap();
15318 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15319 let ModuleControlRequest::RouteBind {
15320 admission_facts, ..
15321 } = bind
15322 else {
15323 panic!("{corpus_id}: expected route.bind")
15324 };
15325 assert_eq!(
15326 admission_facts,
15327 Some(expected),
15328 "{corpus_id}: relay must not add, drop or reshape any field"
15329 );
15330
15331 handler
15332 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15333 .await
15334 .unwrap();
15335 route_task.await.unwrap();
15336 }
15337 }
15338
15339 #[tokio::test]
15340 async fn route_bind_never_relays_the_openers_build_class() {
15341 let registry = Arc::new(Registry::default());
15342 let forwarding = Arc::new(ForwardingTable::default());
15343 let handler =
15344 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15345 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
15346 hello_via_sink(
15347 &handler,
15348 &target_ctx,
15349 &mut target_rx,
15350 hello_frame("target", PROTOCOL_VERSION, 1),
15351 )
15352 .await;
15353
15354 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
15355 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
15356 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15357 busy_refusals: None,
15358 target: RouteTarget::ToolProvider {
15359 module_id: "target".to_string(),
15360 },
15361 identity,
15362 consumer_identity: None,
15363 consumer_capabilities: None,
15364 role_versions: None,
15365 admission_facts: None,
15366 scope: None,
15367 })
15368 .unwrap();
15369 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
15370
15371 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
15372 let route_handler = handler.clone();
15373 let route_task = tokio::spawn(async move {
15374 route_handler
15375 .handle_control_frame(&client_ctx, open)
15376 .await
15377 .unwrap()
15378 });
15379 let bind_frame = target_rx.recv().await.unwrap();
15380 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15381 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
15382 panic!("expected route.bind")
15383 };
15384 assert_eq!(
15385 identity.build_class, None,
15386 "route.bind carried the build class"
15387 );
15388 assert!(
15389 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
15390 "the route.bind bytes still name build_class"
15391 );
15392 handler
15393 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15394 .await
15395 .unwrap();
15396 route_task.await.unwrap();
15397 }
15398
15399 #[tokio::test]
15400 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
15401 let registry = Arc::new(Registry::default());
15402 let forwarding = Arc::new(ForwardingTable::default());
15403 let supervisor = SupervisorHandle::new();
15404 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15405 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
15406 let handler =
15407 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15408 .with_supervisor(supervisor)
15409 .with_admission_facts_config(
15410 Some("fed".to_string()),
15411 Some(vec!["target".to_string()]),
15412 );
15413
15414 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
15415 hello_via_sink(
15416 &handler,
15417 &target_ctx,
15418 &mut target_rx,
15419 hello_frame("target", PROTOCOL_VERSION, 1),
15420 )
15421 .await;
15422 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
15423 hello_via_sink(
15424 &handler,
15425 &other_ctx,
15426 &mut other_rx,
15427 hello_frame("other", PROTOCOL_VERSION, 2),
15428 )
15429 .await;
15430
15431 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
15432 let expected_facts = facts.clone();
15433 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
15434 let route_handler = handler.clone();
15435 let route_task = tokio::spawn(async move {
15436 route_handler
15437 .handle_control_frame(
15438 &client_ctx,
15439 route_open_frame_with_admission_facts(
15440 10,
15441 "target",
15442 unique_project_root("admission-facts"),
15443 Some(subc_control::ConsumerIdentity {
15444 module_id: "fed".to_string(),
15445 launch_nonce: "fed-nonce".to_string(),
15446 }),
15447 Some(facts.clone()),
15448 ),
15449 )
15450 .await
15451 .unwrap()
15452 });
15453 let bind_frame = target_rx.recv().await.unwrap();
15454 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15455 let ModuleControlRequest::RouteBind {
15456 admission_facts, ..
15457 } = bind
15458 else {
15459 panic!("expected route.bind")
15460 };
15461 assert_eq!(admission_facts, Some(expected_facts));
15462 handler
15463 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15464 .await
15465 .unwrap();
15466 assert!(route_task.await.unwrap().is_empty());
15467 assert!(matches!(
15468 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
15469 .unwrap(),
15470 ClientControlResponse::RouteOpen { .. }
15471 ));
15472
15473 let direct = handler
15474 .handle_control_frame(
15475 &route_ctx(ConnectionId::new(73)).0,
15476 route_open_frame_with_admission_facts(
15477 11,
15478 "target",
15479 unique_project_root("admission-facts"),
15480 None,
15481 Some(json!({"x": 1})),
15482 ),
15483 )
15484 .await
15485 .unwrap();
15486 assert_eq!(
15487 parse_error(&direct[0])["code"],
15488 "admission_facts_not_permitted"
15489 );
15490
15491 let different_reserved = handler
15492 .handle_control_frame(
15493 &route_ctx(ConnectionId::new(77)).0,
15494 route_open_frame_with_admission_facts(
15495 15,
15496 "target",
15497 unique_project_root("admission-facts"),
15498 Some(subc_control::ConsumerIdentity {
15499 module_id: "other".to_string(),
15500 launch_nonce: "other-nonce".to_string(),
15501 }),
15502 Some(json!({"x": 1})),
15503 ),
15504 )
15505 .await
15506 .unwrap();
15507 assert_eq!(
15508 parse_error(&different_reserved[0])["code"],
15509 "admission_facts_not_permitted"
15510 );
15511
15512 let other_target = handler
15513 .handle_control_frame(
15514 &route_ctx(ConnectionId::new(74)).0,
15515 route_open_frame_with_admission_facts(
15516 12,
15517 "other",
15518 unique_project_root("admission-facts"),
15519 Some(subc_control::ConsumerIdentity {
15520 module_id: "fed".to_string(),
15521 launch_nonce: "fed-nonce".to_string(),
15522 }),
15523 Some(json!({"x": 1})),
15524 ),
15525 )
15526 .await
15527 .unwrap();
15528 assert_eq!(
15529 parse_error(&other_target[0])["code"],
15530 "admission_facts_target_not_allowed"
15531 );
15532
15533 let nonexistent = handler
15534 .handle_control_frame(
15535 &route_ctx(ConnectionId::new(75)).0,
15536 route_open_frame_with_admission_facts(
15537 13,
15538 "missing",
15539 unique_project_root("admission-facts"),
15540 None,
15541 Some(json!({"x": 1})),
15542 ),
15543 )
15544 .await
15545 .unwrap();
15546 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
15547
15548 let described = handler
15549 .handle_control_frame(
15550 &route_ctx(ConnectionId::new(76)).0,
15551 Frame::build(
15552 FrameType::Request,
15553 control_flags(),
15554 0,
15555 0,
15556 14,
15557 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15558 )
15559 .unwrap(),
15560 )
15561 .await
15562 .unwrap();
15563 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15564 serde_json::from_slice(&described[0].body).unwrap()
15565 else {
15566 panic!("expected server.describe response")
15567 };
15568 assert!(capabilities
15569 .iter()
15570 .any(|cap| cap == "admission_facts_relay_v1"));
15571 }
15572
15573 #[tokio::test]
15574 async fn admission_facts_without_configured_carrier_are_rejected() {
15575 let registry = Arc::new(Registry::default());
15576 let forwarding = Arc::new(ForwardingTable::default());
15577 let handler = ControlHandler::with_forwarding(registry, forwarding);
15578 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
15579 hello_via_sink(
15580 &handler,
15581 &target_ctx,
15582 &mut target_rx,
15583 hello_frame("target", PROTOCOL_VERSION, 1),
15584 )
15585 .await;
15586
15587 let responses = handler
15588 .handle_control_frame(
15589 &route_ctx(ConnectionId::new(79)).0,
15590 route_open_frame_with_admission_facts(
15591 16,
15592 "target",
15593 unique_project_root("admission-facts"),
15594 None,
15595 Some(json!({"x": 1})),
15596 ),
15597 )
15598 .await
15599 .unwrap();
15600 assert_eq!(
15601 parse_error(&responses[0])["code"],
15602 "admission_facts_not_permitted"
15603 );
15604 }
15605
15606 #[tokio::test]
15607 async fn route_open_relays_consumer_capabilities_verbatim() {
15608 let registry = Arc::new(Registry::default());
15609 let forwarding = Arc::new(ForwardingTable::default());
15610 let handler =
15611 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15612 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
15613 hello_via_sink(
15614 &handler,
15615 &module_ctx,
15616 &mut module_rx,
15617 hello_frame("aft", PROTOCOL_VERSION, 7),
15618 )
15619 .await;
15620
15621 let expected = vec!["elicitation".to_string(), "roots".to_string()];
15622 let expected_for_request = expected.clone();
15623 let project_root = unique_project_root("consumer-capabilities-present");
15624 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
15625 let route_handler = handler.clone();
15626 let route_task = tokio::spawn(async move {
15627 route_handler
15628 .handle_control_frame(
15629 &client_ctx,
15630 route_open_frame_with_consumer_capabilities(
15631 401,
15632 "aft",
15633 project_root,
15634 Some(expected_for_request),
15635 ),
15636 )
15637 .await
15638 .unwrap()
15639 });
15640 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15641 .await
15642 .unwrap()
15643 .unwrap();
15644 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15645 let ModuleControlRequest::RouteBind {
15646 consumer_capabilities,
15647 ..
15648 } = bind
15649 else {
15650 panic!("expected route.bind request, got {bind:?}");
15651 };
15652 assert_eq!(consumer_capabilities, Some(expected.clone()));
15653
15654 handler
15655 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15656 .await
15657 .unwrap();
15658 let route_response = route_task.await.unwrap();
15659 assert!(route_response.is_empty());
15660 let published = client_rx.recv().await.unwrap();
15661 assert!(matches!(
15662 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15663 ClientControlResponse::RouteOpen { .. }
15664 ));
15665 }
15666
15667 #[tokio::test]
15668 async fn route_open_without_consumer_capabilities_relays_none() {
15669 let registry = Arc::new(Registry::default());
15670 let forwarding = Arc::new(ForwardingTable::default());
15671 let handler =
15672 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15673 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
15674 hello_via_sink(
15675 &handler,
15676 &module_ctx,
15677 &mut module_rx,
15678 hello_frame("aft", PROTOCOL_VERSION, 7),
15679 )
15680 .await;
15681
15682 let project_root = unique_project_root("consumer-capabilities-absent");
15683 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
15684 let route_handler = handler.clone();
15685 let route_task = tokio::spawn(async move {
15686 route_handler
15687 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
15688 .await
15689 .unwrap()
15690 });
15691 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15692 .await
15693 .unwrap()
15694 .unwrap();
15695 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15696 let ModuleControlRequest::RouteBind {
15697 consumer_capabilities,
15698 ..
15699 } = bind
15700 else {
15701 panic!("expected route.bind request, got {bind:?}");
15702 };
15703 assert_eq!(consumer_capabilities, None);
15704
15705 handler
15706 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15707 .await
15708 .unwrap();
15709 let route_response = route_task.await.unwrap();
15710 assert!(route_response.is_empty());
15711 let published = client_rx.recv().await.unwrap();
15712 assert!(matches!(
15713 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15714 ClientControlResponse::RouteOpen { .. }
15715 ));
15716 }
15717
15718 async fn bind_role_versions_for(
15722 sent: Option<BTreeMap<String, String>>,
15723 connection: u64,
15724 ) -> Option<BTreeMap<String, String>> {
15725 let registry = Arc::new(Registry::default());
15726 let forwarding = Arc::new(ForwardingTable::default());
15727 let handler =
15728 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15729 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
15730 hello_via_sink(
15731 &handler,
15732 &module_ctx,
15733 &mut module_rx,
15734 hello_frame("aft", PROTOCOL_VERSION, 7),
15735 )
15736 .await;
15737 let project_root = unique_project_root("role-versions");
15738 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
15739 let route_handler = handler.clone();
15740 let route_task = tokio::spawn(async move {
15741 route_handler
15742 .handle_control_frame(
15743 &client_ctx,
15744 route_open_frame_with_role_versions(403, "aft", project_root, sent),
15745 )
15746 .await
15747 .unwrap()
15748 });
15749 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15750 .await
15751 .expect("a well-formed route.open reaches the module as a bind")
15752 .unwrap();
15753 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15754 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
15755 panic!("expected route.bind request, got {bind:?}");
15756 };
15757 handler
15758 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15759 .await
15760 .unwrap();
15761 assert!(route_task.await.unwrap().is_empty());
15762 let published = client_rx.recv().await.unwrap();
15763 assert!(matches!(
15764 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15765 ClientControlResponse::RouteOpen { .. }
15766 ));
15767 role_versions
15768 }
15769
15770 #[tokio::test]
15771 async fn route_open_relays_role_versions_verbatim() {
15772 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
15773 assert_eq!(
15774 bind_role_versions_for(Some(sent.clone()), 141).await,
15775 Some(sent)
15776 );
15777 }
15778
15779 #[tokio::test]
15782 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
15783 assert_eq!(bind_role_versions_for(None, 143).await, None);
15784 assert_eq!(
15785 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
15786 None
15787 );
15788 }
15789
15790 #[tokio::test]
15791 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
15792 let registry = Arc::new(Registry::default());
15793 let forwarding = Arc::new(ForwardingTable::default());
15794 let handler =
15795 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15796 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
15797 hello_via_sink(
15798 &handler,
15799 &module_ctx,
15800 &mut module_rx,
15801 hello_frame("aft", PROTOCOL_VERSION, 7),
15802 )
15803 .await;
15804
15805 let nine: BTreeMap<String, String> = (0..9)
15806 .map(|index| (format!("role-{index}"), "v1".to_string()))
15807 .collect();
15808 for (label, malformed) in [
15809 (
15810 "invalid role name",
15811 role_versions(&[("Tool_Provider", "v1")]),
15812 ),
15813 ("invalid version", role_versions(&[("tool-provider", "v0")])),
15814 ("nine entries", nine),
15815 ] {
15816 let responses = tokio::time::timeout(
15819 Duration::from_secs(1),
15820 handler.handle_control_frame(
15821 &route_ctx(ConnectionId::new(148)).0,
15822 route_open_frame_with_role_versions(
15823 404,
15824 "aft",
15825 unique_project_root("role-versions-malformed"),
15826 Some(malformed),
15827 ),
15828 ),
15829 )
15830 .await
15831 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
15832 .unwrap();
15833 assert_eq!(responses.len(), 1, "{label}");
15834 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
15835 let error = parse_error(&responses[0]);
15836 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
15837 assert_eq!(
15838 error["detail"]["field"], "role_versions",
15839 "{label}: {error}"
15840 );
15841 assert!(
15842 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
15843 "{label}: a malformed declaration is terminal"
15844 );
15845 assert!(
15846 module_rx.try_recv().is_err(),
15847 "{label}: the module must never see a bind"
15848 );
15849 }
15850 }
15851
15852 #[tokio::test]
15856 async fn route_role_versions_capability_is_advertised() {
15857 let handler = ControlHandler::new(Arc::new(Registry::default()));
15858 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
15859 let ack = hello_via_sink(
15860 &handler,
15861 &ctx,
15862 &mut rx,
15863 hello_frame("m", PROTOCOL_VERSION, 1),
15864 )
15865 .await;
15866 let ack = parse_ack(&ack);
15867 assert!(
15868 ack.subc_capabilities
15869 .iter()
15870 .any(|c| c == "route-role-versions/v1"),
15871 "{:?}",
15872 ack.subc_capabilities
15873 );
15874
15875 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15876 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15877 let reply = handler
15878 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
15879 .await
15880 .unwrap()
15881 .pop()
15882 .unwrap();
15883 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15884 serde_json::from_slice(&reply.body).unwrap()
15885 else {
15886 panic!("not a server.describe reply");
15887 };
15888 assert!(
15889 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
15890 "{capabilities:?}"
15891 );
15892 }
15893
15894 #[tokio::test]
15898 async fn catalog_events_capability_is_advertised() {
15899 let handler = ControlHandler::new(Arc::new(Registry::default()));
15900 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15901 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15902 let reply = handler
15903 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
15904 .await
15905 .unwrap()
15906 .pop()
15907 .unwrap();
15908 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15909 serde_json::from_slice(&reply.body).unwrap()
15910 else {
15911 panic!("not a server.describe reply");
15912 };
15913 assert!(
15914 capabilities
15915 .iter()
15916 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
15917 "{capabilities:?}"
15918 );
15919 }
15920
15921 #[tokio::test]
15922 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
15923 let handler = ControlHandler::new(Arc::new(Registry::default()));
15924 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
15925 let ack = hello_via_sink(
15926 &handler,
15927 &module,
15928 &mut module_rx,
15929 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
15930 )
15931 .await;
15932 let ack = parse_ack(&ack);
15933 assert!(
15934 ack.subc_capabilities
15935 .iter()
15936 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15937 "{:?}",
15938 ack.subc_capabilities
15939 );
15940 assert!(ack
15941 .subc_capabilities
15942 .iter()
15943 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15944
15945 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
15946 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15947 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15948 let reply = handler
15949 .handle_control_frame(&client, frame)
15950 .await
15951 .unwrap()
15952 .pop()
15953 .unwrap();
15954 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15955 serde_json::from_slice(&reply.body).unwrap()
15956 else {
15957 panic!("not a server.describe reply");
15958 };
15959 assert!(
15960 capabilities
15961 .iter()
15962 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15963 "{capabilities:?}"
15964 );
15965 assert!(capabilities
15966 .iter()
15967 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15968 }
15969
15970 #[tokio::test]
15971 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
15972 let handler = ControlHandler::new(Arc::new(Registry::default()));
15973 let (module, mut rx) = route_ctx(ConnectionId::new(156));
15974 let ack = hello_via_sink(
15975 &handler,
15976 &module,
15977 &mut rx,
15978 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
15979 )
15980 .await;
15981 assert!(parse_ack(&ack)
15982 .subc_capabilities
15983 .iter()
15984 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15985 assert!(parse_ack(&ack)
15986 .subc_capabilities
15987 .iter()
15988 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15989 let (client, _rx) = route_ctx(ConnectionId::new(157));
15990 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15991 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15992 let reply = handler
15993 .handle_control_frame(&client, frame)
15994 .await
15995 .unwrap()
15996 .pop()
15997 .unwrap();
15998 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15999 serde_json::from_slice(&reply.body).unwrap()
16000 else {
16001 panic!("server.describe reply")
16002 };
16003 assert!(capabilities
16004 .iter()
16005 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
16006 assert!(capabilities
16007 .iter()
16008 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
16009 }
16010
16011 #[tokio::test]
16012 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
16013 let handler = ControlHandler::new(Arc::new(Registry::default()));
16014 let (module, mut rx) = route_ctx(ConnectionId::new(158));
16015 let ack = hello_via_sink(
16016 &handler,
16017 &module,
16018 &mut rx,
16019 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
16020 )
16021 .await;
16022 assert!(parse_ack(&ack)
16023 .subc_capabilities
16024 .iter()
16025 .any(|cap| cap == "scope-workdir/v1"));
16026 let (client, _rx) = route_ctx(ConnectionId::new(159));
16027 let frame = Frame::build(
16028 FrameType::Request,
16029 control_flags(),
16030 0,
16031 0,
16032 5,
16033 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16034 )
16035 .unwrap();
16036 let reply = handler
16037 .handle_control_frame(&client, frame)
16038 .await
16039 .unwrap()
16040 .remove(0);
16041 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16042 serde_json::from_slice(&reply.body).unwrap()
16043 else {
16044 panic!("server.describe reply")
16045 };
16046 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
16047 }
16048
16049 #[tokio::test]
16050 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
16051 let handler = ControlHandler::new(Arc::new(Registry::default()));
16052 let (module, mut rx) = route_ctx(ConnectionId::new(160));
16053 let ack = hello_via_sink(
16054 &handler,
16055 &module,
16056 &mut rx,
16057 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
16058 )
16059 .await;
16060 assert!(parse_ack(&ack)
16061 .subc_capabilities
16062 .iter()
16063 .any(|cap| cap == "worker-runs/v1"));
16064 let (client, _rx) = route_ctx(ConnectionId::new(161));
16065 let frame = Frame::build(
16066 FrameType::Request,
16067 control_flags(),
16068 0,
16069 0,
16070 5,
16071 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16072 )
16073 .unwrap();
16074 let reply = handler
16075 .handle_control_frame(&client, frame)
16076 .await
16077 .unwrap()
16078 .remove(0);
16079 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16080 serde_json::from_slice(&reply.body).unwrap()
16081 else {
16082 panic!("server.describe reply")
16083 };
16084 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
16085 }
16086
16087 #[tokio::test]
16088 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
16089 let handler = ControlHandler::new(Arc::new(Registry::default()));
16090 let (module, mut rx) = route_ctx(ConnectionId::new(154));
16091 let ack = hello_via_sink(
16092 &handler,
16093 &module,
16094 &mut rx,
16095 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
16096 )
16097 .await;
16098 assert!(parse_ack(&ack)
16099 .subc_capabilities
16100 .iter()
16101 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16102 let (client, _rx) = route_ctx(ConnectionId::new(155));
16103 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16104 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16105 let reply = handler
16106 .handle_control_frame(&client, frame)
16107 .await
16108 .unwrap()
16109 .pop()
16110 .unwrap();
16111 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16112 serde_json::from_slice(&reply.body).unwrap()
16113 else {
16114 panic!("not a server.describe reply")
16115 };
16116 assert!(capabilities
16117 .iter()
16118 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16119 }
16120
16121 #[tokio::test]
16122 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
16123 let registry = Arc::new(Registry::default());
16124 let forwarding = Arc::new(ForwardingTable::default());
16125 let handler =
16126 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16127 .with_health_probe_timeout(Duration::from_secs(5));
16128 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
16129 hello_via_sink(
16130 &handler,
16131 &module_ctx,
16132 &mut module_rx,
16133 non_routable_hello_frame_with_control_ops(
16134 "mcp",
16135 300,
16136 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16137 ),
16138 )
16139 .await;
16140 assert!(registry
16141 .get_module("mcp")
16142 .unwrap()
16143 .unwrap()
16144 .manifest
16145 .provides
16146 .is_empty());
16147
16148 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
16149 let route_response = handler
16150 .handle_control_frame(
16151 &route_client_ctx,
16152 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
16153 )
16154 .await
16155 .unwrap();
16156 assert_eq!(route_response[0].header.ty, FrameType::Error);
16157 assert_eq!(
16158 parse_error(&route_response[0])["code"],
16159 "target_unavailable"
16160 );
16161 assert!(parse_error(&route_response[0])["message"]
16162 .as_str()
16163 .unwrap()
16164 .contains("does not provide the requested target"));
16165 assert!(module_rx.try_recv().is_err());
16166
16167 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
16168 let health_handler = handler.clone();
16169 let health_task = tokio::spawn(async move {
16170 health_handler
16171 .handle_control_frame(
16172 &health_client_ctx,
16173 supervisor_health_probe_frame(302, "mcp"),
16174 )
16175 .await
16176 .unwrap()
16177 });
16178 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16179 .await
16180 .unwrap()
16181 .unwrap();
16182 assert_eq!(
16183 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
16184 ModuleControlRequest::HealthCheck {}
16185 );
16186 handler
16187 .handle_control_frame(
16188 &module_ctx,
16189 health_response(health_frame.header.corr, HealthStatus::Ok),
16190 )
16191 .await
16192 .unwrap();
16193 let health_response = health_task.await.unwrap();
16194 assert_eq!(health_response[0].header.ty, FrameType::Response);
16195 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
16196 ClientControlResponse::SupervisorHealthProbe {
16197 module_id, status, ..
16198 } => {
16199 assert_eq!(module_id, "mcp");
16200 assert_eq!(status, HealthStatus::Ok);
16201 }
16202 other => panic!("unexpected health response: {other:?}"),
16203 }
16204
16205 forwarding
16210 .cleanup_connection(module_ctx.connection_id)
16211 .unwrap();
16212 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
16213 let cleanup_response = tokio::time::timeout(
16214 Duration::from_millis(200),
16215 handler.handle_control_frame(
16216 &cleanup_probe_ctx,
16217 supervisor_health_probe_frame(303, "mcp"),
16218 ),
16219 )
16220 .await
16221 .expect("probe should fail immediately when the control lane is gone")
16222 .unwrap();
16223 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
16224 assert_eq!(
16225 parse_error(&cleanup_response[0])["code"],
16226 "target_unavailable"
16227 );
16228 assert!(parse_error(&cleanup_response[0])["message"]
16229 .as_str()
16230 .unwrap()
16231 .contains("no module connection"));
16232
16233 handler
16234 .cleanup_connection(module_ctx.connection_id)
16235 .unwrap();
16236 }
16237
16238 #[tokio::test]
16239 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
16240 let registry = Arc::new(Registry::default());
16241 let supervisor_handle = SupervisorHandle::new();
16242 let supervisor =
16243 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
16244 .with_handle(supervisor_handle.clone())
16245 .with_connection_file_path(
16246 std::env::temp_dir()
16247 .join(format!("subc-route-open-warming-{}", std::process::id())),
16248 );
16249 let module = supervisor
16250 .supervise_configured(
16251 ModuleSpec {
16252 script: None,
16253 carry_containment: false,
16254 module_id: "warming".to_string(),
16255 program: fake_aft_stub_path(),
16256 args: Vec::new(),
16257 env: Vec::new(),
16258 reserved: false,
16259 reserved_prefixes: Vec::new(),
16260 protocol: ModuleProtocol::Subc,
16261 overlap: Default::default(),
16262 },
16263 true,
16264 )
16265 .unwrap();
16266 assert_eq!(module.state().unwrap(), ModuleState::Running);
16267
16268 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16269 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
16270 let response = handler
16271 .handle_control_frame(
16272 &ctx,
16273 route_open_frame(304, "warming", unique_project_root("warming")),
16274 )
16275 .await
16276 .unwrap();
16277 module.stop().await.unwrap();
16278
16279 assert_eq!(response[0].header.ty, FrameType::Error);
16280 let error = parse_error(&response[0]);
16281 assert_eq!(error["code"], "module_warming");
16282 assert!(error["message"]
16283 .as_str()
16284 .unwrap()
16285 .contains("state=running, enabled=true, live=false"));
16286 }
16287
16288 #[test]
16289 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
16290 let handler = ControlHandler::new(Arc::new(Registry::default()));
16291 let capture = EventCapture::default();
16292 let _subscriber =
16293 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16294 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
16295 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
16296 let pending = (0..limit).collect::<Vec<_>>();
16297 let response = handler
16298 .route_open_capacity_refusal(
16299 &ctx,
16300 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
16301 "busy",
16302 pending.len(),
16303 limit,
16304 )
16305 .unwrap();
16306 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16307 let event = capture
16308 .events()
16309 .into_iter()
16310 .find(|event| {
16311 event.target == "control"
16312 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
16313 })
16314 .expect("connection admission refusal event");
16315 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16316 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16317 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
16320 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
16321 }
16322
16323 #[test]
16324 fn route_open_target_cap_logs_admission_reason_and_capacity() {
16325 let handler = ControlHandler::new(Arc::new(Registry::default()));
16326 let capture = EventCapture::default();
16327 let _subscriber =
16328 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16329 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
16330 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
16331 let guards = (0..limit)
16332 .map(|_| {
16333 handler
16334 .route_bind_concurrency
16335 .try_admit("busy", limit)
16336 .unwrap()
16337 })
16338 .collect::<Vec<_>>();
16339 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
16340 Err(in_flight) => in_flight,
16341 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
16342 };
16343 let response = handler
16344 .route_open_target_capacity_refusal(
16345 &ctx,
16346 &RouteOpenCaller::unknown(),
16347 &route_open_frame(397, "busy", unique_project_root("target-cap")),
16348 "busy",
16349 in_flight,
16350 )
16351 .unwrap();
16352 assert_eq!(parse_error(&response)["code"], "target_unavailable");
16353 let event = capture
16354 .events()
16355 .into_iter()
16356 .find(|event| {
16357 event.target == "control"
16358 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
16359 })
16360 .expect("target admission refusal event");
16361 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16362 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16363 drop(guards);
16364 }
16365
16366 #[cfg(unix)]
16367 #[tokio::test(start_paused = true)]
16368 async fn supervisor_list_shows_a_failed_modules_next_retry() {
16369 let registry = Arc::new(Registry::default());
16370 let handle = SupervisorHandle::new();
16371 let supervisor =
16372 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
16373 .with_handle(handle.clone());
16374 let module = supervisor
16375 .spawn(crate::ModuleSpec {
16376 script: None,
16377 carry_containment: false,
16378 module_id: "retry-list".into(),
16379 program: PathBuf::from("/bin/sh"),
16380 args: vec!["-c".into(), "exit 17".into()],
16381 env: vec![],
16382 reserved: false,
16383 reserved_prefixes: vec![],
16384 protocol: ModuleProtocol::None,
16385 overlap: Default::default(),
16386 })
16387 .unwrap();
16388 let deadline = std::time::Instant::now() + Duration::from_secs(10);
16389 while module.state().unwrap() != crate::ModuleState::Failed {
16390 assert!(
16391 std::time::Instant::now() < deadline,
16392 "waiting for budget failure"
16393 );
16394 tokio::task::yield_now().await;
16395 }
16396 let handler = ControlHandler::new(registry).with_supervisor(handle);
16397 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16398 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16399 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16400 assert_eq!(body["modules"][0]["state"], "failed");
16401 let retry = body["modules"][0]["next_retry_at_ms"]
16402 .as_u64()
16403 .expect("scheduled retry on the wire");
16404 let now = std::time::SystemTime::now()
16405 .duration_since(std::time::UNIX_EPOCH)
16406 .unwrap()
16407 .as_millis() as u64;
16408 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
16409 module.set_enabled(false).await.unwrap();
16410 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16411 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16412 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16413 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
16414 }
16415
16416 #[tokio::test]
16417 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
16418 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
16419 let supervisor =
16420 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
16421 let module = supervisor
16422 .spawn(crate::ModuleSpec {
16423 script: None,
16424 carry_containment: false,
16425 module_id: "pid-read-budget".into(),
16426 program: std::env::current_exe().unwrap(),
16427 args: vec![
16428 "--exact".into(),
16429 "control::tests::pid_resource_read_fixture".into(),
16430 ],
16431 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16432 .into_iter()
16433 .map(|key| (key.into(), dir.display().to_string()))
16434 .chain(std::iter::once((
16435 "SUBC_SUBTREE_PID_FIXTURE".into(),
16436 "1".into(),
16437 )))
16438 .collect(),
16439 reserved: false,
16440 reserved_prefixes: Vec::new(),
16441 protocol: ModuleProtocol::None,
16442 overlap: Default::default(),
16443 })
16444 .unwrap();
16445 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16446 while !matches!(
16447 module.child_resource_usage(),
16448 subc_control::ChildResourceUsage::Measured(_)
16449 ) {
16450 assert!(
16451 tokio::time::Instant::now() < ready_deadline,
16452 "fixture process was not measurable"
16453 );
16454 tokio::time::sleep(Duration::from_millis(10)).await;
16455 }
16456 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
16457 let reading = {
16458 let _guard = crate::subtree_resources::fault_read_guard().await;
16459 crate::subtree_resources::read_process(module.clone()).await
16460 };
16461 stop_resource_fixture(&module, &dir).await;
16462 assert!(
16463 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
16464 "{reading:?}"
16465 );
16466 }
16467
16468 #[test]
16469 fn pid_resource_read_fixture() {
16470 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
16471 let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
16472 let deadline = std::time::Instant::now() + Duration::from_secs(60);
16473 while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
16474 std::thread::sleep(Duration::from_millis(5));
16475 }
16476 std::fs::write(dir.join("module-stopped"), "").unwrap();
16477 }
16478 }
16479
16480 async fn stop_resource_fixture(
16481 module: &crate::supervise::SupervisedModule,
16482 dir: &std::path::Path,
16483 ) {
16484 let stopping = module.stop();
16485 tokio::pin!(stopping);
16486 tokio::select! {
16487 result = &mut stopping => result.unwrap(),
16488 () = async {
16489 while module.state().unwrap() != crate::supervise::ModuleState::Draining {
16490 tokio::time::sleep(Duration::from_millis(5)).await;
16491 }
16492 std::fs::write(dir.join("stop-module"), "").unwrap();
16493 } => stopping.await.unwrap(),
16494 }
16495 #[cfg(windows)]
16496 assert!(
16497 dir.join("module-stopped").exists(),
16498 "the resource fixture must exit cooperatively, not wait for the drain kill"
16499 );
16500 assert_eq!(
16501 module.state().unwrap(),
16502 crate::supervise::ModuleState::Stopped
16503 );
16504 }
16505
16506 #[tokio::test]
16507 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
16508 let registry = Arc::new(Registry::default());
16509 let handle = SupervisorHandle::new();
16510 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16511 .with_handle(handle.clone());
16512 supervisor
16513 .supervise_configured(
16514 crate::ModuleSpec {
16515 script: None,
16516 module_id: "disabled-resource-reading".into(),
16517 program: std::env::current_exe().unwrap(),
16518 args: Vec::new(),
16519 env: Vec::new(),
16520 reserved: false,
16521 reserved_prefixes: Vec::new(),
16522 protocol: ModuleProtocol::None,
16523 overlap: Default::default(),
16524 carry_containment: false,
16525 },
16526 false,
16527 )
16528 .unwrap();
16529 let handler = ControlHandler::new(registry).with_supervisor(handle);
16530 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16531 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16532 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16533 assert!(body["modules"][0].get("tree").is_none());
16534 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
16535 }
16536
16537 #[tokio::test]
16538 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
16539 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
16540 let registry = Arc::new(Registry::default());
16541 let handle = SupervisorHandle::new();
16542 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16543 .with_handle(handle.clone());
16544 let module = supervisor
16545 .spawn(crate::ModuleSpec {
16546 script: None,
16547 carry_containment: false,
16548 module_id: "list-process-priority".into(),
16549 program: std::env::current_exe().unwrap(),
16550 args: vec![
16551 "--exact".into(),
16552 "control::tests::pid_resource_read_fixture".into(),
16553 ],
16554 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16555 .into_iter()
16556 .map(|key| (key.into(), dir.display().to_string()))
16557 .chain(std::iter::once((
16558 "SUBC_SUBTREE_PID_FIXTURE".into(),
16559 "1".into(),
16560 )))
16561 .collect(),
16562 reserved: false,
16563 reserved_prefixes: Vec::new(),
16564 protocol: ModuleProtocol::None,
16565 overlap: Default::default(),
16566 })
16567 .unwrap();
16568 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16569 loop {
16570 let image = module.running_image_agreement().await;
16571 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
16572 break;
16573 }
16574 assert!(
16575 tokio::time::Instant::now() < ready_deadline,
16576 "the fixture process never became observable: {image:?}"
16577 );
16578 tokio::time::sleep(Duration::from_millis(10)).await;
16579 }
16580 let guard = crate::subtree_resources::fault_read_guard().await;
16581 let (target, release, _) = crate::subtree_resources::held();
16582 module.set_subtree_target_for_test(target);
16583 let handler = ControlHandler::new(registry).with_supervisor(handle);
16584 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16585 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16586 drop(release);
16587 drop(guard);
16588 stop_resource_fixture(&module, &dir).await;
16589 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16590 assert_eq!(
16591 body["modules"][0]["resources"]["status"], "measured",
16592 "the optional tree timeout must not discard the process reading: {body}"
16593 );
16594 assert!(body["modules"][0].get("tree").is_none());
16595 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
16596 assert_eq!(
16597 body["modules"][0]["pending_reload"]["image"]["status"], "match",
16598 "the tree timeout must not hide a completed image observation"
16599 );
16600 }
16601
16602 #[cfg(any(target_os = "linux", target_os = "macos"))]
16603 #[tokio::test]
16604 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
16605 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
16606 .with_daemon_provenance(
16607 std::process::id(),
16608 0,
16609 Some(std::env::current_exe().unwrap()),
16610 None,
16611 None,
16612 );
16613 let (probe, release, began) = ExecutableIdentityProbe::held();
16614 handler.daemon_provenance.probe = probe;
16615 let observe = |handler: ControlHandler, corr| {
16616 tokio::spawn(async move {
16617 let (ctx, _) = route_ctx(ConnectionId::new(corr));
16618 let frame =
16619 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
16620 .unwrap();
16621 handler
16622 .handle_supervisor_provenance(&ctx, frame, None)
16623 .await
16624 .unwrap()
16625 })
16626 };
16627 let explicit = observe(handler.clone(), 114);
16628 began.await.unwrap();
16629 let concurrent = observe(handler.clone(), 115);
16630 tokio::time::sleep(Duration::from_millis(300)).await;
16633 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16634 let list = tokio::time::timeout(
16635 Duration::from_secs(1),
16636 handler.handle_supervisor_list(frame),
16637 )
16638 .await;
16639 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
16640 drop(release);
16641 let replies = explicit.await.unwrap();
16642 let concurrent_replies = concurrent.await.unwrap();
16643 assert!(
16644 explicit_was_pending,
16645 "explicit provenance must await the full observation"
16646 );
16647 let list = list
16648 .expect("a concurrent explicit probe must not stall list")
16649 .unwrap();
16650 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
16651 assert_eq!(list_body["op"], "supervisor.list");
16652 for replies in [replies, concurrent_replies] {
16653 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16654 assert_eq!(
16655 body["daemon"]["daemon_observed"]["running_image"]["status"],
16656 "match"
16657 );
16658 }
16659 }
16660
16661 #[tokio::test]
16662 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
16663 let registry = Arc::new(Registry::default());
16664 let handle = SupervisorHandle::new();
16665 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16666 .with_handle(handle.clone());
16667 let module = supervisor
16668 .supervise_configured(
16669 crate::ModuleSpec {
16670 script: None,
16671 carry_containment: false,
16672 module_id: "memory-read".into(),
16673 program: PathBuf::from("disabled-module"),
16674 args: Vec::new(),
16675 env: Vec::new(),
16676 reserved: false,
16677 reserved_prefixes: Vec::new(),
16678 protocol: ModuleProtocol::Subc,
16679 overlap: Default::default(),
16680 },
16681 true,
16682 )
16683 .unwrap();
16684 let _guard = crate::subtree_resources::fault_read_guard().await;
16685 let (target, release, began) = crate::subtree_resources::held();
16686 let stalled_target = target.clone();
16689 let stalled = tokio::task::spawn_blocking(move || {
16690 stalled_target.tree(&subc_os::ProcessTable::default())
16691 });
16692 began.await.unwrap();
16693 module.set_subtree_target_for_test(target);
16694 let handler = ControlHandler::new(registry).with_supervisor(handle);
16695 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
16696 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16697 let replies = handler
16698 .handle_client_control_request(
16699 &ctx,
16700 frame,
16701 ClientControlRequest::SupervisorList {},
16702 &mut None,
16703 )
16704 .await
16705 .unwrap();
16706 assert!(
16707 replies.is_empty(),
16708 "list reply must be detached from the connection reader"
16709 );
16710 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16711 let replies = handler
16712 .handle_client_control_request(
16713 &ctx,
16714 frame,
16715 ClientControlRequest::SupervisorSpawnSnapshot {},
16716 &mut None,
16717 )
16718 .await
16719 .unwrap();
16720 assert_eq!(
16721 replies.len(),
16722 1,
16723 "another request must complete while the subtree read stalls"
16724 );
16725 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16726 .await
16727 .unwrap()
16728 .unwrap();
16729 release.send(()).unwrap();
16730 drop(release);
16731 stalled.await.unwrap();
16732 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
16733 assert!(
16734 body["modules"][0].get("tree").is_none(),
16735 "an unknown tree reading must be omitted after the bounded read expires"
16736 );
16737 handler
16741 .resource_tracker
16742 .read(handler.supervisor.list(), false, "released_tree_fixture")
16743 .await;
16744 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16745 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16746 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16747 assert_eq!(
16748 body["modules"][0]["tree"]["source"], "process_tree",
16749 "the next request must export the completed refresh"
16750 );
16751 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
16752 }
16753
16754 #[tokio::test]
16755 async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
16756 use subc_control::{
16757 AgentCommandResources, ChildMemoryKind, ChildResourceReading,
16758 ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
16759 TreeReading, TreeSource,
16760 };
16761 let registry = Arc::new(Registry::default());
16762 let handle = SupervisorHandle::new();
16763 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16764 .with_handle(handle.clone());
16765 supervisor
16766 .supervise_configured(
16767 crate::ModuleSpec {
16768 script: None,
16769 carry_containment: false,
16770 module_id: "budget-worker".into(),
16771 program: PathBuf::from("disabled-module"),
16772 args: Vec::new(),
16773 env: Vec::new(),
16774 reserved: false,
16775 reserved_prefixes: Vec::new(),
16776 protocol: ModuleProtocol::Subc,
16777 overlap: Default::default(),
16778 },
16779 true,
16780 )
16781 .unwrap();
16782 let handler = ControlHandler::new(registry).with_supervisor(handle);
16783 let now = u64::try_from(
16784 std::time::SystemTime::now()
16785 .duration_since(std::time::UNIX_EPOCH)
16786 .unwrap()
16787 .as_millis(),
16788 )
16789 .unwrap();
16790 let measured = now - 1_000;
16791 let hour = CpuLastHour::new(100, 60_000);
16792 let daemon = DaemonResources::new(7, 1, "test")
16793 .with_measured_at_ms(Some(measured))
16794 .with_resources(Some(ChildResourceUsage::Measured(
16795 ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
16796 )))
16797 .with_cpu_last_hour(Some(hour.clone()));
16798 let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
16799 .with_measured_at_ms(Some(measured))
16800 .with_cpu_last_hour(Some(hour.clone()));
16801 let tree = TreeReading::new(
16802 150,
16803 50,
16804 ChildMemoryKind::ResidentSet,
16805 1,
16806 TreeSource::ProcessTree,
16807 )
16808 .with_measured_at_ms(Some(measured));
16809 handler.resource_tracker.seed_committed_for_test(
16810 &handler.supervisor.list(),
16811 crate::resource_totals::Readings {
16812 modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
16813 daemon: Some(daemon),
16814 agent_commands: Some(vec![agents]),
16815 ..Default::default()
16816 },
16817 now,
16818 );
16819 list_diagnostics::scope(async {
16820 tokio::time::sleep(Duration::from_millis(110)).await;
16821 assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
16822 let request =
16823 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16824 let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
16825 let ClientControlResponse::SupervisorList {
16826 modules,
16827 daemon,
16828 agent_commands,
16829 ..
16830 } = serde_json::from_slice(&replies[0].body).unwrap()
16831 else {
16832 panic!("expected a list response")
16833 };
16834 assert!(matches!(
16835 modules[0].resources,
16836 Some(ChildResourceUsage::Unavailable {
16837 reason: ChildResourceUnavailableReason::Unreadable
16838 })
16839 ));
16840 assert_eq!(
16841 (
16842 modules[0].tree.as_ref().unwrap().cpu_ms,
16843 modules[0].tree.as_ref().unwrap().measured_at_ms
16844 ),
16845 (150, Some(measured))
16846 );
16847 let daemon = daemon.unwrap();
16848 let agents = agent_commands.unwrap();
16849 assert_eq!(daemon.measured_at_ms, Some(measured));
16850 assert_eq!(
16851 (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
16852 (1, 50, Some(measured))
16853 );
16854 for hour in [
16855 modules[0].cpu_last_hour.as_ref(),
16856 daemon.cpu_last_hour.as_ref(),
16857 agents[0].cpu_last_hour.as_ref(),
16858 ] {
16859 let hour = hour.unwrap();
16860 assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
16861 }
16862 })
16863 .await;
16864 }
16865
16866 #[tokio::test]
16867 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
16868 let registry = Arc::new(Registry::default());
16869 let handle = SupervisorHandle::new();
16870 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16871 .with_handle(handle.clone());
16872 supervisor
16873 .supervise_configured(
16874 crate::ModuleSpec {
16875 script: None,
16876 carry_containment: false,
16877 module_id: "drift".into(),
16878 program: PathBuf::from("disabled-module"),
16879 args: Vec::new(),
16880 env: Vec::new(),
16881 reserved: false,
16882 reserved_prefixes: Vec::new(),
16883 protocol: ModuleProtocol::Subc,
16884 overlap: Default::default(),
16885 },
16886 false,
16887 )
16888 .unwrap();
16889 let handler = ControlHandler::new(registry).with_supervisor(handle);
16890 for configured in [None, Some(false), Some(true)] {
16891 handler.capability_evaluator.configure(
16892 configured.map(|enabled| ("drift".to_string(), enabled)),
16893 BTreeMap::new(),
16894 );
16895 let frame =
16896 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16897 let replies = handler
16898 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
16899 .await
16900 .unwrap();
16901 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16902 assert!(body.get("configured_enabled").is_none());
16903 assert_eq!(body["op"], "supervisor.ack");
16904 let frame =
16905 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16906 let replies = handler.handle_supervisor_list(frame).await.unwrap();
16907 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16908 assert_eq!(
16909 body["modules"][0].get("configured_enabled"),
16910 configured.map(Value::Bool).as_ref()
16911 );
16912 assert!(body["modules"][0].get("tree").is_none());
16913 assert!(body["modules"][0].get("subtree").is_none());
16914 }
16915 }
16916
16917 #[tokio::test(flavor = "current_thread")]
16918 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
16919 let handler = ControlHandler::new(Arc::new(Registry::default()));
16920 let capture = EventCapture::default();
16921 let _subscriber =
16922 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16923 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
16924 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16925
16926 let _ = handler
16927 .handle_client_control_request(
16928 &ctx,
16929 frame,
16930 ClientControlRequest::SupervisorSetEnabled {
16931 module_id: "broca".to_string(),
16932 enabled: false,
16933 },
16934 &mut None,
16935 )
16936 .await
16937 .unwrap();
16938
16939 let events = capture
16940 .events()
16941 .into_iter()
16942 .filter(|event| {
16943 event.target == "control"
16944 && event.fields.get("message").map(String::as_str)
16945 == Some("supervisor request received")
16946 })
16947 .collect::<Vec<_>>();
16948 assert_eq!(events.len(), 1, "one supervisor request log line");
16949 let event = &events[0];
16950 assert_eq!(event.level, tracing::Level::INFO);
16951 assert_eq!(
16952 event.fields.get("op"),
16953 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
16954 );
16955 assert_eq!(
16956 event.fields.get("module_id"),
16957 Some(&"\"broca\"".to_string())
16958 );
16959 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
16960 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
16961 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
16962 }
16963
16964 #[tokio::test(flavor = "current_thread")]
16965 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
16966 let registry = Arc::new(Registry::default());
16967 let module_connection = ConnectionId::new(303);
16968 registry
16969 .register_with_control_ops(
16970 manifest("aft", PROTOCOL_VERSION),
16971 PROTOCOL_VERSION,
16972 module_connection,
16973 module_baseline_control_ops(),
16974 )
16975 .unwrap();
16976 let handler = ControlHandler::new(registry);
16977 let capture = EventCapture::default();
16978 let _subscriber =
16979 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16980 let (ctx, _rx) = route_ctx(module_connection);
16981 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16982
16983 let _ = handler
16984 .handle_client_control_request(
16985 &ctx,
16986 frame,
16987 ClientControlRequest::SupervisorSetEnabled {
16988 module_id: "broca".to_string(),
16989 enabled: false,
16990 },
16991 &mut None,
16992 )
16993 .await
16994 .unwrap();
16995
16996 let callers = capture
16997 .events()
16998 .into_iter()
16999 .filter(|event| {
17000 event.fields.get("message").map(String::as_str)
17001 == Some("supervisor request received")
17002 })
17003 .map(|event| event.fields.get("caller").cloned())
17004 .collect::<Vec<_>>();
17005 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
17006 }
17007
17008 #[tokio::test(flavor = "current_thread")]
17009 async fn supervisor_list_does_not_log_request_received() {
17010 let handler = ControlHandler::new(Arc::new(Registry::default()));
17011 let capture = EventCapture::default();
17012 let _subscriber =
17013 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17014 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
17015 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
17016
17017 handler
17018 .handle_client_control_request(
17019 &ctx,
17020 frame,
17021 ClientControlRequest::SupervisorList {},
17022 &mut None,
17023 )
17024 .await
17025 .unwrap();
17026
17027 assert!(capture.events().into_iter().all(|event| {
17028 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17029 }));
17030 }
17031
17032 #[tokio::test(flavor = "current_thread")]
17033 async fn supervisor_rescan_preview_does_not_log_request_received() {
17034 let handler = ControlHandler::new(Arc::new(Registry::default()));
17035 let capture = EventCapture::default();
17036 let _subscriber =
17037 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17038 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
17039 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
17040
17041 handler
17042 .handle_client_control_request(
17043 &ctx,
17044 frame,
17045 ClientControlRequest::SupervisorRescan { preview: true },
17046 &mut None,
17047 )
17048 .await
17049 .unwrap();
17050
17051 assert!(capture.events().into_iter().all(|event| {
17052 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17053 }));
17054 }
17055
17056 struct ApproveOperator;
17057 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
17058 fn prompt(
17059 &self,
17060 _: &str,
17061 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
17062 ) -> crate::operator_confirm::ProviderResult {
17063 crate::operator_confirm::ProviderResult::Approved
17064 }
17065 }
17066
17067 #[tokio::test]
17068 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
17069 assert_eq!(
17070 ControlHandler::default().privileged_operator_prompt,
17071 PrivilegedOperatorPrompt::Log
17072 );
17073 for (initial, changed, running) in [
17074 ("log", "enforce", PrivilegedOperatorPrompt::Log),
17075 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
17076 ] {
17077 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
17078 let path = dir.join("subc.jsonc");
17079 std::fs::write(
17080 &path,
17081 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
17082 )
17083 .unwrap();
17084 let config = crate::daemon_config::load(&path).unwrap().unwrap();
17085 let registry = Arc::new(Registry::default());
17086 let handle = SupervisorHandle::new();
17087 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
17088 .with_handle(handle.clone());
17089 let handler = ControlHandler::new(registry)
17090 .with_supervisor(handle)
17091 .with_privileged_operator_prompt(config.privileged_operator_prompt)
17092 .with_supervisor_rescan(supervisor, &path, config.port);
17093 handler.forwarding.operator_confirms().configure(
17094 Arc::new(ApproveOperator),
17095 crate::operator_confirm::OperatorLimits::default(),
17096 );
17097 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
17098
17099 for (disk_mode, needs_restart) in [
17100 (Some(initial), false),
17101 (Some(changed), true),
17102 (Some(changed), true),
17103 (None, running == PrivilegedOperatorPrompt::Enforce),
17104 (Some(initial), false),
17105 ] {
17106 let mut doc = json!({"version":1});
17107 if let Some(mode) = disk_mode {
17108 doc["privileged_operator_prompt"] = json!(mode);
17109 }
17110 std::fs::write(&path, doc.to_string()).unwrap();
17111 for preview in [true, false] {
17112 let request = ClientControlRequest::SupervisorRescan { preview };
17113 let frame = Frame::build(
17114 FrameType::Request,
17115 control_flags(),
17116 0,
17117 0,
17118 1,
17119 serde_json::to_vec(&request).unwrap(),
17120 )
17121 .unwrap();
17122 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17123 assert_eq!(frames.len(), 1);
17124 let ClientControlResponse::SupervisorRescan { result } =
17125 serde_json::from_slice(&frames[0].body).unwrap()
17126 else {
17127 panic!("expected a rescan response");
17128 };
17129 let expected = if needs_restart {
17130 vec!["privileged_operator_prompt"]
17131 } else {
17132 Vec::new()
17133 };
17134 assert_eq!(result.restart_required, expected);
17135 assert_eq!(handler.privileged_operator_prompt, running);
17136 assert_eq!(
17137 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
17138 running
17139 );
17140 }
17141 }
17142 }
17143 }
17144
17145 #[tokio::test]
17146 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
17147 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
17148 let path = dir.join("subc.jsonc");
17149 let registry = Arc::new(Registry::default());
17150 let handle = SupervisorHandle::new();
17151 let supervisor =
17152 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
17153 let handler = ControlHandler::new(registry)
17154 .with_supervisor(handle)
17155 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
17156 .with_supervisor_rescan(supervisor, &path, None);
17157 handler.forwarding.operator_confirms().configure(
17158 Arc::new(ApproveOperator),
17159 crate::operator_confirm::OperatorLimits::default(),
17160 );
17161 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
17162 for preview in [true, false] {
17163 std::fs::write(
17164 &path,
17165 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
17166 )
17167 .unwrap();
17168 let request = ClientControlRequest::SupervisorRescan { preview };
17169 let frame = Frame::build(
17170 FrameType::Request,
17171 control_flags(),
17172 0,
17173 0,
17174 1,
17175 serde_json::to_vec(&request).unwrap(),
17176 )
17177 .unwrap();
17178 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17179 assert_eq!(frames.len(), 1);
17180 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
17181 assert_eq!(
17182 handler.privileged_operator_prompt,
17183 PrivilegedOperatorPrompt::Enforce
17184 );
17185 }
17186 }
17187
17188 #[tokio::test]
17193 async fn route_open_refusal_names_the_check_that_refused() {
17194 let handler = ControlHandler::new(Arc::new(Registry::default()));
17195 let capture = EventCapture::default();
17196 let _subscriber =
17197 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17198 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17199 let response = handler
17200 .handle_control_frame(
17201 &ctx,
17202 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
17203 )
17204 .await
17205 .unwrap();
17206
17207 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17208 let event = capture
17209 .events()
17210 .into_iter()
17211 .find(|event| {
17212 event.target == "control"
17213 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17214 })
17215 .expect("route.open refusal event");
17216 assert_eq!(
17217 event.fields.get("reason"),
17218 Some(&"\"not_registered\"".to_string())
17219 );
17220 }
17221
17222 #[tokio::test]
17223 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
17224 let registry = Arc::new(Registry::default());
17225 let supervisor_handle = SupervisorHandle::new();
17226 let supervisor =
17227 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17228 .with_handle(supervisor_handle.clone())
17229 .with_connection_file_path(std::env::temp_dir().join(format!(
17230 "subc-route-open-refusal-info-{}",
17231 std::process::id()
17232 )));
17233 let module = supervisor
17234 .supervise_configured(
17235 ModuleSpec {
17236 script: None,
17237 carry_containment: false,
17238 module_id: "warming".to_string(),
17239 program: fake_aft_stub_path(),
17240 args: Vec::new(),
17241 env: Vec::new(),
17242 reserved: false,
17243 reserved_prefixes: Vec::new(),
17244 protocol: ModuleProtocol::Subc,
17245 overlap: Default::default(),
17246 },
17247 true,
17248 )
17249 .unwrap();
17250 assert_eq!(module.state().unwrap(), ModuleState::Running);
17251
17252 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17253 assert!(handler
17254 .counters()
17255 .snapshot()
17256 .get("route_open_refused_by_code")
17257 .is_none());
17258 let capture = EventCapture::default();
17259 let _subscriber =
17260 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17261 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
17262 let response = handler
17263 .handle_control_frame(
17264 &ctx,
17265 route_open_frame(394, "warming", unique_project_root("refusal-info")),
17266 )
17267 .await
17268 .unwrap();
17269 module.stop().await.unwrap();
17270
17271 assert_eq!(parse_error(&response[0])["code"], "module_warming");
17272 let event = capture
17273 .events()
17274 .into_iter()
17275 .find(|event| {
17276 event.target == "control"
17277 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
17278 })
17279 .expect("route.open refusal event");
17280 assert_eq!(
17281 event.fields.get("module_id"),
17282 Some(&"\"warming\"".to_string())
17283 );
17284 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
17285 assert_eq!(
17286 event.fields.get("reason"),
17287 Some(&"\"supervised_not_registered\"".to_string())
17288 );
17289 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
17290 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
17291 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
17292 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
17293 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
17294 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
17295 assert_eq!(
17296 handler.counters().snapshot()["route_open_refused_by_code"],
17297 json!({ "module_warming": 1 })
17298 );
17299 }
17300
17301 const OUTAGE_START: &str = "route.open refusing module: not serving";
17302 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
17303
17304 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
17305 capture
17306 .events()
17307 .into_iter()
17308 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
17309 .collect()
17310 }
17311
17312 fn supervise_stub(
17313 registry: &Arc<Registry>,
17314 module_id: &str,
17315 enabled: bool,
17316 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
17317 let supervisor_handle = SupervisorHandle::new();
17318 let supervisor =
17319 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
17320 .with_handle(supervisor_handle.clone())
17321 .with_connection_file_path(std::env::temp_dir().join(format!(
17322 "subc-route-outage-{module_id}-{}",
17323 std::process::id()
17324 )));
17325 let module = supervisor
17326 .supervise_configured(
17327 ModuleSpec {
17328 script: None,
17329 carry_containment: false,
17330 module_id: module_id.to_string(),
17331 program: fake_aft_stub_path(),
17332 args: Vec::new(),
17333 env: Vec::new(),
17334 reserved: false,
17335 reserved_prefixes: Vec::new(),
17336 protocol: ModuleProtocol::Subc,
17337 overlap: Default::default(),
17338 },
17339 enabled,
17340 )
17341 .unwrap();
17342 (supervisor_handle, module)
17343 }
17344
17345 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
17346 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
17347 module_id: module_id.to_string(),
17348 drain_timeout_ms: Some(50),
17349 })
17350 .unwrap();
17351 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
17352 }
17353
17354 #[test]
17358 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
17359 let registry = Arc::new(Registry::default());
17360 let forwarding = Arc::new(ForwardingTable::default());
17361 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17362 let second = ControlHandler::with_forwarding(registry, forwarding);
17363 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
17364 }
17365
17366 #[tokio::test(flavor = "current_thread")]
17370 async fn route_open_unknown_module_refusals_add_no_outage_state() {
17371 let handler = ControlHandler::new(Arc::new(Registry::default()));
17372 let capture = EventCapture::default();
17373 let _subscriber =
17374 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17375 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
17376 for corr in 0..8 {
17377 let response = handler
17378 .handle_control_frame(
17379 &ctx,
17380 route_open_frame(
17381 380 + corr,
17382 &format!("nobody-{corr}"),
17383 unique_project_root("outage-unknown"),
17384 ),
17385 )
17386 .await
17387 .unwrap();
17388 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17389 }
17390
17391 assert_eq!(handler.route_outages.tracked_module_count(), 0);
17392 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
17393 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
17394 }
17395
17396 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17401 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
17402 let registry = Arc::new(Registry::default());
17403 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
17404 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17405 let capture = EventCapture::default();
17406 let _subscriber =
17407 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17408 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
17409 handler.route_outages.record_accepted("outage-restart");
17412
17413 let response = handler
17414 .handle_control_frame(
17415 &ctx,
17416 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
17417 )
17418 .await
17419 .unwrap();
17420 assert_eq!(response[0].header.ty, FrameType::Error);
17421 let starts = outage_lines(&capture, OUTAGE_START);
17422 assert_eq!(starts.len(), 1, "{starts:?}");
17423 assert_eq!(starts[0].level, tracing::Level::WARN);
17424 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
17425 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
17426 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
17427 handler.route_outages.record_accepted("outage-restart");
17428 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
17429
17430 let restart = handler
17431 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
17432 .await
17433 .unwrap();
17434 assert_eq!(
17435 restart[0].header.ty,
17436 FrameType::Response,
17437 "{:?}",
17438 parse_error(&restart[0])
17439 );
17440 handler
17441 .handle_control_frame(
17442 &ctx,
17443 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
17444 )
17445 .await
17446 .unwrap();
17447 module.stop().await.unwrap();
17448
17449 let starts = outage_lines(&capture, OUTAGE_START);
17450 assert_eq!(starts.len(), 2, "{starts:?}");
17451 assert_eq!(starts[1].level, tracing::Level::INFO);
17452 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
17453 }
17454
17455 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17459 async fn failed_operator_restart_leaves_no_operator_mark() {
17460 let registry = Arc::new(Registry::default());
17461 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
17462 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17463 let capture = EventCapture::default();
17464 let _subscriber =
17465 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17466 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
17467 handler.route_outages.record_accepted("outage-disabled");
17468
17469 let restart = handler
17470 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
17471 .await
17472 .unwrap();
17473 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
17474 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
17475
17476 handler
17477 .handle_control_frame(
17478 &ctx,
17479 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
17480 )
17481 .await
17482 .unwrap();
17483 let starts = outage_lines(&capture, OUTAGE_START);
17484 assert_eq!(starts.len(), 1, "{starts:?}");
17485 assert_eq!(starts[0].level, tracing::Level::WARN);
17486 }
17487
17488 #[tokio::test(flavor = "current_thread")]
17489 async fn route_open_unknown_module_escapes_target_module_id() {
17490 let handler = ControlHandler::new(Arc::new(Registry::default()));
17491 let capture = EventCapture::default();
17492 let _subscriber =
17493 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17494 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
17495 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17496 let response = handler
17497 .handle_control_frame(
17498 &ctx,
17499 route_open_frame(
17500 395,
17501 hostile_module_id,
17502 unique_project_root("hostile-target-module-id"),
17503 ),
17504 )
17505 .await
17506 .unwrap();
17507
17508 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17509 let event = capture
17510 .events()
17511 .into_iter()
17512 .find(|event| {
17513 event.target == "control"
17514 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17515 })
17516 .expect("route.open unknown-module refusal event");
17517 let logged = event.fields.get("module_id").expect("module_id field");
17518 assert!(!logged.bytes().any(|byte| byte < 0x20));
17519 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17520 }
17521
17522 #[tokio::test(flavor = "current_thread")]
17525 async fn route_open_refusal_carries_daemon_origin() {
17526 let handler = ControlHandler::new(Arc::new(Registry::default()));
17527 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17528 let response = handler
17529 .handle_control_frame(
17530 &ctx,
17531 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
17532 )
17533 .await
17534 .unwrap();
17535
17536 assert_eq!(response.len(), 1);
17537 assert_eq!(response[0].header.ty, FrameType::Error);
17538 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17539 assert!(
17540 response[0].header.flags.is_daemon_origin(),
17541 "route.open refusal must carry daemon origin: {:?}",
17542 response[0].header.flags
17543 );
17544 }
17545
17546 #[tokio::test(flavor = "current_thread")]
17550 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
17551 let registry = Arc::new(Registry::default());
17552 let forwarding = Arc::new(ForwardingTable::default());
17553 let handler =
17554 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17555 let module_connection = ConnectionId::new(95);
17556 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17557 hello_via_sink(
17558 &handler,
17559 &module_ctx,
17560 &mut module_rx,
17561 hello_frame("aft", PROTOCOL_VERSION, 395),
17562 )
17563 .await;
17564 forwarding
17565 .begin_module_drain("aft", RouteCloseReason::Restart)
17566 .unwrap()
17567 .expect("the module endpoint drains");
17568
17569 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
17570 let response = handler
17571 .handle_control_frame(
17572 &client_ctx,
17573 route_open_frame(396, "aft", unique_project_root("drain-origin")),
17574 )
17575 .await
17576 .unwrap();
17577
17578 assert_eq!(response.len(), 1);
17579 assert_eq!(response[0].header.ty, FrameType::Error);
17580 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17581 assert!(
17582 response[0].header.flags.is_daemon_origin(),
17583 "drain module_reloading must carry daemon origin: {:?}",
17584 response[0].header.flags
17585 );
17586 }
17587
17588 #[tokio::test(flavor = "current_thread")]
17592 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
17593 let registry = Arc::new(Registry::default());
17594 let forwarding = Arc::new(ForwardingTable::default());
17595 let handler =
17596 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17597 let module_connection = ConnectionId::new(95);
17598 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17599 hello_via_sink(
17600 &handler,
17601 &module_ctx,
17602 &mut module_rx,
17603 hello_frame("aft", PROTOCOL_VERSION, 395),
17604 )
17605 .await;
17606
17607 let client_connection = ConnectionId::new(96);
17608 let (client_ctx, _client_rx) = route_ctx(client_connection);
17609 let (route_task, bind) = relay_route_open(
17610 &handler,
17611 client_connection,
17612 &client_ctx.egress,
17613 &mut module_rx,
17614 396,
17615 "aft",
17616 "spoofed-reloading",
17617 )
17618 .await;
17619 let rejection = Frame::build(
17622 FrameType::Error,
17623 crate::router::daemon_error_flags(),
17624 0,
17625 0,
17626 bind.header.corr,
17627 serde_json::to_vec(&ErrorBody::new(
17628 "module_reloading",
17629 "module refused route.bind",
17630 ))
17631 .unwrap(),
17632 )
17633 .unwrap();
17634 handler
17635 .handle_control_frame(&module_ctx, rejection)
17636 .await
17637 .unwrap();
17638
17639 let response = route_task.await.unwrap();
17640 assert_eq!(response.len(), 1);
17641 assert_eq!(response[0].header.ty, FrameType::Error);
17642 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17643 assert!(
17644 !response[0].header.flags.is_daemon_origin(),
17645 "a module's own rejection must not carry daemon origin: {:?}",
17646 response[0].header.flags
17647 );
17648 }
17649
17650 #[tokio::test(flavor = "current_thread")]
17651 async fn route_open_module_rejection_uses_daemon_counter_key() {
17652 let registry = Arc::new(Registry::default());
17653 let forwarding = Arc::new(ForwardingTable::default());
17654 let handler =
17655 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17656 let module_connection = ConnectionId::new(95);
17657 let (module_ctx, mut module_rx) = route_ctx(module_connection);
17658 hello_via_sink(
17659 &handler,
17660 &module_ctx,
17661 &mut module_rx,
17662 hello_frame("aft", PROTOCOL_VERSION, 395),
17663 )
17664 .await;
17665
17666 let client_connection = ConnectionId::new(96);
17667 let (client_ctx, _client_rx) = route_ctx(client_connection);
17668 let capture = EventCapture::default();
17669 static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
17676 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
17677 let _subscriber =
17678 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17679 let (route_task, bind) = relay_route_open(
17680 &handler,
17681 client_connection,
17682 &client_ctx.egress,
17683 &mut module_rx,
17684 396,
17685 "aft",
17686 "hostile-module-code",
17687 )
17688 .await;
17689 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
17690 let rejection = Frame::build(
17691 FrameType::Error,
17692 control_flags(),
17693 0,
17694 0,
17695 bind.header.corr,
17696 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
17697 )
17698 .unwrap();
17699 handler
17700 .handle_control_frame(&module_ctx, rejection)
17701 .await
17702 .unwrap();
17703
17704 let response = route_task.await.unwrap();
17705 assert_eq!(parse_error(&response[0])["code"], hostile_code);
17706 let counters = handler.counters().snapshot();
17707 assert_eq!(
17708 counters["route_open_refused_by_code"],
17709 json!({ "module_rejected": 1 })
17710 );
17711 assert!(counters["route_open_refused_by_code"]
17712 .get(hostile_code)
17713 .is_none());
17714
17715 let event = capture
17716 .events()
17717 .into_iter()
17718 .find(|event| {
17719 event.target == "control"
17720 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
17721 })
17722 .expect("route.open module-rejection refusal event");
17723 let logged = event.fields.get("module_code").expect("module_code field");
17724 assert!(!logged.bytes().any(|byte| byte < 0x20));
17725 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17726 }
17727
17728 #[tokio::test]
17729 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
17730 let registry = Arc::new(Registry::default());
17731 let supervisor_handle = SupervisorHandle::new();
17732 let missing_program = std::env::temp_dir().join(format!(
17733 "subc-route-open-missing-program-{}",
17734 std::process::id()
17735 ));
17736 let supervisor =
17737 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
17738 .with_handle(supervisor_handle.clone());
17739 let module = supervisor
17740 .supervise_configured(
17741 ModuleSpec {
17742 script: None,
17743 carry_containment: false,
17744 module_id: "failed".to_string(),
17745 program: missing_program,
17746 args: Vec::new(),
17747 env: Vec::new(),
17748 reserved: false,
17749 reserved_prefixes: Vec::new(),
17750 protocol: ModuleProtocol::Subc,
17751 overlap: Default::default(),
17752 },
17753 true,
17754 )
17755 .unwrap();
17756 assert_eq!(module.state().unwrap(), ModuleState::Failed);
17757
17758 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17759 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
17760 let response = handler
17761 .handle_control_frame(
17762 &ctx,
17763 route_open_frame(305, "failed", unique_project_root("failed")),
17764 )
17765 .await
17766 .unwrap();
17767
17768 assert_eq!(response[0].header.ty, FrameType::Error);
17769 let error = parse_error(&response[0]);
17770 assert_eq!(error["code"], "target_unavailable");
17771 assert!(error["message"]
17772 .as_str()
17773 .unwrap()
17774 .contains("state=failed, enabled=true, live=false"));
17775 }
17776
17777 #[tokio::test]
17778 async fn route_open_role_mismatch_remains_target_unavailable() {
17779 let registry = Arc::new(Registry::default());
17780 let handler = ControlHandler::new(Arc::clone(®istry));
17781 handler
17782 .handle_control(
17783 ConnectionId::new(41),
17784 non_routable_hello_frame_with_control_ops("health-only", 306, None),
17785 )
17786 .unwrap();
17787
17788 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
17789 let response = handler
17790 .handle_control_frame(
17791 &ctx,
17792 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
17793 )
17794 .await
17795 .unwrap();
17796
17797 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17798 assert!(parse_error(&response[0])["message"]
17799 .as_str()
17800 .unwrap()
17801 .contains("does not provide the requested target"));
17802 }
17803
17804 #[tokio::test]
17805 async fn route_open_inactive_registration_remains_target_unavailable() {
17806 let registry = Arc::new(Registry::default());
17807 let handler = ControlHandler::new(Arc::clone(®istry));
17808 handler
17809 .handle_control(
17810 ConnectionId::new(43),
17811 hello_frame("inactive", PROTOCOL_VERSION, 308),
17812 )
17813 .unwrap();
17814 assert!(registry
17815 .set_module_state_for_test("inactive", ChannelState::Closed)
17816 .unwrap());
17817
17818 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
17819 let response = handler
17820 .handle_control_frame(
17821 &ctx,
17822 route_open_frame(309, "inactive", unique_project_root("inactive")),
17823 )
17824 .await
17825 .unwrap();
17826
17827 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17828 assert!(parse_error(&response[0])["message"]
17829 .as_str()
17830 .unwrap()
17831 .contains("is not active"));
17832 }
17833
17834 #[tokio::test]
17835 async fn late_health_reply_is_recorded_through_the_module_response_path() {
17836 let registry = Arc::new(Registry::default());
17837 let forwarding = Arc::new(ForwardingTable::default());
17838 let supervisor_handle = SupervisorHandle::new();
17839 let supervisor =
17840 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
17841 .with_forwarding(Arc::clone(&forwarding))
17842 .with_handle(supervisor_handle.clone());
17843 let module = supervisor
17844 .supervise_configured(
17845 crate::ModuleSpec {
17846 script: None,
17847 carry_containment: false,
17848 module_id: "late-health-response".to_string(),
17849 program: PathBuf::from("disabled-module"),
17850 args: Vec::new(),
17851 env: Vec::new(),
17852 reserved: false,
17853 reserved_prefixes: Vec::new(),
17854 protocol: ModuleProtocol::Subc,
17855 overlap: Default::default(),
17856 },
17857 false,
17858 )
17859 .unwrap();
17860 let handler =
17861 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17862 .with_supervisor(supervisor_handle);
17863 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
17864 handler
17865 .handle_control_frame(
17866 &module_ctx,
17867 hello_frame_with_control_ops(
17868 "late-health-response",
17869 PROTOCOL_VERSION,
17870 7,
17871 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17872 ),
17873 )
17874 .await
17875 .unwrap();
17876 let probe_started_at = Instant::now() - Duration::from_millis(80);
17877 let pending = forwarding
17878 .begin_health_probe_rpc_for(
17879 "late-health-response",
17880 MODULE_CONTROL_OP_HEALTH_CHECK,
17881 probe_started_at,
17882 Instant::now() - Duration::from_millis(1),
17883 )
17884 .unwrap();
17885 assert!(forwarding
17886 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
17887 .unwrap());
17888
17889 let responses = handler
17890 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
17891 .await
17892 .unwrap();
17893
17894 assert!(responses.is_empty());
17895 let health = module.status().unwrap().health;
17896 assert_eq!(health.late_answer_count, 1);
17897 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
17898 }
17899
17900 #[tokio::test]
17901 async fn health_probe_timeout_and_module_death_are_typed() {
17902 let registry = Arc::new(Registry::default());
17903 let forwarding = Arc::new(ForwardingTable::default());
17904 let handler =
17905 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17906 .with_health_probe_timeout(Duration::from_millis(50));
17907 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
17908 hello_via_sink(
17909 &handler,
17910 &module_ctx,
17911 &mut module_rx,
17912 hello_frame_with_control_ops(
17913 "aft",
17914 PROTOCOL_VERSION,
17915 7,
17916 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17917 ),
17918 )
17919 .await;
17920
17921 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
17922 let responses = handler
17923 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
17924 .await
17925 .unwrap();
17926 assert_eq!(responses[0].header.ty, FrameType::Error);
17927 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
17928 let _ = module_rx.try_recv();
17929
17930 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
17931 let health_handler = handler.clone();
17932 let death_task = tokio::spawn(async move {
17933 health_handler
17934 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
17935 .await
17936 .unwrap()
17937 });
17938 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
17939 .await
17940 .unwrap()
17941 .unwrap();
17942 handler
17943 .cleanup_connection(module_ctx.connection_id)
17944 .unwrap();
17945 let responses = death_task.await.unwrap();
17946 assert_eq!(responses[0].header.ty, FrameType::Error);
17947 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
17948 }
17949
17950 #[test]
17951 fn hello_requires_exact_protocol_version() {
17952 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
17953 let registry = Arc::new(Registry::default());
17954 let handler = ControlHandler::new(Arc::clone(®istry));
17955 let responses = handler
17956 .handle_control(
17957 ConnectionId::new(connection),
17958 hello_frame("aft", offered, 9),
17959 )
17960 .unwrap();
17961
17962 assert_eq!(responses.len(), 1);
17963 assert_eq!(responses[0].header.ty, FrameType::Error);
17964 let error = parse_error(&responses[0]);
17965 assert_eq!(error["code"], "version_unsupported");
17966 assert!(registry.get_module("aft").unwrap().is_none());
17967 assert_eq!(registry.active_registration_count().unwrap(), 0);
17968 }
17969 }
17970
17971 #[test]
17972 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
17973 let registry = Arc::new(Registry::default());
17974 let forwarding = Arc::new(ForwardingTable::default());
17975 let handler =
17976 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17977 let module_connection = ConnectionId::new(301);
17978 let registration = registry
17979 .register_with_control_ops(
17980 manifest("aft-push", PROTOCOL_VERSION),
17981 PROTOCOL_VERSION,
17982 module_connection,
17983 module_baseline_control_ops(),
17984 )
17985 .unwrap();
17986 let (module_tx, _module_rx) = mpsc::channel(8);
17987 let endpoint = forwarding
17988 .register_module_connection(
17989 module_connection,
17990 "aft-push".to_string(),
17991 PROTOCOL_VERSION,
17992 manifest_concurrency(®istration.manifest),
17993 FrameSink::new(module_tx),
17994 )
17995 .unwrap();
17996
17997 let unknown = Frame::build(
17999 FrameType::Push,
18000 control_flags(),
18001 0,
18002 0,
18003 5,
18004 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
18005 )
18006 .unwrap();
18007 let out = handler.handle_status_update(endpoint, unknown).unwrap();
18008 assert!(
18009 out.is_empty(),
18010 "unknown push op must be ignored, got {out:?}"
18011 );
18012
18013 let malformed = Frame::build(
18015 FrameType::Push,
18016 control_flags(),
18017 0,
18018 0,
18019 6,
18020 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
18021 )
18022 .unwrap();
18023 let out = handler.handle_status_update(endpoint, malformed).unwrap();
18024 assert_eq!(out.len(), 1);
18025 assert_eq!(out[0].header.ty, FrameType::Error);
18026 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
18027 }
18028
18029 #[test]
18030 fn hello_rejected_when_connection_already_owns_client_routes() {
18031 let registry = Arc::new(Registry::default());
18032 let forwarding = Arc::new(ForwardingTable::default());
18033 let handler =
18034 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18035 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
18037 let client_connection = ConnectionId::new(202);
18038
18039 let responses = handler
18042 .handle_control(
18043 client_connection,
18044 hello_frame("aft-second", PROTOCOL_VERSION, 9),
18045 )
18046 .unwrap();
18047 assert_eq!(responses[0].header.ty, FrameType::Error);
18048 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
18049 assert!(registry.get_module("aft-second").unwrap().is_none());
18050 }
18051
18052 #[tokio::test]
18053 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
18054 let registry = Arc::new(Registry::default());
18055 let forwarding = Arc::new(ForwardingTable::default());
18056 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
18057 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
18058 hello_via_sink(
18059 &handler,
18060 &module_ctx,
18061 &mut module_rx,
18062 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
18063 )
18064 .await;
18065 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
18066 let pending = forwarding
18067 .begin_route_bind_relay_for_test(
18068 client_ctx.connection_id,
18069 client_ctx.egress.clone(),
18070 2,
18071 "alpha",
18072 )
18073 .unwrap();
18074 forwarding
18075 .complete_pending_relay(
18076 module_ctx.connection_id,
18077 pending.corr,
18078 RouteBindRelayOutcome::Accepted,
18079 )
18080 .unwrap();
18081 client_rx.try_recv().unwrap();
18082 for module_id in ["beta", "alpha"] {
18083 let replies = handler
18084 .handle_control_frame(
18085 &module_ctx,
18086 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
18087 )
18088 .await
18089 .unwrap();
18090 assert_eq!(replies.len(), 1, "second HELLO must be refused");
18091 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
18092 }
18093 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
18094 assert!(registry.get_module("beta").unwrap().is_none());
18095 assert!(matches!(
18096 forwarding
18097 .lookup_data_route(
18098 client_ctx.connection_id,
18099 pending.client_channel,
18100 pending.client_epoch,
18101 )
18102 .unwrap(),
18103 DataRoute::Client(DataRouteState::Bound(_))
18104 ));
18105 assert!(handler
18106 .hello_launch_nonces
18107 .lock()
18108 .unwrap()
18109 .presented(module_ctx.connection_id, Some("alpha-nonce")));
18110 assert!(module_rx.try_recv().is_err());
18111 }
18112
18113 #[test]
18114 fn reserved_module_hello_requires_matching_launch_nonce() {
18115 let registry = Arc::new(Registry::default());
18116 let supervisor = SupervisorHandle::new();
18117 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
18120 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18121
18122 let no_nonce = handler
18124 .handle_control(
18125 ConnectionId::new(1),
18126 hello_frame("vault", PROTOCOL_VERSION, 1),
18127 )
18128 .unwrap();
18129 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
18130 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
18131 assert!(registry.get_module("vault").unwrap().is_none());
18132
18133 let wrong = handler
18135 .handle_control(
18136 ConnectionId::new(2),
18137 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
18138 )
18139 .unwrap();
18140 assert_eq!(wrong[0].header.ty, FrameType::Error);
18141 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
18142 assert!(registry.get_module("vault").unwrap().is_none());
18143
18144 let ok = handler
18146 .handle_control(
18147 ConnectionId::new(3),
18148 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
18149 )
18150 .unwrap();
18151 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
18152 assert!(registry.get_module("vault").unwrap().is_some());
18153 }
18154
18155 #[test]
18156 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
18157 let registry = Arc::new(Registry::default());
18158 let supervisor = SupervisorHandle::new();
18159 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18160 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18161 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18162
18163 let squat = handler
18164 .handle_control(
18165 ConnectionId::new(1),
18166 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
18167 )
18168 .unwrap();
18169 assert_eq!(squat[0].header.ty, FrameType::Error);
18170 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
18171 assert!(parse_error(&squat[0])["message"]
18172 .as_str()
18173 .unwrap()
18174 .contains("fed:"));
18175
18176 let accepted_peer = handler
18177 .handle_control(
18178 ConnectionId::new(2),
18179 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
18180 )
18181 .unwrap();
18182 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
18183
18184 let accepted_short = handler
18185 .handle_control(
18186 ConnectionId::new(3),
18187 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
18188 )
18189 .unwrap();
18190 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
18191
18192 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
18193 let response = handler
18194 .handle_control(
18195 ConnectionId::new(conn),
18196 hello_frame(module_id, PROTOCOL_VERSION, conn),
18197 )
18198 .unwrap();
18199 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
18200 }
18201 }
18202
18203 #[test]
18204 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
18205 let registry = Arc::new(Registry::default());
18206 let supervisor = SupervisorHandle::new();
18207 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18208 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18209 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
18210 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
18211
18212 let owner_nonce = handler
18213 .handle_control(
18214 ConnectionId::new(1),
18215 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
18216 )
18217 .unwrap();
18218 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
18219 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
18220 assert!(registry.get_module("fed:special").unwrap().is_none());
18221
18222 let exact_nonce = handler
18223 .handle_control(
18224 ConnectionId::new(2),
18225 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
18226 )
18227 .unwrap();
18228 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
18229 assert!(registry.get_module("fed:special").unwrap().is_some());
18230 }
18231
18232 #[test]
18233 fn non_reserved_module_ignores_launch_nonce() {
18234 let registry = Arc::new(Registry::default());
18235 let handler = ControlHandler::new(Arc::clone(®istry));
18238 let no_nonce = handler
18239 .handle_control(
18240 ConnectionId::new(1),
18241 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
18242 )
18243 .unwrap();
18244 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
18245 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
18246
18247 let echoed_nonce = handler
18248 .handle_control(
18249 ConnectionId::new(2),
18250 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
18251 )
18252 .unwrap();
18253 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
18254 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
18255 }
18256
18257 #[test]
18258 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
18259 let handler = ControlHandler::default();
18260 let conn = ConnectionId::new(1);
18261 let malformed = Frame::build(
18262 FrameType::Hello,
18263 control_flags(),
18264 0,
18265 0,
18266 3,
18267 b"{not json".to_vec(),
18268 )
18269 .unwrap();
18270
18271 let error = handler.handle_control(conn, malformed).unwrap();
18272 assert_eq!(error[0].header.ty, FrameType::Error);
18273 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
18274
18275 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18276 let pong = handler.handle_control(conn, ping).unwrap();
18277 assert_eq!(pong[0].header.ty, FrameType::Pong);
18278 assert_eq!(pong[0].header.corr, 4);
18279 }
18280
18281 #[test]
18282 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
18283 let registry = Arc::new(Registry::default());
18284 let handler = ControlHandler::new(Arc::clone(®istry));
18285
18286 handler
18287 .handle_control(
18288 ConnectionId::new(1),
18289 hello_frame("aft", PROTOCOL_VERSION, 1),
18290 )
18291 .unwrap();
18292 let duplicate = handler
18293 .handle_control(
18294 ConnectionId::new(2),
18295 hello_frame("aft", PROTOCOL_VERSION, 2),
18296 )
18297 .unwrap();
18298
18299 assert_eq!(duplicate[0].header.ty, FrameType::Error);
18300 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18301 let registration = registry.get_module("aft").unwrap().unwrap();
18302 assert_eq!(registration.connection_id, ConnectionId::new(1));
18303 }
18304
18305 #[test]
18306 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
18307 let registry = Arc::new(Registry::default());
18308 let forwarding = Arc::new(ForwardingTable::default());
18309 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
18310 let handler =
18311 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18312 .with_process_liveness(process_liveness);
18313 let (ctx, route_channel, route_epoch) =
18314 bind_liveness_route(®istry, &forwarding, "aft-dead");
18315 let responses = handler
18316 .handle_route_poll(
18317 &ctx,
18318 route_poll_frame(41, PollKind::Liveness, route_channel),
18319 route_channel,
18320 route_epoch,
18321 PollKind::Liveness,
18322 )
18323 .unwrap();
18324
18325 assert_eq!(responses.len(), 1);
18326 assert_eq!(responses[0].header.ty, FrameType::Response);
18327 assert_route_poll_liveness(&responses[0], false);
18328 }
18329
18330 #[test]
18331 fn liveness_poll_without_process_source_uses_bound_route() {
18332 let registry = Arc::new(Registry::default());
18333 let forwarding = Arc::new(ForwardingTable::default());
18334 let handler =
18335 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
18336 let (ctx, route_channel, route_epoch) =
18337 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
18338 let responses = handler
18339 .handle_route_poll(
18340 &ctx,
18341 route_poll_frame(42, PollKind::Liveness, route_channel),
18342 route_channel,
18343 route_epoch,
18344 PollKind::Liveness,
18345 )
18346 .unwrap();
18347
18348 assert_route_poll_liveness(&responses[0], true);
18349 }
18350
18351 #[test]
18352 fn liveness_poll_untracked_process_source_uses_bound_route() {
18353 let registry = Arc::new(Registry::default());
18354 let forwarding = Arc::new(ForwardingTable::default());
18355 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
18356 let handler =
18357 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18358 .with_process_liveness(process_liveness);
18359 let (ctx, route_channel, route_epoch) =
18360 bind_liveness_route(®istry, &forwarding, "aft-untracked");
18361 let responses = handler
18362 .handle_route_poll(
18363 &ctx,
18364 route_poll_frame(43, PollKind::Liveness, route_channel),
18365 route_channel,
18366 route_epoch,
18367 PollKind::Liveness,
18368 )
18369 .unwrap();
18370
18371 assert_route_poll_liveness(&responses[0], true);
18372 }
18373
18374 #[tokio::test]
18375 async fn unknown_op_returns_unknown_control_op() {
18376 let handler = ControlHandler::default();
18377 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
18378 let request = Frame::build(
18379 FrameType::Request,
18380 control_flags(),
18381 0,
18382 0,
18383 55,
18384 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
18385 )
18386 .unwrap();
18387
18388 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18389
18390 assert_eq!(response.len(), 1);
18391 assert_eq!(response[0].header.ty, FrameType::Error);
18392 assert_eq!(response[0].header.corr, 55);
18393 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
18394 }
18395
18396 #[tokio::test]
18397 async fn supervisor_provenance_rejects_unknown_exact_module() {
18398 let handler = ControlHandler::default();
18399 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
18400 let request = Frame::build(
18401 FrameType::Request,
18402 control_flags(),
18403 0,
18404 0,
18405 57,
18406 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
18407 )
18408 .unwrap();
18409
18410 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18411
18412 assert_eq!(response.len(), 1);
18413 assert_eq!(response[0].header.ty, FrameType::Error);
18414 assert_eq!(response[0].header.corr, 57);
18415 let error = parse_error(&response[0]);
18416 assert_eq!(error["code"], "unknown_module");
18417 assert_eq!(error["message"], "module_id 'missing' is not supervised");
18418 }
18419
18420 #[test]
18421 fn provenance_probe_override_keeps_handler_tests_deterministic() {
18422 let expected = subc_control::RunningImageAgreement::Unavailable {
18423 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18424 };
18425 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
18426 assert_eq!(handler.provenance_probe_override, Some(expected));
18427 }
18428
18429 #[test]
18430 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
18431 let verdict = reload_verdict(
18432 std::path::Path::new("/bin/new"),
18433 Some(std::path::Path::new("/bin/old")),
18434 subc_control::RunningImageAgreement::Unavailable {
18435 reason: subc_control::RunningImageUnavailableReason::HashFailed,
18436 },
18437 );
18438 assert!(matches!(
18439 verdict.path,
18440 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
18441 if configured == std::path::Path::new("/bin/new")
18442 && spawned_from == std::path::Path::new("/bin/old")
18443 ));
18444 }
18445
18446 #[test]
18447 fn launch_spec_without_running_digest_is_unavailable_not_match() {
18448 use subc_control::{
18449 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
18450 };
18451 let configured = crate::supervise::ModuleSpec {
18452 script: None,
18453 carry_containment: false,
18454 module_id: "legacy".into(),
18455 program: "/module".into(),
18456 args: Vec::new(),
18457 env: Vec::new(),
18458 reserved: false,
18459 reserved_prefixes: Vec::new(),
18460 protocol: subc_control::ModuleProtocol::Subc,
18461 overlap: Default::default(),
18462 };
18463 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
18464 assert_eq!(
18465 launch_spec_agreement(&configured, false, Some(&digest)),
18466 Agreement::Unavailable {
18467 reason: Reason::NotRunning
18468 }
18469 );
18470 assert_eq!(
18471 launch_spec_agreement(&configured, true, None),
18472 Agreement::Unavailable {
18473 reason: Reason::SpawnedSpecUnavailable
18474 }
18475 );
18476 assert_eq!(
18477 launch_spec_agreement(&configured, true, Some(&digest)),
18478 Agreement::Match
18479 );
18480 }
18481
18482 #[cfg(windows)]
18483 #[test]
18484 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
18485 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
18486 let configured = dir.join("module.exe");
18487 std::fs::write(&configured, b"executable path fixture").unwrap();
18488 let spawned = std::fs::canonicalize(&configured).unwrap();
18489 assert_ne!(
18490 configured, spawned,
18491 "fixture must exercise different path spellings"
18492 );
18493 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
18494 let lower_case = configured.to_string_lossy().to_lowercase();
18495 for spelling in [
18496 &configured,
18497 Path::new(&forward_slashes),
18498 Path::new(&lower_case),
18499 ] {
18500 let verdict = reload_verdict(
18501 spelling,
18502 Some(&spawned),
18503 subc_control::RunningImageAgreement::Unavailable {
18504 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18505 },
18506 );
18507 assert_eq!(
18508 verdict.path,
18509 ReloadPathAgreement::Match,
18510 "configured {spelling:?}, spawned {spawned:?}"
18511 );
18512 }
18513 }
18514
18515 #[cfg(windows)]
18516 #[test]
18517 fn windows_reload_path_keeps_different_existing_program_pending() {
18518 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
18519 let old = dir.join("old.exe");
18520 let new = dir.join("new.exe");
18521 std::fs::write(&old, b"same executable content").unwrap();
18522 std::fs::copy(&old, &new).unwrap();
18523 let spawned = std::fs::canonicalize(&old).unwrap();
18524 let image = subc_control::RunningImageAgreement::Unavailable {
18525 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18526 };
18527 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
18528 assert!(
18529 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
18530 if configured == new && spawned_from == spawned)
18531 );
18532 let missing = dir.join("missing.exe");
18533 assert!(matches!(
18534 reload_verdict(&missing, Some(&spawned), image).path,
18535 ReloadPathAgreement::Mismatch { .. }
18536 ));
18537 }
18538
18539 #[test]
18540 fn reload_verdict_detects_replaced_image_at_same_path() {
18541 let image = subc_control::RunningImageAgreement::Mismatch {
18542 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
18543 digest: "old".into(),
18544 },
18545 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
18546 digest: "new".into(),
18547 },
18548 };
18549 let verdict = reload_verdict(
18550 std::path::Path::new("/bin/same"),
18551 Some(std::path::Path::new("/bin/same")),
18552 image.clone(),
18553 );
18554 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18555 assert_eq!(verdict.image, image);
18556 }
18557
18558 #[test]
18559 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
18560 let image = subc_control::RunningImageAgreement::Unavailable {
18561 reason: subc_control::RunningImageUnavailableReason::NotRunning,
18562 };
18563 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
18564 assert_eq!(
18565 verdict.path,
18566 subc_control::ReloadPathAgreement::Unavailable {
18567 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
18568 }
18569 );
18570 assert_eq!(verdict.image, image);
18571
18572 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
18573 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18574 };
18575 let verdict = reload_verdict(
18576 std::path::Path::new("/bin/same"),
18577 Some(std::path::Path::new("/bin/same")),
18578 unconfirmed.clone(),
18579 );
18580 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18581 assert_eq!(verdict.image, unconfirmed);
18582 }
18583
18584 #[test]
18585 fn reload_verdict_preserves_each_image_unavailability_reason() {
18586 use subc_control::RunningImageUnavailableReason as Reason;
18587
18588 for reason in [
18589 Reason::NotRunning,
18590 Reason::UnsupportedPlatform,
18591 Reason::RunningExecutableUnreadable,
18592 Reason::SpawnedPathUnreadable,
18593 Reason::HashFailed,
18594 Reason::ProcessIdentityUnconfirmed,
18595 Reason::Unknown("future_probe_reason".to_string()),
18596 ] {
18597 let image = subc_control::RunningImageAgreement::Unavailable {
18598 reason: reason.clone(),
18599 };
18600 let verdict = reload_verdict(
18601 std::path::Path::new("/bin/same"),
18602 Some(std::path::Path::new("/bin/same")),
18603 image.clone(),
18604 );
18605 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18606 assert_eq!(verdict.image, image, "{reason:?}");
18607 }
18608 }
18609
18610 #[tokio::test]
18611 async fn malformed_control_bodies_return_invalid_control_body() {
18612 let handler = ControlHandler::default();
18613 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
18614
18615 for (corr, body) in [
18616 (56, br#"{"route_channel":1}"#.as_slice()),
18617 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
18618 (
18619 58,
18620 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
18621 ),
18622 ] {
18623 let request = Frame::build(
18624 FrameType::Request,
18625 control_flags(),
18626 0,
18627 0,
18628 corr,
18629 body.to_vec(),
18630 )
18631 .unwrap();
18632 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18633
18634 assert_eq!(response.len(), 1);
18635 assert_eq!(response[0].header.ty, FrameType::Error);
18636 assert_eq!(response[0].header.corr, corr);
18637 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
18638 }
18639 }
18640
18641 #[tokio::test]
18642 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
18643 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18644 let registry = Arc::new(Registry::default());
18645 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18646 let router = Router::with_control_handler(Arc::clone(&control));
18647 let connection = router.begin_connection();
18648 let (ctx, mut rx) = route_ctx(connection.id());
18649
18650 router
18651 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
18652 .await
18653 .unwrap();
18654 let response = rx.recv().await.unwrap();
18655 let ack = parse_ack(&response);
18656 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18657 let channel = 1;
18658
18659 let goodbye =
18660 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
18661 router.route_for_connection(&ctx, goodbye).await.unwrap();
18662 assert!(rx.try_recv().is_err());
18663 assert!(registry.get_module("aft").unwrap().is_none());
18664
18665 router
18666 .route_for_connection(&ctx, channel_request(channel, 13))
18667 .await
18668 .unwrap();
18669 let error_frame = rx.recv().await.unwrap();
18670 assert_eq!(error_frame.header.ty, FrameType::Error);
18671 assert_eq!(error_frame.header.channel, channel);
18672 let captured = crate::router::test_log::captured_logs(&logs);
18673 assert_eq!(
18674 captured
18675 .lines()
18676 .filter(|line| {
18677 line.contains("module_id=aft")
18678 && line.contains("reason=explicit_goodbye")
18679 && line.contains("module registration ended")
18680 })
18681 .count(),
18682 1,
18683 "unexpected GOODBYE registry log: {captured}"
18684 );
18685 }
18686
18687 #[tokio::test]
18688 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
18689 let registry = Arc::new(Registry::default());
18690 let handler = ControlHandler::new(registry).with_capability_config(
18691 [("prov".to_string(), true), ("cons".to_string(), true)],
18692 BTreeMap::new(),
18693 );
18694 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
18695 register_capability_manifest(
18696 &handler,
18697 &provider_ctx,
18698 &mut provider_rx,
18699 capability_manifest("prov", &["thing/v1"], &[]),
18700 1,
18701 )
18702 .await;
18703 let mut consumer = capability_manifest("cons", &[], &[]);
18704 consumer.capabilities.as_mut().unwrap().requires.push(
18705 subc_protocol::manifest::CapabilityRequirement {
18706 capability: "thing/v1".to_string(),
18707 need: subc_protocol::manifest::CapabilityNeed::Required,
18708 },
18709 );
18710 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
18711 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
18712 assert_eq!(
18713 handler.capability_evaluator.verdict("cons", "thing/v1"),
18714 Some(CapabilityVerdict::Provided)
18715 );
18716 let (mut client_rx, _) = open_route_for_capability_test(
18717 &handler,
18718 &provider_ctx,
18719 &mut provider_rx,
18720 703,
18721 3,
18722 "prov",
18723 None,
18724 )
18725 .await;
18726 let goodbye =
18727 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18728 handler
18729 .handle_control_frame(&provider_ctx, goodbye)
18730 .await
18731 .unwrap();
18732 assert_eq!(
18733 handler.capability_evaluator.verdict("cons", "thing/v1"),
18734 Some(CapabilityVerdict::NeverProvided)
18735 );
18736 let closed = client_rx
18737 .try_recv()
18738 .expect("GOODBYE pushes route.closed before route GOODBYE");
18739 assert!(
18740 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
18741 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
18742 );
18743 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
18744 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
18745 }
18746
18747 struct RouteClosedFixture {
18748 handler: ControlHandler,
18749 #[cfg(unix)]
18750 supervisor: Supervisor,
18751 module_ctx: RouteCtx,
18752 #[cfg(unix)]
18753 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18754 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18755 route: Arc<crate::forwarding::RouteBinding>,
18756 }
18757
18758 async fn route_closed_fixture() -> RouteClosedFixture {
18759 let registry = Arc::new(Registry::default());
18760 let handler = ControlHandler::new(Arc::clone(®istry));
18761 #[cfg(unix)]
18762 let supervisor = Supervisor::new(registry, RestartPolicy::default())
18763 .with_forwarding(Arc::clone(&handler.forwarding));
18764 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
18765 register_capability_manifest(
18766 &handler,
18767 &module_ctx,
18768 &mut module_rx,
18769 capability_manifest("prov", &[], &[]),
18770 1,
18771 )
18772 .await;
18773 let (client_rx, bind) = open_route_for_capability_test(
18774 &handler,
18775 &module_ctx,
18776 &mut module_rx,
18777 702,
18778 2,
18779 "prov",
18780 None,
18781 )
18782 .await;
18783 let ModuleControlRequest::RouteBind {
18784 route_channel: channel,
18785 epoch: route_epoch,
18786 ..
18787 } = bind
18788 else {
18789 panic!("expected route.bind");
18790 };
18791 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
18792 handler
18793 .forwarding
18794 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
18795 .unwrap()
18796 else {
18797 panic!("the acknowledged route must be bound");
18798 };
18799 RouteClosedFixture {
18800 handler,
18801 #[cfg(unix)]
18802 supervisor,
18803 module_ctx,
18804 #[cfg(unix)]
18805 module_rx,
18806 client_rx,
18807 route,
18808 }
18809 }
18810
18811 impl RouteClosedFixture {
18812 #[cfg(unix)]
18813 async fn cut(&mut self) {
18814 let notices = async {
18815 let mut draining = self.module_rx.recv().await.unwrap();
18816 assert!(matches!(
18817 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
18818 &draining.body
18819 )
18820 .unwrap(),
18821 subc_protocol::session::ModuleControlCommand::Draining {
18822 reason: RouteCloseReason::Disable,
18823 ..
18824 }
18825 ));
18826 draining.flushed.take().unwrap().send(()).unwrap();
18827 let mut closing = self.client_rx.recv().await.unwrap();
18828 assert!(matches!(
18829 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
18830 ClientControlPush::RouteClosing {
18831 reason: RouteCloseReason::Restart,
18832 ..
18833 }
18834 ));
18835 closing.flushed.take().unwrap().send(()).unwrap();
18836 };
18837 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
18838 result.unwrap();
18839 }
18840
18841 fn disconnect(&mut self) -> ClientControlPush {
18842 self.handler
18843 .cleanup_connection(self.module_ctx.connection_id)
18844 .unwrap();
18845 let closed = self.client_rx.try_recv().unwrap();
18846 assert_eq!(closed.header.ty, FrameType::Push);
18847 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
18848 assert!(matches!(
18849 &push,
18850 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
18851 if channels == &[self.route.client_channel]
18852 ));
18853 assert_eq!(
18854 self.client_rx.try_recv().unwrap().header.ty,
18855 FrameType::Goodbye
18856 );
18857 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
18858 push
18859 }
18860 }
18861
18862 #[cfg(unix)]
18863 #[tokio::test(start_paused = true)]
18864 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
18865 let mut fixture = route_closed_fixture().await;
18866 fixture.cut().await;
18867 assert!(matches!(
18868 fixture.disconnect(),
18869 ClientControlPush::RouteClosed {
18870 reason: RouteCloseReason::Restart,
18871 drained: true,
18872 ..
18873 }
18874 ));
18875 }
18876
18877 #[cfg(unix)]
18878 #[tokio::test(start_paused = true)]
18879 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
18880 let mut fixture = route_closed_fixture().await;
18881 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
18882 let started = tokio::time::Instant::now();
18883 fixture.cut().await;
18884 assert!(started.elapsed() >= Duration::from_secs(2));
18885 assert_eq!(fixture.route.flow.in_flight(), 1);
18886 assert!(matches!(
18887 fixture.disconnect(),
18888 ClientControlPush::RouteClosed {
18889 reason: RouteCloseReason::Restart,
18890 drained: false,
18891 ..
18892 }
18893 ));
18894 }
18895
18896 #[tokio::test]
18897 async fn module_connection_crash_reports_crash_and_not_drained() {
18898 let mut fixture = route_closed_fixture().await;
18899 assert!(matches!(
18903 fixture.disconnect(),
18904 ClientControlPush::RouteClosed {
18905 reason: RouteCloseReason::Crash,
18906 drained: false,
18907 ..
18908 }
18909 ));
18910 }
18911
18912 #[tokio::test]
18913 async fn dropping_router_connection_releases_registration() {
18914 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18915 let registry = Arc::new(Registry::default());
18916 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
18917 let router = Router::with_control_handler(Arc::clone(&control));
18918 let connection = router.begin_connection();
18919 let connection_id = connection.id();
18920 let (ctx, mut rx) = route_ctx(connection_id);
18921
18922 router
18923 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
18924 .await
18925 .unwrap();
18926 let response = rx.recv().await.unwrap();
18927 let ack = parse_ack(&response);
18928 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18929 assert!(registry.get_module("aft").unwrap().is_some());
18930
18931 drop(connection);
18932
18933 assert!(registry.get_module("aft").unwrap().is_none());
18934 assert_eq!(registry.active_registration_count().unwrap(), 0);
18935
18936 control
18937 .cleanup_connection(ConnectionId::new(u64::MAX))
18938 .unwrap();
18939 let captured = crate::router::test_log::captured_logs(&logs);
18940 println!("captured registry lifecycle logs:\n{captured}");
18941 let events: Vec<_> = captured
18942 .lines()
18943 .filter(|line| {
18944 line.contains("module registered module_id=aft ")
18945 || line.contains("module registration ended")
18946 })
18947 .collect();
18948 assert_eq!(
18949 events.len(),
18950 2,
18951 "unexpected registry lifecycle logs: {captured}"
18952 );
18953 assert!(events[0].contains("module registered module_id=aft "));
18954 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
18955 assert!(events[1].contains(&format!(
18956 "module_id=aft connection_id={}",
18957 connection_id.get()
18958 )));
18959 assert!(events[1].contains("reason=connection_closed"));
18960 assert!(events[1].contains("module registration ended"));
18961 assert_eq!(
18962 captured
18963 .lines()
18964 .filter(|line| line.contains("module registered module_id=aft "))
18965 .count(),
18966 1,
18967 "legacy registration admission line must appear once: {captured}"
18968 );
18969 }
18970
18971 #[tokio::test]
18972 async fn hello_registration_keeps_legacy_module_registered_line_once() {
18973 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18974 let registry = Arc::new(Registry::default());
18975 let control = ControlHandler::new(Arc::clone(®istry));
18976 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
18977 hello_via_sink(
18978 &control,
18979 &ctx,
18980 &mut rx,
18981 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
18982 )
18983 .await;
18984
18985 let captured = crate::router::test_log::captured_logs(&logs);
18986 let admissions: Vec<_> = captured
18987 .lines()
18988 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
18989 .collect();
18990 assert_eq!(
18991 admissions.len(),
18992 1,
18993 "legacy admission line must remain exactly once: {captured}"
18994 );
18995 assert!(admissions[0].contains("routable_provider=true"));
18996 assert!(admissions[0].contains("connection_id=777"));
18997 }
18998
18999 fn capability_manifest(
19000 module_id: &str,
19001 provides: &[&str],
19002 must_never_reach: &[&str],
19003 ) -> ModuleManifest {
19004 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
19005 manifest.capabilities = Some(CapabilityDeclarations {
19006 provides: provides
19007 .iter()
19008 .map(|capability| (*capability).to_string())
19009 .collect(),
19010 requires: Vec::new(),
19011 must_never_reach: must_never_reach
19012 .iter()
19013 .map(|capability| (*capability).to_string())
19014 .collect(),
19015 });
19016 manifest
19017 }
19018
19019 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
19020 Frame::build(
19021 FrameType::Hello,
19022 control_flags(),
19023 0,
19024 0,
19025 corr,
19026 serde_json::to_vec(&ModuleHelloBody {
19027 protocol_ver: manifest.protocol_ver,
19028 manifest,
19029 control_ops: None,
19030 launch_nonce: None,
19031 })
19032 .expect("capability test HELLO serializes"),
19033 )
19034 .expect("capability test HELLO frame builds")
19035 }
19036
19037 fn catalog_update_with_capabilities_frame(
19038 corr: u64,
19039 capabilities: CapabilityDeclarations,
19040 ) -> Frame {
19041 Frame::build(
19042 FrameType::Request,
19043 control_flags(),
19044 0,
19045 0,
19046 corr,
19047 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
19048 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
19049 capabilities: Some(capabilities),
19050 ready: None,
19051 })
19052 .expect("capability catalog.update serializes"),
19053 )
19054 .expect("capability catalog.update frame builds")
19055 }
19056
19057 async fn register_capability_manifest(
19058 handler: &ControlHandler,
19059 ctx: &RouteCtx,
19060 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19061 manifest: ModuleManifest,
19062 corr: u64,
19063 ) {
19064 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
19065 }
19066
19067 async fn open_route_for_capability_test(
19068 handler: &ControlHandler,
19069 target_ctx: &RouteCtx,
19070 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19071 client_connection_id: u64,
19072 corr: u64,
19073 target_module_id: &str,
19074 consumer_identity: Option<ConsumerIdentity>,
19075 ) -> (
19076 mpsc::Receiver<crate::router::OutboundFrame>,
19077 ModuleControlRequest,
19078 ) {
19079 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
19080 let route_handler = handler.clone();
19081 let target_module_id = target_module_id.to_string();
19082 let route_task = tokio::spawn(async move {
19083 route_handler
19084 .handle_control_frame(
19085 &client_ctx,
19086 route_open_frame_with_admission_facts(
19087 corr,
19088 &target_module_id,
19089 unique_project_root("admission-facts"),
19090 consumer_identity,
19091 None,
19092 ),
19093 )
19094 .await
19095 .expect("capability test route.open succeeds")
19096 });
19097 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
19098 .await
19099 .expect("capability test route.open must reach route.bind")
19100 .expect("target control receiver stays open");
19101 let bind_request: ModuleControlRequest =
19102 serde_json::from_slice(&bind.body).expect("route.bind decodes");
19103 handler
19104 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
19105 .await
19106 .expect("capability test route.bind ACK succeeds");
19107 assert!(route_task.await.expect("route.open task joins").is_empty());
19108 let opened = client_rx
19109 .recv()
19110 .await
19111 .expect("successful route.open publishes a response");
19112 assert!(matches!(
19113 serde_json::from_slice::<ClientControlResponse>(&opened.body),
19114 Ok(ClientControlResponse::RouteOpen { .. })
19115 ));
19116 (client_rx, bind_request)
19117 }
19118
19119 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
19120 assert_eq!(frame.header.ty, FrameType::Push);
19121 assert_eq!(frame.header.channel, 0);
19122 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
19123 .expect("route.closed control push decodes");
19124 let ClientControlPush::RouteClosed { channels, .. } = &push else {
19125 panic!("expected route.closed");
19126 };
19127 assert_eq!(channels.len(), 1, "exactly one violating route closed");
19128 let closed_channel = channels[0];
19129 let channels = channels.clone();
19130 assert_eq!(
19131 push,
19132 ClientControlPush::RouteClosed {
19133 module_id: target_module_id.to_string(),
19134 channels,
19135 reason: RouteCloseReason::CapabilityDenied,
19136 drained: false,
19137 abandoned: 0,
19138 excluded_subscriptions: 0,
19139 terminal: Some(false),
19140 }
19141 );
19142 closed_channel
19143 }
19144
19145 #[tokio::test]
19146 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
19147 let registry = Arc::new(Registry::default());
19148 let forwarding = Arc::new(ForwardingTable::default());
19149 let supervisor = SupervisorHandle::new();
19150 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19151 let handler =
19152 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19153 .with_supervisor(supervisor);
19154 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
19155 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
19156 register_capability_manifest(
19157 &handler,
19158 &target_ctx,
19159 &mut target_rx,
19160 capability_manifest("target", &["credentials-provider/v1"], &[]),
19161 1,
19162 )
19163 .await;
19164 register_capability_manifest(
19165 &handler,
19166 &opener_ctx,
19167 &mut opener_rx,
19168 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19169 2,
19170 )
19171 .await;
19172
19173 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
19174 let replies = handler
19175 .handle_control_frame(
19176 &client_ctx,
19177 route_open_frame_with_admission_facts(
19178 3,
19179 "target",
19180 unique_project_root("admission-facts"),
19181 Some(ConsumerIdentity {
19182 module_id: "opener".to_string(),
19183 launch_nonce: "opener-nonce".to_string(),
19184 }),
19185 None,
19186 ),
19187 )
19188 .await
19189 .expect("denied route.open returns a typed frame");
19190 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19191 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19192 assert!(
19193 target_rx.try_recv().is_err(),
19194 "forbidden route.open must not relay route.bind"
19195 );
19196 }
19197
19198 #[tokio::test]
19199 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
19200 let registry = Arc::new(Registry::default());
19201 let forwarding = Arc::new(ForwardingTable::default());
19202 let supervisor = SupervisorHandle::new();
19203 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19204 let handler =
19205 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19206 .with_supervisor(supervisor);
19207 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
19208 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
19209 register_capability_manifest(
19210 &handler,
19211 &target_ctx,
19212 &mut target_rx,
19213 capability_manifest("target", &["credentials-provider/v1"], &[]),
19214 1,
19215 )
19216 .await;
19217 register_capability_manifest(
19218 &handler,
19219 &old_opener_ctx,
19220 &mut old_opener_rx,
19221 capability_manifest("opener", &[], &[]),
19222 2,
19223 )
19224 .await;
19225 let (mut client_rx, _) = open_route_for_capability_test(
19226 &handler,
19227 &target_ctx,
19228 &mut target_rx,
19229 712,
19230 3,
19231 "target",
19232 Some(ConsumerIdentity {
19233 module_id: "opener".to_string(),
19234 launch_nonce: "opener-nonce".to_string(),
19235 }),
19236 )
19237 .await;
19238 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19239
19240 handler
19241 .cleanup_connection(old_opener_ctx.connection_id)
19242 .expect("old opener registration cleans up");
19243 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
19244 register_capability_manifest(
19245 &handler,
19246 &new_opener_ctx,
19247 &mut new_opener_rx,
19248 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19249 4,
19250 )
19251 .await;
19252
19253 let closed_channel = assert_capability_denied_push(
19254 client_rx
19255 .try_recv()
19256 .expect("HELLO deny addition must emit route.closed")
19257 .frame,
19258 "target",
19259 );
19260 let goodbye = client_rx
19261 .try_recv()
19262 .expect("HELLO deny addition must send the client a route GOODBYE")
19263 .frame;
19264 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19265 assert_eq!(goodbye.header.channel, closed_channel);
19266 assert_ne!(goodbye.header.channel, 0);
19267 assert_ne!(goodbye.header.epoch, 0);
19268 assert!(client_rx.try_recv().is_err());
19269 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19270 assert!(matches!(
19271 target_rx.try_recv(),
19272 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19273 ));
19274 }
19275
19276 #[tokio::test]
19277 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
19278 let registry = Arc::new(Registry::default());
19279 let forwarding = Arc::new(ForwardingTable::default());
19280 let supervisor = SupervisorHandle::new();
19281 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19282 let handler =
19283 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19284 .with_supervisor(supervisor);
19285 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
19286 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
19287 register_capability_manifest(
19288 &handler,
19289 &target_ctx,
19290 &mut target_rx,
19291 capability_manifest("target", &[], &[]),
19292 1,
19293 )
19294 .await;
19295 register_capability_manifest(
19296 &handler,
19297 &opener_ctx,
19298 &mut opener_rx,
19299 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19300 2,
19301 )
19302 .await;
19303 let (mut client_rx, _) = open_route_for_capability_test(
19304 &handler,
19305 &target_ctx,
19306 &mut target_rx,
19307 722,
19308 3,
19309 "target",
19310 Some(ConsumerIdentity {
19311 module_id: "opener".to_string(),
19312 launch_nonce: "opener-nonce".to_string(),
19313 }),
19314 )
19315 .await;
19316 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19317
19318 let replies = handler
19319 .handle_control_frame(
19320 &target_ctx,
19321 catalog_update_with_capabilities_frame(
19322 4,
19323 CapabilityDeclarations {
19324 provides: vec!["credentials-provider/v1".to_string()],
19325 requires: Vec::new(),
19326 must_never_reach: Vec::new(),
19327 },
19328 ),
19329 )
19330 .await
19331 .expect("claim catalog.update succeeds");
19332 assert!(matches!(
19333 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
19334 Ok(ModuleControlResponseToModule::CatalogUpdate {})
19335 ));
19336 let closed_channel = assert_capability_denied_push(
19337 client_rx
19338 .try_recv()
19339 .expect("claim addition must emit route.closed")
19340 .frame,
19341 "target",
19342 );
19343 let goodbye = client_rx
19344 .try_recv()
19345 .expect("claim addition must send the client a route GOODBYE")
19346 .frame;
19347 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19348 assert_eq!(goodbye.header.channel, closed_channel);
19349 assert_ne!(goodbye.header.channel, 0);
19350 assert_ne!(goodbye.header.epoch, 0);
19351 assert!(client_rx.try_recv().is_err());
19352 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19353 assert!(matches!(
19354 target_rx.try_recv(),
19355 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19356 ));
19357 }
19358
19359 #[tokio::test]
19360 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
19361 let registry = Arc::new(Registry::default());
19362 let forwarding = Arc::new(ForwardingTable::default());
19363 let supervisor = SupervisorHandle::new();
19364 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19365 let handler =
19366 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19367 .with_supervisor(supervisor);
19368 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
19369 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
19370 register_capability_manifest(
19371 &handler,
19372 &target_ctx,
19373 &mut target_rx,
19374 capability_manifest("target", &["credentials-provider/v1"], &[]),
19375 1,
19376 )
19377 .await;
19378 register_capability_manifest(
19379 &handler,
19380 &opener_ctx,
19381 &mut opener_rx,
19382 capability_manifest("opener", &[], &[]),
19383 2,
19384 )
19385 .await;
19386 let (mut client_rx, _) = open_route_for_capability_test(
19387 &handler,
19388 &target_ctx,
19389 &mut target_rx,
19390 732,
19391 3,
19392 "target",
19393 Some(ConsumerIdentity {
19394 module_id: "opener".to_string(),
19395 launch_nonce: "opener-nonce".to_string(),
19396 }),
19397 )
19398 .await;
19399 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19400
19401 handler
19402 .handle_control_frame(
19403 &target_ctx,
19404 catalog_update_with_capabilities_frame(
19405 4,
19406 CapabilityDeclarations {
19407 provides: Vec::new(),
19408 requires: Vec::new(),
19409 must_never_reach: Vec::new(),
19410 },
19411 ),
19412 )
19413 .await
19414 .expect("claim removal catalog.update succeeds");
19415 assert_eq!(
19416 forwarding.active_binding_count().unwrap(),
19417 1,
19418 "removing an attested target claim must leave the route census unchanged"
19419 );
19420 assert!(
19421 client_rx.try_recv().is_err(),
19422 "claim removal must not emit route.closed or a client route GOODBYE"
19423 );
19424 assert!(
19425 target_rx.try_recv().is_err(),
19426 "claim removal must not send the target a route GOODBYE"
19427 );
19428 }
19429
19430 #[tokio::test]
19433 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
19434 let registry = Arc::new(Registry::default());
19435 let forwarding = Arc::new(ForwardingTable::default());
19436 let supervisor = SupervisorHandle::new();
19437 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19438 let handler =
19439 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19440 .with_supervisor(supervisor);
19441 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
19442 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
19443 register_capability_manifest(
19444 &handler,
19445 &target_ctx,
19446 &mut target_rx,
19447 capability_manifest("target", &["credentials-provider/v1"], &[]),
19448 1,
19449 )
19450 .await;
19451 register_capability_manifest(
19452 &handler,
19453 &opener_ctx,
19454 &mut opener_rx,
19455 capability_manifest("opener", &[], &["credentials-provider/v1"]),
19456 2,
19457 )
19458 .await;
19459
19460 let (_client_rx, bind) = open_route_for_capability_test(
19461 &handler,
19462 &target_ctx,
19463 &mut target_rx,
19464 742,
19465 3,
19466 "target",
19467 None,
19468 )
19469 .await;
19470 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
19471 panic!("direct scope-honesty route must bind");
19472 };
19473 assert_eq!(principal, Some(Principal::Direct));
19474 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19475 }
19476
19477 #[tokio::test]
19480 async fn must_never_reach_self_route_is_capability_forbidden() {
19481 let registry = Arc::new(Registry::default());
19482 let forwarding = Arc::new(ForwardingTable::default());
19483 let supervisor = SupervisorHandle::new();
19484 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
19485 let handler =
19486 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
19487 .with_supervisor(supervisor);
19488 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
19489 register_capability_manifest(
19490 &handler,
19491 &self_ctx,
19492 &mut self_rx,
19493 capability_manifest(
19494 "self-provider",
19495 &["credentials-provider/v1"],
19496 &["credentials-provider/v1"],
19497 ),
19498 1,
19499 )
19500 .await;
19501
19502 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
19503 let replies = handler
19504 .handle_control_frame(
19505 &client_ctx,
19506 route_open_frame_with_admission_facts(
19507 2,
19508 "self-provider",
19509 unique_project_root("admission-facts"),
19510 Some(ConsumerIdentity {
19511 module_id: "self-provider".to_string(),
19512 launch_nonce: "self-nonce".to_string(),
19513 }),
19514 None,
19515 ),
19516 )
19517 .await
19518 .expect("self-route refusal returns a typed frame");
19519 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19520 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19521 assert!(
19522 self_rx.try_recv().is_err(),
19523 "self denial must not relay route.bind"
19524 );
19525 }
19526
19527 #[test]
19528 fn unsupported_channel_zero_frame_returns_error() {
19529 let handler = ControlHandler::default();
19530 let request = Frame::build(
19531 FrameType::Request,
19532 control_flags(),
19533 0,
19534 0,
19535 21,
19536 b"opaque".to_vec(),
19537 )
19538 .unwrap();
19539
19540 let response = handler
19541 .handle_control(ConnectionId::new(1), request)
19542 .unwrap();
19543
19544 assert_eq!(response[0].header.ty, FrameType::Error);
19545 assert_eq!(
19546 parse_error(&response[0])["code"],
19547 "unsupported_control_frame"
19548 );
19549 }
19550
19551 mod swap {
19556 use super::*;
19557
19558 const INCUMBENT: ConnectionId = ConnectionId::new(30);
19559 const CANDIDATE: ConnectionId = ConnectionId::new(40);
19560
19561 struct Swap {
19562 registry: Arc<Registry>,
19563 forwarding: Arc<ForwardingTable>,
19564 handler: ControlHandler,
19565 incumbent_ctx: RouteCtx,
19566 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19567 candidate_ctx: RouteCtx,
19568 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19569 }
19570
19571 async fn swap_with_incumbent() -> Swap {
19572 let registry = Arc::new(Registry::default());
19573 let forwarding = Arc::new(ForwardingTable::default());
19574 let handler =
19575 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
19576 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
19577 hello_via_sink(
19578 &handler,
19579 &incumbent_ctx,
19580 &mut incumbent_rx,
19581 hello_frame("aft", PROTOCOL_VERSION, 7),
19582 )
19583 .await;
19584 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
19585 Swap {
19586 registry,
19587 forwarding,
19588 handler,
19589 incumbent_ctx,
19590 incumbent_rx,
19591 candidate_ctx,
19592 candidate_rx,
19593 }
19594 }
19595
19596 fn register_candidate(swap: &Swap, ready: Option<bool>) {
19597 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
19598 candidate_manifest.ready = ready;
19599 let registration = swap
19600 .registry
19601 .register_candidate_with_control_ops(
19602 candidate_manifest,
19603 PROTOCOL_VERSION,
19604 CANDIDATE,
19605 module_baseline_control_ops(),
19606 )
19607 .unwrap();
19608 swap.forwarding
19609 .register_candidate_module_connection(
19610 CANDIDATE,
19611 "aft".to_string(),
19612 PROTOCOL_VERSION,
19613 manifest_concurrency(®istration.manifest),
19614 swap.candidate_ctx.egress.clone(),
19615 )
19616 .unwrap();
19617 }
19618
19619 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
19620 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
19621 swap.registry.promote_candidate("aft").unwrap().unwrap();
19622 cutover.incumbent.unwrap()
19623 }
19624
19625 #[tokio::test]
19634 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
19635 let mut swap = swap_with_incumbent().await;
19636 let hold = Arc::new(std::sync::Barrier::new(2));
19637 let handler = swap
19638 .handler
19639 .clone()
19640 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
19641 handler
19642 .supervisor
19643 .open_swap("aft", "candidate-nonce".to_string());
19644 let hello = {
19645 let handler = handler.clone();
19646 let candidate_ctx = swap.candidate_ctx.clone();
19647 std::thread::spawn(move || {
19648 handler
19649 .handle_hello(
19650 candidate_ctx.connection_id,
19651 Some(candidate_ctx.egress.clone()),
19652 hello_frame_with_nonce(
19653 "aft",
19654 PROTOCOL_VERSION,
19655 8,
19656 Some("candidate-nonce"),
19657 ),
19658 )
19659 .unwrap()
19660 })
19661 };
19662
19663 hold.wait();
19664 assert!(swap
19667 .registry
19668 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
19669 .unwrap()
19670 .is_some());
19671 assert!(swap
19672 .registry
19673 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19674 .unwrap()
19675 .is_none());
19676 assert!(swap
19677 .forwarding
19678 .module_endpoint_for_connection(CANDIDATE)
19679 .unwrap()
19680 .is_none());
19681
19682 hold.wait();
19683 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
19684 assert_eq!(
19685 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19686 FrameType::HelloAck
19687 );
19688 let published = swap
19689 .registry
19690 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19691 .unwrap()
19692 .expect("candidate is published once its HELLO completes");
19693 assert_eq!(published.connection_id, CANDIDATE);
19694 assert!(swap
19695 .forwarding
19696 .module_endpoint_for_connection(CANDIDATE)
19697 .unwrap()
19698 .is_some());
19699 }
19700
19701 #[tokio::test]
19702 async fn candidate_lookup_after_hello_ack_waits_for_publication() {
19703 let mut swap = swap_with_incumbent().await;
19704 let hold = Arc::new(std::sync::Barrier::new(2));
19705 swap.handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
19706 swap.handler
19707 .supervisor
19708 .open_swap("aft", "candidate-nonce".to_string());
19709 let hello = {
19710 let handler = swap.handler.clone();
19711 let candidate_ctx = swap.candidate_ctx.clone();
19712 std::thread::spawn(move || {
19713 handler
19714 .handle_hello(
19715 CANDIDATE,
19716 Some(candidate_ctx.egress),
19717 hello_frame_with_nonce(
19718 "aft",
19719 PROTOCOL_VERSION,
19720 8,
19721 Some("candidate-nonce"),
19722 ),
19723 )
19724 .unwrap()
19725 })
19726 };
19727 hold.wait();
19728 assert_eq!(
19729 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19730 FrameType::HelloAck
19731 );
19732 let reader_registry = Arc::clone(&swap.registry);
19733 let published = lookup_during_hello_publication(&swap.registry, &hold, move || {
19734 reader_registry
19735 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19736 .unwrap()
19737 });
19738 assert!(hello.join().unwrap().is_empty());
19739 assert_eq!(
19740 published
19741 .expect("ACK makes candidate publication visible")
19742 .connection_id,
19743 CANDIDATE
19744 );
19745 assert_eq!(
19746 swap.registry.generation().unwrap(),
19747 1,
19748 "candidate is not a second catalog entry"
19749 );
19750 }
19751
19752 #[tokio::test]
19753 async fn audit4_ungated_removed_candidate_publication_rolls_back_admission() {
19754 let mut swap = swap_with_incumbent().await;
19755 let hold = Arc::new(std::sync::Barrier::new(2));
19756 let handler = swap
19757 .handler
19758 .clone()
19759 .with_hello_registry_to_forwarding_hold(hold.clone());
19760 handler
19761 .supervisor
19762 .open_swap("aft", "candidate-nonce".into());
19763 let hello = {
19764 let handler = handler.clone();
19765 let ctx = swap.candidate_ctx.clone();
19766 std::thread::spawn(move || {
19767 handler
19768 .handle_hello(
19769 CANDIDATE,
19770 Some(ctx.egress),
19771 hello_frame_with_nonce(
19772 "aft",
19773 PROTOCOL_VERSION,
19774 8,
19775 Some("candidate-nonce"),
19776 ),
19777 )
19778 .unwrap()
19779 })
19780 };
19781 hold.wait();
19782 assert_eq!(
19783 swap.registry
19784 .deregister_connection(CANDIDATE)
19785 .unwrap()
19786 .len(),
19787 1
19788 );
19789 hold.wait();
19790 assert!(hello.join().unwrap().is_empty());
19791 assert_eq!(
19792 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19793 FrameType::HelloAck
19794 );
19795 assert!(swap.registry.get_candidate("aft").unwrap().is_none());
19796 assert_eq!(
19797 handler
19798 .supervisor
19799 .swap_hello_admission("aft", Some("candidate-nonce")),
19800 SwapHelloAdmission::Candidate,
19801 "failed publication must roll back candidate_admitted for an ungated nonce swap"
19802 );
19803 handler.supervisor.close_swap("aft");
19804 }
19805
19806 fn keyed_total(counters: &Value, key: &str) -> u64 {
19807 counters[key]
19808 .as_object()
19809 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
19810 .unwrap_or(0)
19811 }
19812
19813 #[tokio::test]
19814 async fn replacement_logs_old_end_and_new_admission_once() {
19815 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19816 let mut swap = swap_with_incumbent().await;
19817 swap.handler
19818 .supervisor
19819 .open_swap("aft", "candidate-nonce".to_string());
19820 hello_via_sink(
19821 &swap.handler,
19822 &swap.candidate_ctx,
19823 &mut swap.candidate_rx,
19824 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
19825 )
19826 .await;
19827 cutover(&swap);
19828 swap.handler.cleanup_connection(INCUMBENT).unwrap();
19829
19830 let captured = crate::router::test_log::captured_logs(&logs);
19831 println!("captured replacement registry logs:\n{captured}");
19832 let new_admissions: Vec<_> = captured
19833 .lines()
19834 .filter(|line| {
19835 line.contains("connection_id=40")
19836 && line.contains("swap candidate registered; not routable until cutover")
19837 })
19838 .collect();
19839 assert_eq!(
19840 new_admissions.len(),
19841 1,
19842 "unexpected admission logs: {captured}"
19843 );
19844 assert!(new_admissions[0].contains("module_id=aft"));
19845 assert!(new_admissions[0].contains("connection_id=40"));
19846 assert!(
19847 new_admissions[0].contains("swap candidate registered; not routable until cutover")
19848 );
19849 assert!(new_admissions[0].contains("ready=true"));
19850
19851 let old_admissions: Vec<_> = captured
19852 .lines()
19853 .filter(|line| {
19854 line.contains("module registered module_id=aft ")
19855 && line.contains("connection_id=30")
19856 })
19857 .collect();
19858 assert_eq!(
19859 old_admissions.len(),
19860 1,
19861 "unexpected incumbent admission: {captured}"
19862 );
19863
19864 let old_ends: Vec<_> = captured
19865 .lines()
19866 .filter(|line| {
19867 line.contains("connection_id=30") && line.contains("module registration ended")
19868 })
19869 .collect();
19870 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
19871 assert!(old_ends[0].contains("module_id=aft"));
19872 assert!(old_ends[0].contains("connection_id=30"));
19873 assert!(old_ends[0].contains("reason=replaced"));
19874 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
19875 assert_eq!(
19876 captured
19877 .lines()
19878 .filter(|line| line.contains("module registration promoted"))
19879 .count(),
19880 1,
19881 "unexpected promotion logs: {captured}"
19882 );
19883 }
19884
19885 #[tokio::test]
19891 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
19892 let mut swap = swap_with_incumbent().await;
19893 let handler = swap.handler.clone();
19894
19895 let cotenant = ConnectionId::new(31);
19897 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
19898 let (cotenant_task, cotenant_bind) = relay_route_open(
19899 &handler,
19900 cotenant,
19901 &cotenant_ctx.egress,
19902 &mut swap.incumbent_rx,
19903 100,
19904 "aft",
19905 "swap-cotenant",
19906 )
19907 .await;
19908 handler
19909 .handle_control_frame(
19910 &swap.incumbent_ctx,
19911 route_bind_ack(cotenant_bind.header.corr),
19912 )
19913 .await
19914 .unwrap();
19915 assert!(cotenant_task.await.unwrap().is_empty());
19916 let (cotenant_channel, cotenant_epoch) =
19917 published_route(&cotenant_rx.recv().await.unwrap());
19918
19919 let caller = ConnectionId::new(32);
19921 let (caller_ctx, mut caller_rx) = route_ctx(caller);
19922 let (caller_task, caller_bind) = relay_route_open(
19923 &handler,
19924 caller,
19925 &caller_ctx.egress,
19926 &mut swap.incumbent_rx,
19927 101,
19928 "aft",
19929 "swap-caller",
19930 )
19931 .await;
19932 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
19933
19934 register_candidate(&swap, None);
19935 cutover(&swap);
19936
19937 let ack = handler
19939 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
19940 .await;
19941 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
19942 if module_loop_error.is_some() {
19943 handler.cleanup_connection(INCUMBENT).unwrap();
19946 }
19947
19948 assert!(
19950 cotenant_rx.try_recv().is_err(),
19951 "the co-tenant route on the incumbent was torn down by one late ack: \
19952 {module_loop_error:?}"
19953 );
19954 assert!(matches!(
19955 swap.forwarding
19956 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
19957 .unwrap(),
19958 DataRoute::Client(DataRouteState::Bound(_))
19959 ));
19960 assert_eq!(module_loop_error, None);
19961 assert!(swap
19962 .registry
19963 .get_module_by_connection(INCUMBENT)
19964 .unwrap()
19965 .is_some());
19966
19967 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
19969 .await
19970 .expect("the incumbent is told to drop the abandoned binding")
19971 .unwrap()
19972 .frame;
19973 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19974 assert_eq!(goodbye.header.channel, abandoned_channel);
19975 assert_eq!(goodbye.header.epoch, abandoned_epoch);
19976 assert!(swap.incumbent_rx.try_recv().is_err());
19977
19978 let response = caller_task.await.unwrap();
19980 assert_eq!(response.len(), 1);
19981 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
19982 assert!(caller_rx.try_recv().is_err());
19983
19984 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
19988 let counters = handler.counters().snapshot();
19989 assert_eq!(
19990 keyed_total(&counters, "route_open_accepted_by_principal"),
19991 1
19992 );
19993 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
19994 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
19995 }
19996
19997 #[tokio::test]
20001 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
20002 let mut swap = swap_with_incumbent().await;
20003 register_candidate(&swap, None);
20004 let incumbent = cutover(&swap);
20005 swap.forwarding
20006 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
20007 .unwrap()
20008 .expect("the incumbent is still registered");
20009
20010 let client = ConnectionId::new(33);
20011 let (client_ctx, mut client_rx) = route_ctx(client);
20012 let route_handler = swap.handler.clone();
20013 let open_ctx = RouteCtx {
20014 connection_id: client,
20015 egress: client_ctx.egress.clone(),
20016 };
20017 let mut route_task = tokio::spawn(async move {
20018 route_handler
20019 .handle_control_frame(
20020 &open_ctx,
20021 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
20022 )
20023 .await
20024 .unwrap()
20025 });
20026 let bind = tokio::select! {
20027 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
20028 response = &mut route_task => {
20029 let response = response.unwrap();
20030 panic!(
20031 "post-cutover route.open was refused instead of relayed to the candidate: {}",
20032 parse_error(&response[0])["code"]
20033 );
20034 }
20035 };
20036 swap.handler
20037 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
20038 .await
20039 .unwrap();
20040 assert!(route_task.await.unwrap().is_empty());
20041 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
20042 match swap
20043 .forwarding
20044 .lookup_data_route(client, channel, epoch)
20045 .unwrap()
20046 {
20047 DataRoute::Client(DataRouteState::Bound(route)) => {
20048 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
20049 }
20050 other => panic!("expected a bound route on the candidate, got {other:?}"),
20051 }
20052 assert!(swap.incumbent_rx.try_recv().is_err());
20053 }
20054
20055 #[tokio::test]
20060 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
20061 let swap = swap_with_incumbent().await;
20062 register_candidate(&swap, Some(false));
20063 let update = Frame::build(
20064 FrameType::Request,
20065 control_flags(),
20066 0,
20067 0,
20068 55,
20069 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
20070 provides: manifest("aft", PROTOCOL_VERSION).provides,
20071 capabilities: None,
20072 ready: Some(true),
20073 })
20074 .unwrap(),
20075 )
20076 .unwrap();
20077
20078 let replies = swap
20079 .handler
20080 .handle_control_frame(&swap.candidate_ctx, update)
20081 .await
20082 .unwrap();
20083
20084 assert_eq!(replies.len(), 1);
20085 assert_eq!(
20086 replies[0].header.ty,
20087 FrameType::Response,
20088 "candidate catalog.update was refused: {:?}",
20089 serde_json::from_slice::<Value>(&replies[0].body).ok()
20090 );
20091 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
20092 assert_eq!(
20093 swap.registry
20094 .get_module("aft")
20095 .unwrap()
20096 .unwrap()
20097 .connection_id,
20098 INCUMBENT
20099 );
20100 }
20101 }
20102
20103 mod swap_admission {
20107 use super::*;
20108
20109 const INCUMBENT_NONCE: &str = "incumbent-nonce";
20110 const CANDIDATE_NONCE: &str = "candidate-nonce";
20111
20112 fn handler_with_incumbent(
20113 module_id: &str,
20114 reserved: bool,
20115 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
20116 let registry = Arc::new(Registry::default());
20117 let supervisor = SupervisorHandle::new();
20118 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
20119 if reserved {
20120 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
20121 }
20122 let handler =
20123 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
20124 let incumbent = handler
20125 .handle_control(
20126 ConnectionId::new(1),
20127 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
20128 )
20129 .unwrap();
20130 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
20131 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
20132 (registry, supervisor, handler)
20133 }
20134
20135 #[test]
20141 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
20142 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
20143
20144 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
20145 let replies = handler
20146 .handle_control(
20147 ConnectionId::new(connection),
20148 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
20149 )
20150 .unwrap();
20151 assert_eq!(replies[0].header.ty, FrameType::Error);
20152 assert_eq!(
20153 parse_error(&replies[0])["code"],
20154 "swap_token_invalid",
20155 "nonce {nonce:?}"
20156 );
20157 }
20158 assert!(registry.get_candidate("aft").unwrap().is_none());
20159 assert_eq!(
20160 registry.get_module("aft").unwrap().unwrap().connection_id,
20161 ConnectionId::new(1)
20162 );
20163
20164 let admitted = handler
20167 .handle_control(
20168 ConnectionId::new(4),
20169 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
20170 )
20171 .unwrap();
20172 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
20173 assert_eq!(
20174 registry
20175 .get_candidate("aft")
20176 .unwrap()
20177 .unwrap()
20178 .connection_id,
20179 ConnectionId::new(4)
20180 );
20181 assert_eq!(
20182 registry.get_module("aft").unwrap().unwrap().connection_id,
20183 ConnectionId::new(1),
20184 "the candidate must not take the active slot"
20185 );
20186 let replayed = handler
20187 .handle_control(
20188 ConnectionId::new(5),
20189 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
20190 )
20191 .unwrap();
20192 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
20193
20194 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
20198 let squatter = handler
20199 .handle_control(
20200 ConnectionId::new(6),
20201 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
20202 )
20203 .unwrap();
20204 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
20205 assert!(
20206 registry.get_module("aft").unwrap().is_none(),
20207 "a squatter took the active slot of an id being swapped"
20208 );
20209 }
20210
20211 #[test]
20216 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
20217 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
20218
20219 let replies = handler
20220 .handle_control(
20221 ConnectionId::new(2),
20222 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20223 )
20224 .unwrap();
20225
20226 assert_eq!(
20227 replies[0].header.ty,
20228 FrameType::HelloAck,
20229 "reserved candidate refused: {:?}",
20230 serde_json::from_slice::<Value>(&replies[0].body).ok()
20231 );
20232 assert_eq!(
20233 registry
20234 .get_candidate("vault")
20235 .unwrap()
20236 .unwrap()
20237 .connection_id,
20238 ConnectionId::new(2)
20239 );
20240 }
20241
20242 #[test]
20245 fn without_an_open_swap_the_ordinary_gates_decide() {
20246 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
20247 supervisor.close_swap("vault");
20248
20249 let candidate = handler
20250 .handle_control(
20251 ConnectionId::new(2),
20252 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20253 )
20254 .unwrap();
20255 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
20256 let duplicate = handler
20257 .handle_control(
20258 ConnectionId::new(3),
20259 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
20260 )
20261 .unwrap();
20262 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
20263 assert!(registry.get_candidate("vault").unwrap().is_none());
20264 }
20265 }
20266
20267 mod scopes {
20271 mod close_reasons {
20272 include!("route_close_reason_tests.rs");
20273 }
20274 mod idle_tests {
20275 use super::*;
20276 include!("scope_idle_tests.rs");
20277 }
20278 mod route_open_timings {
20279 use super::*;
20280 include!("route_open_timing_tests.rs");
20281 }
20282 mod route_open_refusal_log {
20283 use super::*;
20284 include!("route_open_refusal_log_tests.rs");
20285 }
20286 mod activity {
20287 include!("activity_scope_tests.rs");
20288 }
20289 mod delta {
20290 include!("scope_handler_delta_tests.rs");
20291 }
20292 mod call_modes {
20293 include!("scope_call_mode_tests.rs");
20294 }
20295 mod federated {
20296 include!("scope_federated_tests.rs");
20297 }
20298 mod authorize {
20299 include!("route_authorize_tests.rs");
20300 }
20301 #[tokio::test]
20302 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
20303 let rig = rig().await;
20304 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
20305 "s".into(),
20306 1,
20307 "tool".into(),
20308 ))
20309 .unwrap();
20310 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
20311 let (direct, _rx) = wide_ctx(999);
20312 for (ctx, expected) in [
20313 (&rig.owner, FrameType::Response),
20314 (&direct, FrameType::Error),
20315 ] {
20316 let frame =
20317 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
20318 .unwrap();
20319 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
20320 assert_eq!(replies[0].header.ty, expected);
20321 if expected == FrameType::Response {
20322 let reply: subc_control::ScopeServedByReply =
20323 serde_json::from_slice(&replies[0].body).unwrap();
20324 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
20325 } else {
20326 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
20327 }
20328 }
20329 let reply = call(
20330 &rig.handler,
20331 &rig.owner,
20332 &ModuleControlRequestFromModule::ScopeDescribe {
20333 owner: Principal::Reserved {
20334 module_id: OWNER.into(),
20335 },
20336 scope_ref: "s".into(),
20337 },
20338 )
20339 .await;
20340 assert!(matches!(
20341 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
20342 ModuleControlResponseToModule::ScopeDescribe { .. }
20343 ));
20344 }
20345
20346 use subc_protocol::scope::{
20347 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
20348 ScopeStatus,
20349 };
20350
20351 use super::*;
20352
20353 const OWNER: &str = "prefrontal-core";
20354
20355 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
20356 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
20357 }
20358
20359 async fn call(
20360 handler: &ControlHandler,
20361 ctx: &RouteCtx,
20362 request: &ModuleControlRequestFromModule,
20363 ) -> Frame {
20364 let body = serde_json::to_vec(request).unwrap();
20365 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
20366 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
20367 assert_eq!(replies.len(), 1, "{replies:?}");
20368 replies.pop().unwrap()
20369 }
20370
20371 async fn sync(
20372 handler: &ControlHandler,
20373 ctx: &RouteCtx,
20374 generation: u64,
20375 scopes: Vec<ScopeRecord>,
20376 ) -> Result<ModuleControlResponseToModule, String> {
20377 let reply = call(
20378 handler,
20379 ctx,
20380 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
20381 )
20382 .await;
20383 match reply.header.ty {
20384 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
20385 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
20386 }
20387 }
20388
20389 async fn describe(
20390 handler: &ControlHandler,
20391 ctx: &RouteCtx,
20392 owner: &str,
20393 scope_ref: &str,
20394 ) -> ModuleControlResponseToModule {
20395 let reply = call(
20396 handler,
20397 ctx,
20398 &ModuleControlRequestFromModule::ScopeDescribe {
20399 owner: Principal::Reserved {
20400 module_id: owner.to_string(),
20401 },
20402 scope_ref: scope_ref.to_string(),
20403 },
20404 )
20405 .await;
20406 assert_eq!(
20407 reply.header.ty,
20408 FrameType::Response,
20409 "{:?}",
20410 parse_error(&reply)
20411 );
20412 serde_json::from_slice(&reply.body).unwrap()
20413 }
20414
20415 #[tokio::test]
20416 async fn scope_describe_never_echoes_bind_proof_or_hash() {
20417 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
20418 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
20419 let mut rig = rig().await;
20420 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
20421 .await;
20422 let _bound = rig
20423 .bound(
20424 Some(AFT),
20425 PLEXUS,
20426 Some(
20427 rig.selector("s", Some(1))
20428 .with_bind_proof(Some(PROOF.into())),
20429 ),
20430 )
20431 .await;
20432 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
20433 let output = serde_json::to_string(&reply).unwrap();
20434 assert!(output.contains("\"status\":\"live\""));
20435 assert!(!output.contains(PROOF));
20436 assert!(!output.contains(HASH));
20437 }
20438
20439 async fn module(
20441 handler: &ControlHandler,
20442 connection: u64,
20443 module_id: &str,
20444 nonce: Option<&str>,
20445 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20446 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
20447 hello_via_sink(
20448 handler,
20449 &ctx,
20450 &mut rx,
20451 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
20452 )
20453 .await;
20454 (ctx, rx)
20455 }
20456
20457 #[tokio::test]
20460 async fn a_client_connection_cannot_sync_or_describe() {
20461 let handler = ControlHandler::new(Arc::new(Registry::default()));
20462 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
20463 for request in [
20464 ModuleControlRequestFromModule::ScopeSync {
20465 generation: 1,
20466 scopes: vec![head("s", 1)],
20467 },
20468 ModuleControlRequestFromModule::ScopeDescribe {
20469 owner: Principal::Direct,
20470 scope_ref: "s".to_string(),
20471 },
20472 ] {
20473 let reply = call(&handler, &ctx, &request).await;
20474 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
20475 }
20476 assert!(
20477 !handler
20478 .scopes
20479 .read()
20480 .unwrap()
20481 .describe(
20482 &Principal::Reserved {
20483 module_id: OWNER.to_string()
20484 },
20485 "s"
20486 )
20487 .owner_synced
20488 );
20489 }
20490
20491 #[tokio::test]
20494 async fn a_module_without_a_supervised_launch_cannot_sync() {
20495 let handler = ControlHandler::new(Arc::new(Registry::default()));
20496 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
20497 assert_eq!(
20498 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
20499 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20500 );
20501 }
20502
20503 #[tokio::test]
20504 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
20505 let supervisor = SupervisorHandle::new();
20506 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20507 let handler = ControlHandler::new(Arc::new(Registry::default()))
20508 .with_supervisor(supervisor.clone());
20509 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20510 sync(&handler, &incumbent, 1, vec![head("s", 1)])
20511 .await
20512 .expect("the current launch syncs");
20513
20514 supervisor.open_swap(OWNER, "n2".to_string());
20517 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
20518 assert_eq!(
20519 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20520 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20521 );
20522 sync(&handler, &incumbent, 2, vec![head("s", 1)])
20523 .await
20524 .expect("the serving owner syncs during the swap");
20525
20526 supervisor.close_swap(OWNER);
20529 assert_eq!(
20530 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20531 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20532 );
20533 sync(&handler, &incumbent, 3, vec![head("s", 1)])
20534 .await
20535 .expect("the serving owner syncs after the rollback");
20536 handler.cleanup_connection(candidate.connection_id).unwrap();
20537
20538 supervisor.open_swap(OWNER, "n3".to_string());
20543 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
20544 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
20545 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
20546 .await
20547 .expect("the promoted launch takes authority");
20548 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20549 panic!("unexpected reply {reply:?}");
20550 };
20551 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
20552 assert_eq!(
20553 sync(&handler, &incumbent, 4, Vec::new()).await,
20554 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20555 );
20556 }
20557
20558 #[tokio::test]
20561 async fn closing_the_authority_connection_frees_sync_authority() {
20562 let supervisor = SupervisorHandle::new();
20563 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20564 let handler = ControlHandler::new(Arc::new(Registry::default()))
20565 .with_supervisor(supervisor.clone());
20566 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20567 sync(&handler, &first, 10, vec![head("s", 1)])
20568 .await
20569 .unwrap();
20570 handler.cleanup_connection(first.connection_id).unwrap();
20571
20572 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
20573 sync(&handler, &second, 1, vec![head("s", 1)])
20574 .await
20575 .expect("the next connection takes the released authority");
20576 }
20577
20578 #[tokio::test]
20579 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
20580 let supervisor = SupervisorHandle::new();
20581 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20582 let handler =
20583 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
20584 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
20585 sync(&handler, &first, 10, vec![head("s", 1)])
20586 .await
20587 .unwrap();
20588 handler
20589 .handle_control_frame(
20590 &first,
20591 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
20592 )
20593 .await
20594 .unwrap();
20595 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
20596 sync(&handler, &second, 1, vec![head("s", 1)])
20597 .await
20598 .expect("GOODBYE releases authority even if the old socket remains open");
20599 }
20600
20601 #[tokio::test]
20602 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
20603 let registry = Arc::new(Registry::default());
20604 let supervisor_handle = SupervisorHandle::new();
20605 let supervisor =
20606 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20607 .with_handle(supervisor_handle.clone())
20608 .with_daemon_incarnation("incarnation-7".to_string());
20609 supervisor
20612 .supervise_configured(
20613 ModuleSpec {
20614 script: None,
20615 carry_containment: false,
20616 module_id: OWNER.to_string(),
20617 program: PathBuf::from("/nonexistent/prefrontal-core"),
20618 args: Vec::new(),
20619 env: Vec::new(),
20620 reserved: false,
20621 reserved_prefixes: Vec::new(),
20622 protocol: ModuleProtocol::Subc,
20623 overlap: Default::default(),
20624 },
20625 false,
20626 )
20627 .unwrap();
20628 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
20629 let handler =
20630 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
20631 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
20632
20633 let ModuleControlResponseToModule::ScopeDescribe {
20635 status,
20636 daemon_incarnation,
20637 owner_synced,
20638 owner_configured,
20639 scope,
20640 ..
20641 } = describe(&handler, &reader, OWNER, "s").await
20642 else {
20643 panic!("not a describe reply");
20644 };
20645 assert_eq!(status, ScopeStatus::NotLive);
20646 assert_eq!(daemon_incarnation, "incarnation-7");
20647 assert!(!owner_synced);
20648 assert!(owner_configured);
20649 assert!(scope.is_none());
20650
20651 let ModuleControlResponseToModule::ScopeDescribe {
20654 status,
20655 owner_configured,
20656 ..
20657 } = describe(&handler, &reader, "ghost", "s").await
20658 else {
20659 panic!("not a describe reply");
20660 };
20661 assert_eq!(status, ScopeStatus::NotLive);
20662 assert!(!owner_configured);
20663
20664 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
20666 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
20667 let ModuleControlResponseToModule::ScopeDescribe {
20668 status,
20669 scope_epoch,
20670 owner_synced,
20671 scope,
20672 ..
20673 } = describe(&handler, &reader, OWNER, "s").await
20674 else {
20675 panic!("not a describe reply");
20676 };
20677 assert_eq!(status, ScopeStatus::Live);
20678 assert_eq!(scope_epoch, Some(4));
20679 assert!(owner_synced);
20680 let stamp = scope.expect("a live scope carries its stamp");
20681 assert!(
20682 stamp.owner_authorized,
20683 "prefrontal-core is the default authority"
20684 );
20685 assert_eq!(stamp.kind, ScopeKind::Head);
20686 }
20687
20688 #[tokio::test]
20689 async fn scope_authority_owners_decides_owner_authorized() {
20690 let supervisor = SupervisorHandle::new();
20691 supervisor.set_spawn_nonce("broca", "b1".to_string());
20692 let handler = ControlHandler::new(Arc::new(Registry::default()))
20693 .with_supervisor(supervisor)
20694 .with_scope_authority_owners(vec!["broca".to_string()]);
20695 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
20696 let mut gated = head("s", 1);
20697 gated.attributes.agent_id = Some("agent".to_string());
20698 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
20699 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
20700 describe(&handler, &broca, "broca", "s").await
20701 else {
20702 panic!("not a describe reply");
20703 };
20704 assert!(scope.unwrap().owner_authorized);
20705 }
20706
20707 #[tokio::test]
20711 async fn scope_ops_and_the_scopes_capability_are_advertised() {
20712 let handler = ControlHandler::new(Arc::new(Registry::default()));
20713 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
20714 let ack = hello_via_sink(
20715 &handler,
20716 &ctx,
20717 &mut rx,
20718 hello_frame("m", PROTOCOL_VERSION, 1),
20719 )
20720 .await;
20721 let ack = parse_ack(&ack);
20722 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
20723 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
20724 }
20725 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
20726
20727 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
20728 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
20729 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
20730 let reply = handler
20731 .handle_control_frame(&client, frame)
20732 .await
20733 .unwrap()
20734 .pop()
20735 .unwrap();
20736 let ClientControlResponse::ServerDescribe { capabilities, .. } =
20737 serde_json::from_slice(&reply.body).unwrap()
20738 else {
20739 panic!("not a server.describe reply");
20740 };
20741 assert!(
20742 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
20743 "{capabilities:?}"
20744 );
20745 }
20746
20747 const PLEXUS: &str = "plexus";
20750 const OTHER: &str = "other";
20751 const AFT: &str = "aft";
20752 const BROCA: &str = "broca";
20753 const MAGIC: &str = "magic-context";
20754
20755 fn nonce(module_id: &str) -> String {
20756 format!("nonce-{module_id}")
20757 }
20758
20759 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20760 let (tx, rx) = mpsc::channel(64);
20761 (
20762 RouteCtx {
20763 connection_id: ConnectionId::new(connection),
20764 egress: FrameSink::new(tx),
20765 },
20766 rx,
20767 )
20768 }
20769
20770 struct Rig {
20774 handler: ControlHandler,
20775 forwarding: Arc<ForwardingTable>,
20776 owner: RouteCtx,
20777 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20778 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
20779 generation: u64,
20780 next_connection: u64,
20781 _supervisor: Supervisor,
20782 }
20783
20784 async fn rig() -> Rig {
20785 rig_with_flow_support(true).await
20786 }
20787
20788 async fn rig_with_flow_support(flow_support: bool) -> Rig {
20789 rig_with_scope_capabilities(flow_support, false).await
20790 }
20791
20792 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
20793 let mut provides = Vec::new();
20794 if flow_support {
20795 provides.push("flow-scopes/v1");
20796 }
20797 if run_support {
20798 provides.push("agent-run-scopes/v1");
20799 }
20800 rig_with_target_capabilities(&provides).await
20801 }
20802
20803 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
20804 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
20805 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20806 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20809 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
20810 Frame::build(
20811 FrameType::Hello,
20812 control_flags(),
20813 0,
20814 0,
20815 connection,
20816 serde_json::to_vec(&body).unwrap(),
20817 )
20818 .unwrap()
20819 }
20820
20821 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
20822 let registry = Arc::new(Registry::default());
20823 let forwarding = Arc::new(ForwardingTable::default());
20824 let supervisor_handle = SupervisorHandle::new();
20825 let supervisor =
20826 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
20827 .with_handle(supervisor_handle.clone());
20828 supervisor
20829 .supervise_configured(
20830 ModuleSpec {
20831 script: None,
20832 carry_containment: false,
20833 module_id: OWNER.to_string(),
20834 program: PathBuf::from("/nonexistent/prefrontal-core"),
20835 args: Vec::new(),
20836 env: Vec::new(),
20837 reserved: false,
20838 reserved_prefixes: Vec::new(),
20839 protocol: ModuleProtocol::Subc,
20840 overlap: Default::default(),
20841 },
20842 false,
20843 )
20844 .unwrap();
20845 for module_id in [OWNER, AFT, BROCA, MAGIC] {
20846 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
20847 }
20848 let handler =
20849 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
20850 .with_supervisor(supervisor_handle);
20851 let (owner, mut owner_rx) = wide_ctx(1);
20852 hello_via_sink(
20853 &handler,
20854 &owner,
20855 &mut owner_rx,
20856 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
20857 )
20858 .await;
20859 let mut modules = BTreeMap::new();
20860 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
20861 let (ctx, mut rx) = wide_ctx(connection);
20862 let hello = hello_with_capabilities(module_id, connection, provides);
20863 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
20864 modules.insert(module_id.to_string(), (ctx, rx));
20865 }
20866 Rig {
20867 handler,
20868 forwarding,
20869 owner,
20870 _owner_rx: owner_rx,
20871 modules,
20872 generation: 0,
20873 next_connection: 100,
20874 _supervisor: supervisor,
20875 }
20876 }
20877
20878 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
20879 ScopeCarrier::new(Principal::Reserved {
20880 module_id: module_id.to_string(),
20881 })
20882 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
20883 }
20884
20885 fn session(scope_epoch: u64) -> ScopeRecord {
20888 let mut record = head("s", scope_epoch);
20889 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
20890 record.attributes.agent_id = Some("agent-1".to_string());
20891 record.attributes.delegates = true;
20892 record
20893 }
20894
20895 impl Rig {
20896 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
20897 self.generation += 1;
20898 sync(&self.handler, &self.owner, self.generation, scopes)
20899 .await
20900 .expect("the owner's sync is accepted");
20901 }
20902
20903 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
20904 ScopeSelector::new(
20905 Principal::Reserved {
20906 module_id: OWNER.to_string(),
20907 },
20908 scope_ref.to_string(),
20909 scope_epoch,
20910 )
20911 }
20912
20913 fn open_frame(
20914 &mut self,
20915 opener: Option<&str>,
20916 target: &str,
20917 scope: Option<ScopeSelector>,
20918 ) -> (
20919 RouteCtx,
20920 mpsc::Receiver<crate::router::OutboundFrame>,
20921 Frame,
20922 ) {
20923 self.next_connection += 1;
20924 let (ctx, rx) = wide_ctx(self.next_connection);
20925 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
20926 busy_refusals: None,
20927 target: RouteTarget::ToolProvider {
20928 module_id: target.to_string(),
20929 },
20930 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
20931 consumer_identity: opener.map(|module_id| ConsumerIdentity {
20932 module_id: module_id.to_string(),
20933 launch_nonce: nonce(module_id),
20934 }),
20935 consumer_capabilities: None,
20936 role_versions: None,
20937 admission_facts: None,
20938 scope,
20939 })
20940 .unwrap();
20941 let frame = Frame::build(
20942 FrameType::Request,
20943 control_flags(),
20944 0,
20945 0,
20946 self.next_connection,
20947 body,
20948 )
20949 .unwrap();
20950 (ctx, rx, frame)
20951 }
20952
20953 async fn refused(
20955 &mut self,
20956 opener: Option<&str>,
20957 target: &str,
20958 scope: Option<ScopeSelector>,
20959 ) -> String {
20960 self.refusal_body(opener, target, scope).await["code"]
20961 .as_str()
20962 .unwrap()
20963 .to_string()
20964 }
20965
20966 async fn refusal_body(
20967 &mut self,
20968 opener: Option<&str>,
20969 target: &str,
20970 scope: Option<ScopeSelector>,
20971 ) -> Value {
20972 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
20973 let replies = tokio::time::timeout(
20974 Duration::from_secs(2),
20975 self.handler.handle_control_frame(&ctx, frame),
20976 )
20977 .await
20978 .expect("the open must be refused before waiting for a bind ack")
20979 .unwrap();
20980 assert_eq!(replies.len(), 1, "{replies:?}");
20981 assert_eq!(replies[0].header.ty, FrameType::Error);
20982 let (_, module_rx) = self.modules.get_mut(target).unwrap();
20983 assert!(
20984 module_rx.try_recv().is_err(),
20985 "a refused open relays nothing"
20986 );
20987 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
20988 parse_error(&replies[0])
20989 }
20990
20991 async fn relayed(
20993 &mut self,
20994 opener: Option<&str>,
20995 target: &str,
20996 scope: Option<ScopeSelector>,
20997 ) -> Relayed {
20998 self.relayed_with_legacy_root(opener, target, scope, None)
20999 .await
21000 }
21001
21002 async fn relayed_with_legacy_root(
21003 &mut self,
21004 opener: Option<&str>,
21005 target: &str,
21006 scope: Option<ScopeSelector>,
21007 legacy_root: Option<&std::path::Path>,
21008 ) -> Relayed {
21009 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
21010 if let Some(root) = legacy_root {
21011 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
21012 body["identity"]["project_root"] = serde_json::json!(root);
21013 frame.body = serde_json::to_vec(&body).unwrap();
21014 }
21015 let handler = self.handler.clone();
21016 let task_ctx = ctx.clone();
21017 let task = tokio::spawn(async move {
21018 handler
21019 .handle_control_frame(&task_ctx, frame)
21020 .await
21021 .unwrap()
21022 });
21023 let (_, module_rx) = self.modules.get_mut(target).unwrap();
21024 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
21025 .await
21026 .expect("the target receives the relayed route.bind")
21027 .unwrap()
21028 .frame;
21029 Relayed {
21030 target: target.to_string(),
21031 client: ctx,
21032 client_rx: rx,
21033 task,
21034 bind,
21035 }
21036 }
21037
21038 async fn ack(&self, relayed: &Relayed) {
21039 let (module, _) = &self.modules[&relayed.target];
21040 self.handler
21041 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
21042 .await
21043 .unwrap();
21044 }
21045
21046 async fn bound(
21048 &mut self,
21049 opener: Option<&str>,
21050 target: &str,
21051 scope: Option<ScopeSelector>,
21052 ) -> Bound {
21053 let relayed = self.relayed(opener, target, scope).await;
21054 self.commit_relayed(relayed).await
21055 }
21056
21057 async fn bound_with_legacy_root(
21058 &mut self,
21059 opener: Option<&str>,
21060 target: &str,
21061 scope: Option<ScopeSelector>,
21062 root: &std::path::Path,
21063 ) -> Bound {
21064 let relayed = self
21065 .relayed_with_legacy_root(opener, target, scope, Some(root))
21066 .await;
21067 self.commit_relayed(relayed).await
21068 }
21069
21070 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
21071 self.ack(&relayed).await;
21072 let Relayed {
21073 target,
21074 client,
21075 mut client_rx,
21076 task,
21077 bind,
21078 } = relayed;
21079 assert!(
21080 task.await.unwrap().is_empty(),
21081 "the open is answered by commit"
21082 );
21083 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
21084 Bound {
21085 target,
21086 client,
21087 client_rx,
21088 channel,
21089 epoch,
21090 bind,
21091 }
21092 }
21093
21094 fn live(&self, route: &Bound) -> bool {
21095 matches!(
21096 self.forwarding
21097 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21098 .unwrap(),
21099 DataRoute::Client(DataRouteState::Bound(_))
21100 )
21101 }
21102 }
21103
21104 struct Relayed {
21105 target: String,
21106 client: RouteCtx,
21107 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21108 task: tokio::task::JoinHandle<Vec<Frame>>,
21109 bind: Frame,
21110 }
21111
21112 struct Bound {
21113 target: String,
21114 client: RouteCtx,
21115 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21116 channel: u16,
21117 epoch: u32,
21118 bind: Frame,
21119 }
21120
21121 impl Bound {
21122 fn closed_reason(&mut self) -> RouteCloseReason {
21125 let mut reason = None;
21126 let mut goodbye = false;
21127 while let Ok(outbound) = self.client_rx.try_recv() {
21128 let frame = outbound.frame;
21129 match frame.header.ty {
21130 FrameType::Goodbye => {
21131 assert_eq!(
21132 (frame.header.channel, frame.header.epoch),
21133 (self.channel, self.epoch)
21134 );
21135 goodbye = true;
21136 }
21137 FrameType::Push => {
21138 let ClientControlPush::RouteClosed {
21139 reason: r,
21140 module_id,
21141 ..
21142 } = serde_json::from_slice(&frame.body).unwrap()
21143 else {
21144 panic!("unexpected push");
21145 };
21146 assert_eq!(module_id, self.target);
21147 reason = Some(r);
21148 }
21149 other => panic!("unexpected frame {other:?}"),
21150 }
21151 }
21152 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
21153 reason.expect("the client is told why the route closed")
21154 }
21155
21156 fn untouched(&mut self) -> bool {
21157 self.client_rx.try_recv().is_err()
21158 }
21159
21160 fn stamp(&self) -> Option<ScopeStamp> {
21161 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
21162 ModuleControlRequest::RouteBind { scope, .. } => scope,
21163 other => panic!("expected a route.bind, got {other:?}"),
21164 }
21165 }
21166 }
21167
21168 #[tokio::test]
21169 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
21170 ) {
21171 let mut rig = rig().await;
21172 let mut record = session(1);
21173 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
21174 record.child_owners = vec![Principal::Reserved {
21175 module_id: MAGIC.to_string(),
21176 }];
21177 rig.sync(vec![record]).await;
21178 let scope = || Some(rig_selector("s", Some(1)));
21179
21180 rig.bound(Some(OWNER), PLEXUS, scope()).await;
21183 rig.bound(Some(AFT), OTHER, scope()).await;
21184 rig.bound(Some(BROCA), PLEXUS, scope()).await;
21185
21186 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
21190 assert_eq!(
21191 rig.refused(opener, target, scope()).await,
21192 error_codes::SCOPE_NOT_CARRIER,
21193 "{opener:?} -> {target}"
21194 );
21195 }
21196 }
21197
21198 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
21199 ScopeSelector::new(
21200 Principal::Reserved {
21201 module_id: OWNER.to_string(),
21202 },
21203 scope_ref.to_string(),
21204 scope_epoch,
21205 )
21206 }
21207
21208 #[tokio::test]
21209 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
21210 let mut rig = rig().await;
21211 rig.sync(vec![session(1)]).await;
21212 for opener in [OWNER, AFT] {
21213 assert_eq!(
21214 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
21215 .await,
21216 error_codes::SCOPE_EPOCH_REQUIRED,
21217 "{opener}"
21218 );
21219 }
21220 }
21221
21222 #[tokio::test]
21223 async fn admission_separates_not_synced_not_live_and_ended() {
21224 let mut rig = rig().await;
21225 let code = rig
21227 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21228 .await;
21229 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
21230 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
21231
21232 let ghost = ScopeSelector::new(
21234 Principal::Reserved {
21235 module_id: "ghost".to_string(),
21236 },
21237 "s".to_string(),
21238 Some(1),
21239 );
21240 assert_eq!(
21241 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
21242 error_codes::SCOPE_NOT_LIVE
21243 );
21244
21245 rig.sync(vec![session(2)]).await;
21246 assert_eq!(
21247 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
21248 .await,
21249 error_codes::SCOPE_NOT_LIVE
21250 );
21251 for epoch in [1, 3] {
21252 assert_eq!(
21253 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
21254 .await,
21255 error_codes::SCOPE_ENDED,
21256 "epoch {epoch}"
21257 );
21258 }
21259 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
21261 .await;
21262 }
21263
21264 #[tokio::test]
21265 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
21266 let mut rig = rig().await;
21267 rig.sync(vec![session(1)]).await;
21268 let route = rig
21269 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21270 .await;
21271 let stamp = route.stamp().expect("a scoped bind carries the stamp");
21272 assert_eq!(stamp.scope_ref, "s");
21273 assert_eq!(stamp.scope_epoch, 1);
21274 assert_eq!(stamp.kind, ScopeKind::Head);
21275 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
21276 assert!(stamp.attributes.delegates);
21277 assert!(stamp.owner_authorized);
21278 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21279 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
21280
21281 let (broca, mut broca_rx) = wide_ctx(50);
21284 hello_via_sink(
21285 &rig.handler,
21286 &broca,
21287 &mut broca_rx,
21288 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
21289 )
21290 .await;
21291 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
21292 .await
21293 .unwrap();
21294 let own = ScopeSelector::new(
21295 Principal::Reserved {
21296 module_id: BROCA.to_string(),
21297 },
21298 "b".to_string(),
21299 Some(1),
21300 );
21301 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
21302 assert!(!route.stamp().unwrap().owner_authorized);
21303 }
21304
21305 #[tokio::test]
21306 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
21307 let mut rig = rig().await;
21308 let mut record = head("s", 1);
21309 record.carriers = vec![carrier(AFT, None)];
21310 let flow_id = "Flow:run-7/step_2!~";
21311 record.attributes.flow_id = Some(flow_id.to_string());
21312 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
21313 .await
21314 .unwrap();
21315 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
21316 panic!("not a sync reply");
21317 };
21318 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
21319 let route = rig
21320 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21321 .await;
21322 assert!(rig.live(&route), "the stamped bind committed");
21323 let stamp = route.stamp().expect("a flow scope carries a stamp");
21324 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
21325 assert_eq!(stamp.attributes.agent_id, None);
21326 assert!(!stamp.attributes.delegates);
21327 assert!(stamp.owner_authorized);
21328 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21329 assert_eq!(unscoped.stamp(), None);
21330 }
21331
21332 #[tokio::test]
21333 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
21334 let mut rig = rig_with_flow_support(false).await;
21335 let mut record = session(1);
21336 record.attributes.flow_id = Some("flow:7".to_string());
21337 rig.sync(vec![record]).await;
21338 for opener in [OWNER, AFT] {
21339 let body = rig
21340 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21341 .await;
21342 assert_eq!(body["code"], "target_flow_unsupported");
21343 let message = body["message"].as_str().unwrap();
21344 for required in [PLEXUS, "flow-scopes/v1"] {
21345 assert!(message.contains(required), "{message}");
21346 }
21347 }
21348 }
21349
21350 #[tokio::test]
21351 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
21352 let mut rig = rig_with_flow_support(true).await;
21353 let mut record = session(1);
21354 record.attributes.flow_id = Some("flow:7".to_string());
21355 rig.sync(vec![record]).await;
21356 for opener in [OWNER, AFT] {
21357 let route = rig
21358 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21359 .await;
21360 assert!(rig.live(&route));
21361 assert_eq!(
21362 route.stamp().unwrap().attributes.flow_id.as_deref(),
21363 Some("flow:7")
21364 );
21365 }
21366 }
21367
21368 #[tokio::test]
21369 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
21370 use std::future::Future;
21371
21372 let mut rig = rig_with_flow_support(true).await;
21373 let mut record = session(1);
21374 record.attributes.flow_id = Some("flow:7".to_string());
21375 rig.sync(vec![record]).await;
21376 let (client, mut client_rx, frame) =
21377 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
21378 for _ in 0..64 {
21381 client.egress.try_send(route_bind_ack(1)).unwrap();
21382 }
21383 let handler = rig.handler.clone();
21384 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21385 std::future::poll_fn(|cx| {
21386 assert!(open.as_mut().poll(cx).is_pending());
21387 std::task::Poll::Ready(())
21388 })
21389 .await;
21390 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21391
21392 let old_connection = rig.modules[PLEXUS].0.connection_id;
21393 rig.handler.cleanup_connection(old_connection).unwrap();
21394 let (replacement, mut replacement_rx) = wide_ctx(200);
21395 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
21396 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
21397 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
21398 let hello = Frame::build(
21399 FrameType::Hello,
21400 control_flags(),
21401 0,
21402 0,
21403 200,
21404 serde_json::to_vec(&body).unwrap(),
21405 )
21406 .unwrap();
21407 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
21408
21409 client_rx.try_recv().unwrap();
21410 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21411 .await
21412 .expect("the replacement is refused without waiting for a bind ack")
21413 .unwrap();
21414 assert_eq!(replies.len(), 1);
21415 let body = parse_error(&replies[0]);
21416 assert_eq!(body["code"], "target_flow_unsupported");
21417 for required in [PLEXUS, "flow-scopes/v1"] {
21418 assert!(body["message"].as_str().unwrap().contains(required));
21419 }
21420 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
21421 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21422 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21423 assert!(rig
21424 .handler
21425 .registry
21426 .get_module_by_connection(replacement.connection_id)
21427 .unwrap()
21428 .is_some());
21429 }
21430
21431 #[tokio::test]
21432 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
21433 use std::future::Future;
21434 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
21435 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21436 const NEXT_HASH: &str =
21437 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21438 let mut rig = rig().await;
21439 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
21440 record.carriers.push(ScopeCarrier::new(Principal::Direct));
21441 rig.sync(vec![record.clone()]).await;
21442 let (client, mut rx, frame) = rig.open_frame(
21443 None,
21444 PLEXUS,
21445 Some(
21446 rig.selector("s", Some(1))
21447 .with_bind_proof(Some(PROOF.into())),
21448 ),
21449 );
21450 for _ in 0..64 {
21453 client.egress.try_send(route_bind_ack(1)).unwrap();
21454 }
21455 let handler = rig.handler.clone();
21456 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21457 std::future::poll_fn(|cx| {
21458 assert!(open.as_mut().poll(cx).is_pending());
21459 std::task::Poll::Ready(())
21460 })
21461 .await;
21462 record.bind_proof_sha256 = Some(NEXT_HASH.into());
21463 rig.sync(vec![record]).await;
21464 rx.try_recv().unwrap();
21465 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21466 .await
21467 .unwrap()
21468 .unwrap();
21469 assert_eq!(
21470 parse_error(&replies[0])["code"],
21471 error_codes::SCOPE_BIND_PROOF_MISMATCH
21472 );
21473 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
21474 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21475 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21476 }
21477
21478 #[tokio::test]
21479 async fn scope_bind_proof_reserved_openers_ignore_proof() {
21480 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21481 let mut rig = rig().await;
21482 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21483 .await;
21484 for opener in [OWNER, AFT] {
21485 for proof in [None, Some("malformed+proof=")] {
21486 let route = rig
21487 .bound(
21488 Some(opener),
21489 PLEXUS,
21490 Some(
21491 rig.selector("s", Some(1))
21492 .with_bind_proof(proof.map(str::to_string)),
21493 ),
21494 )
21495 .await;
21496 assert!(rig.live(&route));
21497 }
21498 }
21499 }
21500
21501 #[tokio::test]
21502 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
21503 use std::future::Future;
21504 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21505 const NEXT_HASH: &str =
21506 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21507 let mut rig = rig().await;
21508 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21509 .await;
21510 let (client, mut rx, frame) =
21511 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
21512 for _ in 0..64 {
21513 client.egress.try_send(route_bind_ack(1)).unwrap();
21514 }
21515 let handler = rig.handler.clone();
21516 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21517 std::future::poll_fn(|cx| {
21518 assert!(open.as_mut().poll(cx).is_pending());
21519 std::task::Poll::Ready(())
21520 })
21521 .await;
21522 rig.sync(vec![
21523 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
21524 ])
21525 .await;
21526 rx.try_recv().unwrap();
21527 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21528 let bind = std::future::poll_fn(|cx| {
21529 assert!(open.as_mut().poll(cx).is_pending());
21530 module_rx.poll_recv(cx)
21531 })
21532 .await
21533 .unwrap()
21534 .frame;
21535 handler
21536 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
21537 .await
21538 .unwrap();
21539 let replies = tokio::time::timeout(Duration::from_secs(2), open)
21540 .await
21541 .unwrap()
21542 .unwrap();
21543 assert_eq!(replies.len(), 1);
21544 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
21547 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21548 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
21549 let route = rig
21550 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
21551 .await;
21552 assert!(rig.live(&route));
21553 }
21554
21555 #[tokio::test]
21556 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
21557 {
21558 let mut rig = rig_with_flow_support(false).await;
21559 rig.sync(vec![session(1)]).await;
21560 let route = rig
21561 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21562 .await;
21563 assert!(rig.live(&route));
21564 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
21565 let mut record = session(1);
21566 record.attributes.flow_id = Some("flow:7".to_string());
21567 rig.sync(vec![record]).await;
21568 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
21569 assert!(rig.live(&unscoped));
21570 assert_eq!(unscoped.stamp(), None);
21571 }
21572
21573 #[tokio::test]
21574 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
21575 let mut rig = rig().await;
21576 let mut record = session(1);
21577 record.attributes.flow_id = Some("flow:7".to_string());
21578 rig.sync(vec![record.clone()]).await;
21579 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21580 let mut owner_route = rig
21581 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21582 .await;
21583 let mut carrier_route = rig
21584 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
21585 .await;
21586 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21587 rig.sync(vec![record.clone()]).await;
21588 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21589 assert!(rig.live(&owner_route) && owner_route.untouched());
21590 assert!(rig.live(&carrier_route) && carrier_route.untouched());
21591
21592 record.attributes.flow_id = Some("flow:8".to_string());
21593 rig.sync(vec![record]).await;
21594 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
21595 let before = before.unwrap();
21596 assert_eq!(after.scope_epoch, before.scope_epoch);
21597 assert!(after.version > before.version);
21598 for route in [&mut owner_route, &mut carrier_route] {
21599 assert!(!rig.live(route));
21600 assert_eq!(
21601 route.closed_reason(),
21602 RouteCloseReason::ScopeDelegationChanged
21603 );
21604 }
21605 assert!(rig.live(&unscoped) && unscoped.untouched());
21606 for target in [PLEXUS, OTHER] {
21609 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
21610 let goodbye = module_rx
21611 .try_recv()
21612 .expect("the provider sees the drain")
21613 .frame;
21614 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21615 assert!(module_rx.try_recv().is_err());
21616 }
21617 let rebound = rig
21618 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21619 .await;
21620 assert_eq!(
21621 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
21622 Some("flow:8")
21623 );
21624 }
21625
21626 #[tokio::test]
21631 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
21632 let mut rig = rig().await;
21633 rig.sync(vec![session(1)]).await;
21634 let mut cotenant = rig
21635 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21636 .await;
21637
21638 let mut changed = session(1);
21639 changed.child_owners.push(Principal::Reserved {
21640 module_id: MAGIC.to_string(),
21641 });
21642 let mut ended = None;
21643 for (code, next) in [
21644 (error_codes::SCOPE_CHANGED, vec![changed]),
21645 (error_codes::SCOPE_ENDED, Vec::new()),
21646 ] {
21647 let relayed = rig
21648 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21649 .await;
21650 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
21651 ended = Some(next.is_empty());
21652 rig.sync(next).await;
21653 rig.ack(&relayed).await;
21654 let replies = relayed.task.await.unwrap();
21655 assert_eq!(replies.len(), 1, "{replies:?}");
21656 assert_eq!(parse_error(&replies[0])["code"], code);
21657 assert_eq!(
21658 subc_protocol::error_codes::is_retryable_route_open(code),
21659 code == error_codes::SCOPE_CHANGED
21660 );
21661 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21662 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21664 let mut goodbyes = Vec::new();
21667 while let Ok(outbound) = plexus_rx.try_recv() {
21668 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
21669 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
21670 }
21671 assert!(
21672 goodbyes.contains(&(bind_channel, bind_epoch)),
21673 "{goodbyes:?}"
21674 );
21675 assert!(rig
21676 .handler
21677 .registry
21678 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
21679 .unwrap()
21680 .is_some());
21681 }
21682 assert_eq!(ended, Some(true));
21683 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
21686 }
21687
21688 #[tokio::test]
21691 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
21692 struct Case {
21693 name: &'static str,
21694 change: fn(&mut ScopeRecord),
21695 closed: [Option<RouteCloseReason>; 4],
21698 }
21699 use RouteCloseReason::*;
21700 let cases = [
21701 Case {
21702 name: "a carrier entry removed",
21703 change: |r| {
21704 r.carriers.retain(|c| {
21705 c.principal
21706 != Principal::Reserved {
21707 module_id: AFT.to_string(),
21708 }
21709 })
21710 },
21711 closed: [None, Some(ScopeCarrierRemoved), None, None],
21712 },
21713 Case {
21714 name: "a target removed from a carrier",
21715 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
21716 closed: [None, None, None, Some(ScopeCarrierRemoved)],
21717 },
21718 Case {
21719 name: "a bare carrier narrowed to targets",
21720 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
21721 closed: [None, Some(ScopeCarrierRemoved), None, None],
21722 },
21723 Case {
21724 name: "delegates turned off",
21725 change: |r| r.attributes.delegates = false,
21726 closed: [Some(ScopeDelegationChanged); 4],
21727 },
21728 Case {
21729 name: "agent_id changed",
21730 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
21731 closed: [Some(ScopeDelegationChanged); 4],
21732 },
21733 Case {
21734 name: "a carrier added, child owners changed, the record re-sent",
21735 change: |r| {
21736 r.carriers.push(carrier(MAGIC, None));
21737 r.child_owners.push(Principal::Reserved {
21738 module_id: MAGIC.to_string(),
21739 });
21740 },
21741 closed: [None; 4],
21742 },
21743 Case {
21744 name: "a target added",
21745 change: |r| {
21746 r.carriers[1]
21747 .targets
21748 .as_mut()
21749 .unwrap()
21750 .push("third".to_string())
21751 },
21752 closed: [None; 4],
21753 },
21754 Case {
21755 name: "delegates turned on",
21756 change: |r| r.attributes.delegates = true,
21757 closed: [None; 4],
21758 },
21759 ];
21760 for case in cases {
21761 let mut rig = rig().await;
21762 rig.sync(vec![session(1)]).await;
21763 let scope = || Some(rig_selector("s", Some(1)));
21764 let mut routes = [
21765 rig.bound(Some(OWNER), PLEXUS, scope()).await,
21766 rig.bound(Some(AFT), PLEXUS, scope()).await,
21767 rig.bound(Some(BROCA), PLEXUS, scope()).await,
21768 rig.bound(Some(BROCA), OTHER, scope()).await,
21769 ];
21770 let mut record = session(1);
21771 (case.change)(&mut record);
21772 rig.sync(vec![record]).await;
21773 for (index, expected) in case.closed.iter().enumerate() {
21774 let route = &mut routes[index];
21775 match expected {
21776 Some(reason) => {
21777 assert!(!rig.live(route), "{}: route {index} still live", case.name);
21778 assert_eq!(
21779 route.closed_reason(),
21780 *reason,
21781 "{}: route {index}",
21782 case.name
21783 );
21784 }
21785 None => {
21786 assert!(rig.live(route), "{}: route {index} closed", case.name);
21787 assert!(
21788 route.untouched(),
21789 "{}: route {index} was told something",
21790 case.name
21791 );
21792 }
21793 }
21794 }
21795 }
21796 }
21797
21798 #[tokio::test]
21799 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
21800 for next in [Vec::new(), vec![session(2)]] {
21802 let mut rig = rig().await;
21803 rig.sync(vec![session(1)]).await;
21804 let mut route = rig
21805 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21806 .await;
21807 rig.sync(next).await;
21808 assert!(!rig.live(&route));
21809 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
21810 }
21811
21812 let mut rig = rig().await;
21815 let mut child = session(1);
21816 child.scope_ref = "child".to_string();
21817 child.kind = ScopeKind::Worker;
21818 child.parent = Some(ScopeParent::new(
21819 Principal::Reserved {
21820 module_id: OWNER.to_string(),
21821 },
21822 "s".to_string(),
21823 1,
21824 ));
21825 rig.sync(vec![session(1), child.clone()]).await;
21826 let mut child_route = rig
21827 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
21828 .await;
21829 assert_eq!(
21830 child_route.stamp().unwrap().parent_state,
21831 Some(ParentState::Linked)
21832 );
21833 rig.sync(vec![child]).await;
21834 assert!(!rig.live(&child_route));
21835 assert_eq!(
21836 child_route.closed_reason(),
21837 RouteCloseReason::ScopeParentEnded
21838 );
21839 }
21840
21841 #[tokio::test]
21842 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
21843 ) {
21844 let mut rig = rig().await;
21845 rig.sync(vec![session(1)]).await;
21846 let mut route = rig
21847 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21848 .await;
21849 let before = rig.forwarding.published_scope_tag(OWNER, "s");
21850 rig.sync(vec![session(1)]).await;
21851 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21852 assert!(rig.live(&route) && route.untouched());
21853
21854 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
21858 .forwarding
21859 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21860 .unwrap()
21861 else {
21862 panic!("the route is bound");
21863 };
21864 binding.flow.acquire_tagged(9, false).await.unwrap();
21865 let mut widened = session(1);
21866 widened.carriers.push(carrier(MAGIC, None));
21867 rig.sync(vec![widened]).await;
21868 assert!(rig.live(&route) && route.untouched());
21869 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21870 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
21871 assert_eq!(binding.flow.in_flight(), 1);
21874 binding
21875 .flow
21876 .acquire_tagged(10, false)
21877 .await
21878 .expect("the flow is still open");
21879 }
21880
21881 #[tokio::test]
21884 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
21885 let mut rig = rig().await;
21886 rig.sync(vec![session(1)]).await;
21887 let mut on_incumbent = rig
21888 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21889 .await;
21890
21891 let (candidate, _candidate_rx) = wide_ctx(9);
21894 let registration = rig
21895 .handler
21896 .registry
21897 .register_candidate_with_control_ops(
21898 manifest(PLEXUS, PROTOCOL_VERSION),
21899 PROTOCOL_VERSION,
21900 candidate.connection_id,
21901 module_baseline_control_ops(),
21902 )
21903 .unwrap();
21904 rig.forwarding
21905 .register_candidate_module_connection(
21906 candidate.connection_id,
21907 PLEXUS.to_string(),
21908 PROTOCOL_VERSION,
21909 manifest_concurrency(®istration.manifest),
21910 candidate.egress.clone(),
21911 )
21912 .unwrap();
21913 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
21914 rig.handler
21915 .registry
21916 .promote_candidate(PLEXUS)
21917 .unwrap()
21918 .unwrap();
21919 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
21920
21921 rig.sync(Vec::new()).await;
21922 assert!(!rig.live(&on_incumbent));
21923 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
21924 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21925 let goodbye = incumbent_rx
21926 .try_recv()
21927 .expect("the superseded endpoint is told")
21928 .frame;
21929 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21930 }
21931 }
21932}
21933
21934#[cfg(test)]
21935mod concurrency_default_exposure_tests {
21936 use super::*;
21937
21938 fn hello_body(role_json: &str) -> Vec<u8> {
21939 format!(
21940 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
21941 )
21942 .into_bytes()
21943 }
21944
21945 fn manifest_from(body: &[u8]) -> ModuleManifest {
21946 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
21947 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
21948 .expect("manifest parses")
21949 }
21950
21951 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
21952
21953 #[test]
21954 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
21955 let body = hello_body(&format!(
21956 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
21957 ));
21958 let manifest = manifest_from(&body);
21959 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21962 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
21963 }
21964
21965 #[test]
21966 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
21967 let body = hello_body(&format!(
21968 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
21969 ));
21970 let manifest = manifest_from(&body);
21971 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21972 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21973 }
21974
21975 #[test]
21976 fn non_management_roles_are_never_reported() {
21977 let body = hello_body(
21978 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
21979 );
21980 let manifest = manifest_from(&body);
21981 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21982 }
21983}