1#![forbid(unsafe_code)]
2
3pub mod activity;
4mod agent_commands;
5pub mod config;
6mod control_ops;
7pub use agent_commands::AgentCommandsError;
8pub use subc_protocol::session::{AgentCommandFinished, AgentCommandRoot};
9pub mod consumer;
10mod operator_app;
11pub mod policy_cache;
12#[cfg(any(test, feature = "test-support"))]
13mod stand_in;
14#[cfg(feature = "test-support")]
15pub mod test_support;
16pub use activity::*;
17pub use consumer::{
18 is_retryable_route_open_code, AppOpenError, CallError, CallOptions, CallReply, CancelHandle,
19 CancelOutcome, CatalogList, CatalogSubscription, CloseRouteOptions, ConfigApplyResponse,
20 ConfigEdit, ConfigEditAction, ConfigFileError, ConfigModule, ConfigModulesResponse,
21 ConfigPlanRefusal, ConfigPlanView, ConfigSchemaResponse, ConfigSettingsLayer,
22 ConfigShowResponse, ConfigSubscription, ConfigThrough, ConfigWatchEvent, ConnectionFacts,
23 ConnectionState, ConsumerCapabilityError, ConsumerError, ConsumerOptions, ControlPush,
24 CorrelationId, Digest, HealthProbeResponse, NotSentCause, OutcomeUnknownCause, PendingReply,
25 PushEvent, RequestBody, RequestReservation, RetryBackoff, ReverseRequestContext,
26 ReverseRequestError, ReverseRequestRegistrationError, ReverseRequestRegistry,
27 RouteCloseDisposition, RouteCloseReason, RouteEndReason, RouteOpenTarget, RouteOpenTargetKind,
28 RoutePollResult, ScopeBindProofRefusal, ScopeSelector, ServerDescription, SpawnStreamError,
29 SpawnSubscription, StderrTailResponse, SubcConsumer, SubscribeOptions, Subscription,
30 SubscriptionClosed, SubscriptionEnd, SupervisorAck, SupervisorEntry, SupervisorHealthResponse,
31 SupervisorListResponse, SupervisorProvenanceResponse, ACCEPTED_CLOSE_REASONS,
32 DEFAULT_CALL_TIMEOUT, DEFAULT_LIVENESS_PROBE_WINDOW, DEFAULT_ROUTE_RETRY_DEADLINE,
33 SPAWN_CURSOR_INCARNATION_MISMATCH, SPAWN_CURSOR_TOO_OLD, SPAWN_SUBSCRIBER_LAGGED,
34};
35pub use operator_app::{operator_app_candidates, operator_app_path, OperatorAppPathError};
36pub use policy_cache::{
37 PolicyResolveError, PolicyResolver, PolicyResolverConfig, PolicyResolverFootprint,
38 PolicyVerdict, ProjectRef, Subject, DEFAULT_POLICY_RESOLVER_MODULE_ID,
39};
40pub use subc_control::{AppOpenReply, AppOpenStatus, CatalogOrigin, CatalogWatchResponse};
41pub use subc_control::{
42 ScopeRef, ScopeWatchEvent, ScopeWatchPush, ScopeWatchResponse, ScopeWatchSelector,
43 ScopeWatchState, ScopeWatchStatus,
44};
45
46mod module_subscription;
47pub use module_subscription::ScopeSubscription;
48
49use std::{
50 collections::{BTreeMap, HashMap, VecDeque},
51 env,
52 error::Error,
53 ffi::OsString,
54 fmt,
55 future::Future,
56 io,
57 panic::{catch_unwind, AssertUnwindSafe},
58 path::{Path, PathBuf},
59 pin::Pin,
60 sync::{
61 atomic::{AtomicBool, AtomicU64, Ordering},
62 Arc, Mutex,
63 },
64 time::{Duration, SystemTime, UNIX_EPOCH},
65};
66
67pub use async_trait::async_trait;
68pub use subc_control::{CatalogEntry, ConsumerIdentity};
69mod scope_served_by;
70pub use scope_served_by::{ScopeServedByAnswer, ScopeServedByReply};
71pub mod registration;
72mod route_authorize;
73pub use route_authorize::{
74 RouteAuthorizeExpected, RouteAuthorizeOutcome, RouteAuthorizeResponse, RouteAuthorizeScope,
75};
76mod request_pools;
77pub use registration::ClaimError;
78use request_pools::{wait_for_permit, DispatchPools, PermitWaitOutcome, WaitingRequest};
79pub use request_pools::{RequestClass, RequestPool, RequestPoolSnapshot, RequestPools};
80pub use subc_os::launch_nonce;
87use subc_protocol::{
88 call_mode::derive_call_mode,
89 error_codes,
90 manifest::ModuleManifest,
91 scope::{
92 ScopeEnd, ScopeEndResult, ScopeEnded, ScopeRecord, ScopeRecordResult, ScopeStamp,
93 ScopeStatus, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
94 },
95 session::{
96 ModuleControlCommand, ModuleControlRequest, ModuleControlRequestFromModule,
97 ModuleControlResponse, ModuleControlResponseToModule, OperatorConfirmKind,
98 OperatorConfirmReply, OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK,
99 MODULE_TO_SUBC_OP_CATALOG_UPDATE, MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
100 },
101 BindIdentity, ErrorBody, Flags, Frame, FrameBuildError, FrameType, ModuleHelloAckBody,
102 ModuleHelloBody, Principal, Priority, RouteTarget, PROTOCOL_VERSION, SUBC_MODULE_ID_ENV,
103};
104pub use subc_protocol::{
105 call_mode::{CallMode, CALL_MODES_CAPABILITY},
106 manifest::{
107 build_provenance, CapabilityDeclarations, CapabilityNeed, CapabilityRequirement,
108 ExecutionMode, LaunchNonceSource, ManifestProvenance, ProvenanceFormError, ProviderRole,
109 Tool, PROVENANCE_SENTINELS,
110 },
111 session::{HealthReport, HealthStatus},
112 AdmissionClass, MachineId, SUBC_PROTOCOL_CRATE_VERSION,
113};
114pub use subc_transport::connection_file::{
115 discover, discovery_candidates, Discovered, DiscoveryError,
116};
117
118use subc_transport::{
119 authenticate_client, connection_file, read_frame, write_frame, AuthError, ConnectionFileError,
120 FrameIoError,
121};
122use tokio::{
123 io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufWriter},
124 net::TcpStream,
125 sync::{mpsc, oneshot},
126 time::{timeout, timeout_at, Instant},
127};
128use tokio_util::{
129 sync::{CancellationToken, WaitForCancellationFuture},
130 task::TaskTracker,
131};
132
133const AUTH_DEADLINE: Duration = Duration::from_secs(2);
134const CATALOG_UPDATE_TIMEOUT: Duration = Duration::from_secs(10);
135const OPERATOR_CONFIRM_TIMEOUT: Duration = Duration::from_secs(290);
136const OPERATOR_CONFIRM_OP: &str = "operator.confirm";
137const EGRESS_BUFFER: usize = 64;
138const HANDLER_TASK_CAPACITY: usize = 64;
139const DISPATCH_SATURATION_WAIT: Duration = Duration::from_secs(2);
142const HELLO_CORR: u64 = 1;
143const DEFAULT_WRITER_DRAIN_TIMEOUT: Duration = Duration::from_secs(2);
144const DRAINING_HOOK_START_LIMIT: Duration = Duration::from_secs(2);
152static NEXT_MODULE_CONNECTION_TOKEN: AtomicU64 = AtomicU64::new(1);
153const DETACHED_CONNECTION_TOKEN: u64 = 0;
157
158type RequestKey = (u16, u32, u64);
159type InFlight = Arc<Mutex<HashMap<RequestKey, CancellationToken>>>;
160
161#[derive(Clone, Copy)]
167pub struct RouteHandle {
168 pub channel: u16,
169 pub epoch: u32,
170 connection_token: u64,
171 reverse_request_registry_id: u64,
172}
173
174impl RouteHandle {
175 pub fn detached(channel: u16, epoch: u32) -> Self {
189 Self::new(channel, epoch, DETACHED_CONNECTION_TOKEN)
190 }
191
192 pub(crate) fn new(channel: u16, epoch: u32, connection_token: u64) -> Self {
193 Self {
194 channel,
195 epoch,
196 connection_token,
197 reverse_request_registry_id: 0,
198 }
199 }
200
201 pub(crate) fn new_consumer(
202 channel: u16,
203 epoch: u32,
204 connection_token: u64,
205 reverse_requests: ReverseRequestRegistry,
206 ) -> Self {
207 reverse_requests.seal();
208 Self {
209 channel,
210 epoch,
211 connection_token,
212 reverse_request_registry_id: consumer::install_reverse_request_registry(
213 reverse_requests,
214 ),
215 }
216 }
217
218 pub(crate) fn connection_token(self) -> u64 {
219 self.connection_token
220 }
221
222 pub fn on_request<F, Fut>(
223 &self,
224 method_family: impl Into<String>,
225 handler: F,
226 ) -> Result<(), ReverseRequestRegistrationError>
227 where
228 F: Fn(Vec<u8>, ReverseRequestContext) -> Fut + Send + Sync + 'static,
229 Fut: Future<Output = Vec<u8>> + Send + 'static,
230 {
231 consumer::route_reverse_request_registry(self.reverse_request_registry_id)?
232 .on_request(method_family, handler)
233 }
234
235 pub fn on_request_fallible<F, Fut>(
236 &self,
237 method_family: impl Into<String>,
238 handler: F,
239 ) -> Result<(), ReverseRequestRegistrationError>
240 where
241 F: Fn(Vec<u8>, ReverseRequestContext) -> Fut + Send + Sync + 'static,
242 Fut: Future<Output = Result<Vec<u8>, ReverseRequestError>> + Send + 'static,
243 {
244 consumer::route_reverse_request_registry(self.reverse_request_registry_id)?
245 .on_request_fallible(method_family, handler)
246 }
247
248 pub(crate) fn reverse_request_registry_id(self) -> u64 {
249 self.reverse_request_registry_id
250 }
251}
252
253impl PartialEq for RouteHandle {
254 fn eq(&self, other: &Self) -> bool {
255 self.channel == other.channel
256 && self.epoch == other.epoch
257 && self.connection_token == other.connection_token
258 }
259}
260
261impl Eq for RouteHandle {}
262
263impl std::hash::Hash for RouteHandle {
264 fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
265 std::hash::Hash::hash(&self.channel, state);
266 std::hash::Hash::hash(&self.epoch, state);
267 std::hash::Hash::hash(&self.connection_token, state);
268 }
269}
270
271impl fmt::Debug for RouteHandle {
272 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
273 f.debug_struct("RouteHandle")
274 .field("channel", &self.channel)
275 .field("epoch", &self.epoch)
276 .finish_non_exhaustive()
277 }
278}
279#[derive(Debug, Clone, PartialEq, Eq)]
281pub struct LiveRootsSnapshot {
282 pub roots: Vec<subc_protocol::session::LiveRoot>,
283 pub unknown_root_bindings: u64,
284 pub total_bindings: u64,
285}
286
287#[derive(Debug, Clone, PartialEq, Eq)]
292#[non_exhaustive]
293pub struct ScopeSyncReply {
294 pub generation: u64,
296 pub results: Vec<ScopeRecordResult>,
300 pub ended: Vec<ScopeEnded>,
303}
304
305#[derive(Debug, Clone, PartialEq, Eq)]
309#[non_exhaustive]
310pub struct ScopeApplyReply {
311 pub generation: u64,
313 pub results: Vec<ScopeRecordResult>,
315 pub end_results: Vec<ScopeEndResult>,
318 pub ended: Vec<ScopeEnded>,
321}
322
323#[derive(Debug, Clone, PartialEq, Eq)]
326#[non_exhaustive]
327pub struct ScopeDescribeReply {
328 pub status: ScopeStatus,
329 pub scope_epoch: Option<u64>,
331 pub daemon_incarnation: String,
334 pub owner_synced: bool,
336 pub owner_configured: bool,
338 pub scope: Option<ScopeStamp>,
342}
343
344#[derive(Debug, serde::Deserialize)]
347#[serde(untagged)]
348enum ModuleControlReply {
349 Control(Box<ModuleControlResponseToModule>),
350 RouteAuthorize(RouteAuthorizeResponse),
351}
352
353impl ModuleControlReply {
354 fn control(self) -> Result<ModuleControlResponseToModule, CatalogUpdateError> {
355 match self {
356 Self::Control(reply) => Ok(*reply),
357 other => Err(CatalogUpdateError::Protocol(format!(
358 "unexpected module control response: {other:?}"
359 ))),
360 }
361 }
362}
363
364type CatalogUpdateReply = oneshot::Sender<Result<ModuleControlReply, CatalogUpdateError>>;
365type CatalogUpdateWaiter = oneshot::Receiver<Result<ModuleControlReply, CatalogUpdateError>>;
366type CatalogUpdateRequest = (u64, mpsc::Sender<Frame>, CatalogUpdateWaiter);
367type OperatorConfirmRequestWaiter = (u64, mpsc::Sender<Frame>, oneshot::Receiver<Frame>);
368
369pub type ModuleServeFuture = Pin<Box<dyn Future<Output = Result<(), SubcModuleError>> + Send>>;
371
372#[derive(Clone)]
373struct RequestDispatcher {
374 in_flight: InFlight,
375 control_slots: Arc<tokio::sync::Semaphore>,
376 pools: Arc<DispatchPools>,
379 draining_hooks: Arc<Mutex<Vec<oneshot::Receiver<()>>>>,
384 undecodable_push_logged: Arc<AtomicBool>,
387 failure: Arc<Mutex<Option<SubcModuleError>>>,
388 failed: CancellationToken,
389 tasks: TaskTracker,
390 aborts: Arc<Mutex<Vec<tokio::task::AbortHandle>>>,
391 undelivered_replies: Arc<AtomicU64>,
392}
393
394impl RequestDispatcher {
395 #[cfg(test)]
398 fn new() -> Self {
399 Self::with_pools(Arc::new(DispatchPools::new(RequestPools::new())))
400 }
401
402 fn with_pools(pools: Arc<DispatchPools>) -> Self {
403 Self {
404 in_flight: Arc::new(Mutex::new(HashMap::new())),
405 control_slots: Arc::new(tokio::sync::Semaphore::new(16)),
406 pools,
407 draining_hooks: Arc::new(Mutex::new(Vec::new())),
408 undecodable_push_logged: Arc::new(AtomicBool::new(false)),
409 failure: Arc::new(Mutex::new(None)),
410 failed: CancellationToken::new(),
411 tasks: TaskTracker::new(),
412 aborts: Arc::new(Mutex::new(Vec::new())),
413 undelivered_replies: Arc::new(AtomicU64::new(0)),
414 }
415 }
416
417 fn fail(&self, error: SubcModuleError) {
418 let mut failure = self.failure.lock().unwrap_or_else(|e| e.into_inner());
419 if failure.is_none() {
420 *failure = Some(error);
421 }
422 self.failed.cancel();
423 }
424
425 fn take_failure(&self) -> Option<SubcModuleError> {
426 self.failure
427 .lock()
428 .unwrap_or_else(|e| e.into_inner())
429 .take()
430 }
431
432 fn spawn(&self, future: impl Future<Output = ()> + Send + 'static) {
433 let task = self.tasks.spawn(future);
434 let mut aborts = self.aborts.lock().unwrap_or_else(|e| e.into_inner());
435 aborts.retain(|task| !task.is_finished());
436 aborts.push(task.abort_handle());
437 }
438
439 fn abort_all(&self) {
440 for task in self
441 .aborts
442 .lock()
443 .unwrap_or_else(|e| e.into_inner())
444 .drain(..)
445 {
446 task.abort();
447 }
448 }
449
450 fn fold_health(&self, mut report: HealthReport) -> HealthReport {
451 if let Some(saturation) = self.pools.saturation() {
452 if report.status == HealthStatus::Ok {
453 report.status = HealthStatus::Degraded;
454 }
455 report.detail = Some(match report.detail {
456 Some(detail) => format!("{saturation}; {detail}"),
457 None => saturation,
458 });
459 }
460 report
461 }
462
463 async fn wait_for_draining_hooks_to_start(&self) {
467 let started = std::mem::take(
468 &mut *self
469 .draining_hooks
470 .lock()
471 .unwrap_or_else(|poisoned| poisoned.into_inner()),
472 );
473 let deadline = tokio::time::Instant::now() + DRAINING_HOOK_START_LIMIT;
474 for hook in started {
475 if tokio::time::timeout_at(deadline, hook).await.is_err() {
478 return;
479 }
480 }
481 }
482
483 fn cancel_all(&self) {
488 self.pools.close();
489 let cancelled = match self.in_flight.lock() {
490 Ok(guard) => guard.values().cloned().collect::<Vec<_>>(),
491 Err(poisoned) => poisoned.into_inner().values().cloned().collect(),
492 };
493 for cancellation in cancelled {
494 cancellation.cancel();
495 }
496 }
497}
498
499#[derive(Clone)]
501pub struct ModuleHandle {
502 shared: Arc<ModuleHandleShared>,
503}
504
505struct ModuleHandleShared {
506 registration: registration::Session,
507 negotiated_ver: u8,
508 daemon_capabilities: Vec<String>,
509 supports_catalog_update: bool,
510 supports_role_declarations_update: bool,
511 supports_live_roots: bool,
512 supports_agent_commands: bool,
513 supports_scope_sync: bool,
514 supports_scope_apply: bool,
515 supports_scope_describe: bool,
516 supports_scope_watch: bool,
517 supports_route_authorize: bool,
518 supports_operator_confirm: bool,
519 supports_activity: bool,
520 connection_runtime: tokio::runtime::Handle,
521 connection_token: u64,
522 machine_id: Option<MachineId>,
523 live_routes: Mutex<HashMap<u16, RouteHandle>>,
524 dropped_route_frames: AtomicU64,
525 close_token: CancellationToken,
526 dispatch_pools: Mutex<Arc<DispatchPools>>,
529 catalog_send: tokio::sync::Mutex<()>,
530 config: config::Session,
531 served_control_ops: std::sync::OnceLock<Vec<String>>,
532 inner: Mutex<ModuleHandleState>,
533}
534
535struct ModuleHandleState {
536 pending_control_streams: HashMap<u64, module_subscription::ControlStreamSender>,
537 pending_claim_calls: HashMap<u64, oneshot::Sender<Frame>>,
538 writer: Option<mpsc::Sender<Frame>>,
539 next_corr: Option<u64>,
540 pending_catalog_updates: HashMap<u64, CatalogUpdateReply>,
541 pending_operator_confirms: HashMap<u64, oneshot::Sender<Frame>>,
542 pending_activity_records: HashMap<u64, oneshot::Sender<Frame>>,
543 pending_scope_served_by: HashMap<u64, oneshot::Sender<Frame>>,
544 pending_config_calls: HashMap<u64, oneshot::Sender<Frame>>,
545 pending_agent_commands: HashMap<u64, oneshot::Sender<Frame>>,
546 current_provides: Option<Vec<ProviderRole>>,
547 catalog_changes: VecDeque<CatalogChange>,
548 closed: bool,
549}
550
551struct CatalogChange {
552 corr: u64,
553 provides: Vec<ProviderRole>,
554 outcome: Option<CatalogChangeOutcome>,
555}
556
557enum CatalogChangeOutcome {
558 Accepted,
559 Rejected,
560 Unknown,
561}
562
563struct CatalogUpdateGuard {
564 shared: Arc<ModuleHandleShared>,
565 corr: u64,
566}
567
568impl Drop for CatalogUpdateGuard {
569 fn drop(&mut self) {
570 self.shared.remove_pending_catalog_update(self.corr);
571 }
572}
573
574impl ModuleHandle {
575 fn new(
576 ack: &ModuleHelloAckBody,
577 writer: mpsc::Sender<Frame>,
578 connection_token: u64,
579 close_token: CancellationToken,
580 connection_runtime: tokio::runtime::Handle,
581 ) -> Self {
582 Self {
583 shared: Arc::new(ModuleHandleShared {
584 registration: registration::Session::default(),
585 negotiated_ver: ack.negotiated_ver,
586 daemon_capabilities: ack.subc_capabilities.clone(),
587 supports_catalog_update: ack
588 .subc_ops
589 .iter()
590 .any(|op| op == MODULE_TO_SUBC_OP_CATALOG_UPDATE),
591 supports_role_declarations_update: ack
592 .subc_ops
593 .iter()
594 .any(|op| op == MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS),
595 supports_live_roots: ack.subc_ops.iter().any(|op| op == "supervisor.live_roots"),
596 supports_agent_commands: ack
597 .subc_ops
598 .iter()
599 .any(|op| op == "supervisor.agent_commands"),
600 supports_scope_sync: ack.subc_ops.iter().any(|op| op == SCOPE_SYNC_OP),
601 supports_scope_apply: ack.subc_ops.iter().any(|op| op == SCOPE_APPLY_OP),
602 supports_scope_describe: ack.subc_ops.iter().any(|op| op == SCOPE_DESCRIBE_OP),
603 supports_scope_watch: ack
604 .subc_ops
605 .iter()
606 .any(|op| op == subc_control::ops::SCOPE_WATCH),
607 supports_route_authorize: ack
608 .subc_ops
609 .iter()
610 .any(|op| op == subc_protocol::scope::ROUTE_AUTHORIZE_OP),
611 supports_operator_confirm: ack.subc_ops.iter().any(|op| op == OPERATOR_CONFIRM_OP),
612 supports_activity: ack
613 .subc_capabilities
614 .iter()
615 .any(|capability| capability == ACTIVITY_LEDGER_CAPABILITY),
616 connection_runtime,
617 connection_token,
618 machine_id: ack
621 .machine_id
622 .as_deref()
623 .and_then(|id| MachineId::parse(id).ok()),
624 live_routes: Mutex::new(HashMap::new()),
625 dropped_route_frames: AtomicU64::new(0),
626 close_token,
627 dispatch_pools: Mutex::new(Arc::new(DispatchPools::new(RequestPools::new()))),
628 catalog_send: tokio::sync::Mutex::new(()),
629 config: config::Session::default(),
630 served_control_ops: std::sync::OnceLock::new(),
631 inner: Mutex::new(ModuleHandleState {
632 pending_control_streams: HashMap::new(),
633 pending_claim_calls: HashMap::new(),
634 writer: Some(writer),
635 next_corr: Some(HELLO_CORR + 1),
636 pending_catalog_updates: HashMap::new(),
637 pending_operator_confirms: HashMap::new(),
638 pending_activity_records: HashMap::new(),
639 pending_scope_served_by: HashMap::new(),
640 pending_config_calls: HashMap::new(),
641 pending_agent_commands: HashMap::new(),
642 current_provides: None,
643 catalog_changes: VecDeque::new(),
644 closed: false,
645 }),
646 }),
647 }
648 }
649
650 pub fn machine_id(&self) -> Option<&MachineId> {
657 self.shared.machine_id.as_ref()
658 }
659
660 pub fn daemon_capabilities(&self) -> &[String] {
664 &self.shared.daemon_capabilities
665 }
666
667 pub fn daemon_supports(&self, capability: &str) -> bool {
674 self.shared
675 .daemon_capabilities
676 .iter()
677 .any(|advertised| advertised == capability)
678 }
679
680 pub fn closed(&self) -> impl Future<Output = ()> + Send + 'static {
684 self.shared.close_token.clone().cancelled_owned()
685 }
686
687 pub fn is_closed(&self) -> bool {
689 self.shared.close_token.is_cancelled()
690 }
691
692 pub async fn catalog_update(
697 &self,
698 provides: Vec<ProviderRole>,
699 ) -> Result<(), CatalogUpdateError> {
700 self.catalog_update_inner(Some(provides), None, None).await
701 }
702
703 pub async fn update_role_declarations(
711 &self,
712 role_declarations: BTreeMap<String, serde_json::Map<String, serde_json::Value>>,
713 ) -> Result<(), CatalogUpdateError> {
714 if !self.shared.supports_role_declarations_update {
715 return Err(CatalogUpdateError::NotSupported);
716 }
717 let body = serde_json::to_vec(
718 &ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations {
719 role_declarations: serde_json::to_value(role_declarations)
720 .map_err(|err| CatalogUpdateError::Protocol(err.to_string()))?,
721 },
722 )
723 .map_err(|err| CatalogUpdateError::Protocol(err.to_string()))?;
724 let (corr, writer, rx) = self.shared.begin_catalog_update()?;
725 let _pending = CatalogUpdateGuard {
726 shared: self.shared.clone(),
727 corr,
728 };
729 let frame = Frame::build_with_version(
730 self.shared.negotiated_ver,
731 FrameType::Request,
732 control_flags(),
733 0,
734 0,
735 corr,
736 body,
737 )
738 .map_err(|err| CatalogUpdateError::Protocol(err.to_string()))?;
739 if writer.send(frame).await.is_err() {
740 return Err(CatalogUpdateError::ConnectionClosed);
741 }
742 match timeout(CATALOG_UPDATE_TIMEOUT, rx).await {
743 Ok(Ok(Ok(reply))) => match reply.control()? {
744 ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {} => Ok(()),
745 _ => Err(CatalogUpdateError::Protocol(
746 "unexpected catalog.update_role_declarations response".into(),
747 )),
748 },
749 Ok(Ok(Err(err))) => Err(err),
750 Ok(Err(_)) => Err(CatalogUpdateError::ConnectionClosed),
751 Err(_) => Err(CatalogUpdateError::Timeout),
752 }
753 }
754
755 pub async fn catalog_update_with_capabilities(
761 &self,
762 provides: Vec<ProviderRole>,
763 capabilities: CapabilityDeclarations,
764 ) -> Result<(), CatalogUpdateError> {
765 self.catalog_update_inner(Some(provides), Some(capabilities), None)
766 .await
767 }
768
769 pub async fn set_ready(&self, ready: bool) -> Result<(), CatalogUpdateError> {
781 self.catalog_update_inner(None, None, Some(ready)).await
782 }
783
784 async fn catalog_update_inner(
785 &self,
786 provides: Option<Vec<ProviderRole>>,
787 capabilities: Option<CapabilityDeclarations>,
788 ready: Option<bool>,
789 ) -> Result<(), CatalogUpdateError> {
790 if !self.shared.supports_catalog_update {
791 return Err(CatalogUpdateError::NotSupported);
792 }
793
794 let send_guard = self.shared.catalog_send.lock().await;
797 let provides = match provides {
798 Some(provides) => provides,
799 None => {
800 let inner = self.shared.lock_inner();
801 if inner.closed {
802 return Err(CatalogUpdateError::ConnectionClosed);
803 }
804 inner
805 .current_provides
806 .as_ref()
807 .filter(|_| inner.catalog_changes.is_empty())
808 .cloned()
809 .ok_or_else(|| {
810 CatalogUpdateError::Protocol(
811 "current provider roles are unknown or a catalog update is in flight"
812 .into(),
813 )
814 })?
815 }
816 };
817 let body = serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
818 provides: provides.clone(),
819 capabilities,
820 ready,
821 })
822 .map_err(|err| {
823 CatalogUpdateError::Protocol(format!(
824 "failed to encode catalog.update request body: {err}"
825 ))
826 })?;
827 let (corr, writer, rx) = self.shared.begin_catalog_update()?;
828 let _pending = CatalogUpdateGuard {
829 shared: self.shared.clone(),
830 corr,
831 };
832 let frame = Frame::build_with_version(
833 self.shared.negotiated_ver,
834 FrameType::Request,
835 control_flags(),
836 0,
837 0,
838 corr,
839 body,
840 )
841 .map_err(|err| {
842 self.shared.remove_pending_catalog_update(corr);
843 CatalogUpdateError::Protocol(format!(
844 "failed to build catalog.update request frame: {err}"
845 ))
846 })?;
847
848 let permit = match writer.reserve().await {
849 Ok(permit) => permit,
850 Err(_) => {
851 self.shared.remove_pending_catalog_update(corr);
852 return Err(CatalogUpdateError::ConnectionClosed);
853 }
854 };
855 self.shared
856 .lock_inner()
857 .catalog_changes
858 .push_back(CatalogChange {
859 corr,
860 provides,
861 outcome: None,
862 });
863 permit.send(frame);
864 drop(send_guard);
865
866 match timeout(CATALOG_UPDATE_TIMEOUT, async {
867 rx.await
868 .map(|reply| reply.and_then(ModuleControlReply::control))
869 })
870 .await
871 {
872 Ok(Ok(Ok(ModuleControlResponseToModule::CatalogUpdate {}))) => Ok(()),
873 Ok(Ok(Ok(_))) => Err(CatalogUpdateError::Protocol(
874 "unexpected catalog.update response".into(),
875 )),
876 Ok(Ok(Err(err))) => Err(err),
877 Ok(Err(_)) => Err(CatalogUpdateError::ConnectionClosed),
878 Err(_) => {
879 self.shared.remove_pending_catalog_update(corr);
880 Err(CatalogUpdateError::Timeout)
881 }
882 }
883 }
884
885 pub async fn live_roots(&self) -> Result<LiveRootsSnapshot, CatalogUpdateError> {
887 if !self.shared.supports_live_roots {
888 return Err(CatalogUpdateError::NotSupported);
889 }
890 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {})
891 .map_err(|err| CatalogUpdateError::Protocol(err.to_string()))?;
892 let (corr, writer, rx) = self.shared.begin_catalog_update()?;
893 let frame = Frame::build_with_version(
894 self.shared.negotiated_ver,
895 FrameType::Request,
896 control_flags(),
897 0,
898 0,
899 corr,
900 body,
901 )
902 .map_err(|err| {
903 self.shared.remove_pending_catalog_update(corr);
904 CatalogUpdateError::Protocol(err.to_string())
905 })?;
906 if writer.send(frame).await.is_err() {
907 self.shared.remove_pending_catalog_update(corr);
908 return Err(CatalogUpdateError::ConnectionClosed);
909 }
910 match timeout(CATALOG_UPDATE_TIMEOUT, async {
911 rx.await
912 .map(|reply| reply.and_then(ModuleControlReply::control))
913 })
914 .await
915 {
916 Ok(Ok(Ok(ModuleControlResponseToModule::LiveRoots {
917 roots,
918 unknown_root_bindings,
919 total_bindings,
920 }))) => Ok(LiveRootsSnapshot {
921 roots,
922 unknown_root_bindings,
923 total_bindings,
924 }),
925 Ok(Ok(Ok(_))) => Err(CatalogUpdateError::Protocol(
926 "unexpected live_roots response".into(),
927 )),
928 Ok(Ok(Err(err))) => Err(err),
929 Ok(Err(_)) => Err(CatalogUpdateError::ConnectionClosed),
930 Err(_) => {
931 self.shared.remove_pending_catalog_update(corr);
932 Err(CatalogUpdateError::Timeout)
933 }
934 }
935 }
936
937 pub async fn scope_sync(
953 &self,
954 generation: u64,
955 scopes: Vec<ScopeRecord>,
956 ) -> Result<ScopeSyncReply, ScopeCallError> {
957 if !self.shared.supports_scope_sync {
958 return Err(ScopeCallError::NotSupported { op: SCOPE_SYNC_OP });
959 }
960 let request = ModuleControlRequestFromModule::ScopeSync { generation, scopes };
961 match self
962 .scope_call(SCOPE_SYNC_OP, &request)
963 .await?
964 .control()
965 .map_err(ScopeCallError::from_control_error)?
966 {
967 ModuleControlResponseToModule::ScopeSync {
968 generation,
969 results,
970 ended,
971 } => Ok(ScopeSyncReply {
972 generation,
973 results,
974 ended,
975 }),
976 other => Err(ScopeCallError::Protocol(format!(
977 "unexpected {SCOPE_SYNC_OP} response: {other:?}"
978 ))),
979 }
980 }
981
982 pub async fn scope_apply(
993 &self,
994 generation: u64,
995 upsert: Vec<ScopeRecord>,
996 end: Vec<ScopeEnd>,
997 ) -> Result<ScopeApplyReply, ScopeCallError> {
998 if !self.shared.supports_scope_apply {
999 return Err(ScopeCallError::NotSupported { op: SCOPE_APPLY_OP });
1000 }
1001 let request = ModuleControlRequestFromModule::ScopeApply {
1002 generation,
1003 upsert,
1004 end,
1005 };
1006 match self
1007 .scope_call(SCOPE_APPLY_OP, &request)
1008 .await?
1009 .control()
1010 .map_err(ScopeCallError::from_control_error)?
1011 {
1012 ModuleControlResponseToModule::ScopeApply {
1013 generation,
1014 results,
1015 end_results,
1016 ended,
1017 } => Ok(ScopeApplyReply {
1018 generation,
1019 results,
1020 end_results,
1021 ended,
1022 }),
1023 other => Err(ScopeCallError::Protocol(format!(
1024 "unexpected {SCOPE_APPLY_OP} response: {other:?}"
1025 ))),
1026 }
1027 }
1028
1029 pub async fn scope_describe(
1037 &self,
1038 owner: Principal,
1039 scope_ref: String,
1040 ) -> Result<ScopeDescribeReply, ScopeCallError> {
1041 if !self.shared.supports_scope_describe {
1042 return Err(ScopeCallError::NotSupported {
1043 op: SCOPE_DESCRIBE_OP,
1044 });
1045 }
1046 let request = ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref };
1047 match self
1048 .scope_call(SCOPE_DESCRIBE_OP, &request)
1049 .await?
1050 .control()
1051 .map_err(ScopeCallError::from_control_error)?
1052 {
1053 ModuleControlResponseToModule::ScopeDescribe {
1054 status,
1055 scope_epoch,
1056 daemon_incarnation,
1057 owner_synced,
1058 owner_configured,
1059 scope,
1060 } => Ok(ScopeDescribeReply {
1061 status,
1062 scope_epoch,
1063 daemon_incarnation,
1064 owner_synced,
1065 owner_configured,
1066 scope,
1067 }),
1068 other => Err(ScopeCallError::Protocol(format!(
1069 "unexpected {SCOPE_DESCRIBE_OP} response: {other:?}"
1070 ))),
1071 }
1072 }
1073
1074 pub async fn confirm_operator(
1086 &self,
1087 summary: &str,
1088 caller_route: &RouteHandle,
1089 ) -> Result<(), OperatorConfirmError> {
1090 self.confirm_operator_with_deadline(summary, caller_route, OPERATOR_CONFIRM_TIMEOUT)
1091 .await
1092 }
1093
1094 pub async fn confirm_operation(
1115 &self,
1116 summary: &str,
1117 caller_route: &RouteHandle,
1118 operation: &str,
1119 ) -> Result<(), OperatorConfirmError> {
1120 if operation.is_empty() {
1121 return Err(OperatorConfirmError::OperationInvalid);
1122 }
1123 self.confirm_operation_with_deadline(
1124 summary,
1125 caller_route,
1126 operation,
1127 OPERATOR_CONFIRM_TIMEOUT,
1128 )
1129 .await
1130 }
1131
1132 pub async fn confirm_enrollment(
1150 &self,
1151 summary: &str,
1152 caller_route: &RouteHandle,
1153 ) -> Result<(), OperatorConfirmError> {
1154 self.confirm_inner(
1155 summary,
1156 caller_route,
1157 Some(OperatorConfirmKind::Enrollment),
1158 None,
1159 OPERATOR_CONFIRM_TIMEOUT,
1160 )
1161 .await
1162 }
1163
1164 pub(crate) async fn confirm_operator_with_deadline(
1165 &self,
1166 summary: &str,
1167 caller_route: &RouteHandle,
1168 duration: Duration,
1169 ) -> Result<(), OperatorConfirmError> {
1170 self.confirm_inner(summary, caller_route, None, None, duration)
1171 .await
1172 }
1173
1174 async fn confirm_operation_with_deadline(
1175 &self,
1176 summary: &str,
1177 caller_route: &RouteHandle,
1178 operation: &str,
1179 duration: Duration,
1180 ) -> Result<(), OperatorConfirmError> {
1181 self.confirm_inner(
1182 summary,
1183 caller_route,
1184 Some(OperatorConfirmKind::Operation),
1185 Some(operation.to_owned()),
1186 duration,
1187 )
1188 .await
1189 }
1190
1191 async fn confirm_inner(
1192 &self,
1193 summary: &str,
1194 caller_route: &RouteHandle,
1195 kind: Option<OperatorConfirmKind>,
1196 operation: Option<String>,
1197 duration: Duration,
1198 ) -> Result<(), OperatorConfirmError> {
1199 if caller_route.connection_token() != self.shared.connection_token {
1200 return Err(OperatorConfirmError::NotPermitted);
1201 }
1202 if !self.shared.supports_operator_confirm {
1203 return Err(OperatorConfirmError::Unsupported);
1204 }
1205 let deadline = Instant::now() + duration;
1208 let body = serde_json::to_vec(
1209 &OperatorConfirmRequest::new(summary, caller_route.channel, caller_route.epoch)
1210 .with_kind(kind)
1211 .with_operation(operation),
1212 )
1213 .map_err(|_| OperatorConfirmError::PresenceUnavailable)?;
1214 let (corr, writer, rx) = self.shared.begin_operator_confirm()?;
1215 let mut guard = OperatorConfirmGuard {
1216 shared: Arc::clone(&self.shared),
1217 runtime: self.shared.connection_runtime.clone(),
1218 writer: writer.clone(),
1219 corr,
1220 accepted: false,
1221 armed: true,
1222 };
1223 let frame = Frame::build_with_version(
1224 self.shared.negotiated_ver,
1225 FrameType::Request,
1226 control_flags(),
1227 0,
1228 0,
1229 corr,
1230 body,
1231 )
1232 .map_err(|_| OperatorConfirmError::PresenceUnavailable)?;
1233 tokio::select! {
1234 result = timeout_at(deadline, writer.send(frame)) => {
1235 result.map_err(|_| OperatorConfirmError::PresenceUnavailable)?
1236 .map_err(|_| OperatorConfirmError::PresenceUnavailable)?;
1237 }
1238 _ = self.shared.close_token.cancelled() => {
1239 return Err(OperatorConfirmError::PresenceUnavailable);
1240 }
1241 }
1242 guard.accepted = true;
1243 let reply = timeout_at(deadline, rx)
1244 .await
1245 .map_err(|_| OperatorConfirmError::PresenceUnavailable)?
1246 .map_err(|_| OperatorConfirmError::PresenceUnavailable)?;
1247 guard.armed = false;
1248 match reply.header.ty {
1249 FrameType::Response => {
1250 match serde_json::from_slice::<OperatorConfirmReply>(&reply.body) {
1251 Ok(body) if body.outcome == "confirmed" => Ok(()),
1252 _ => Err(OperatorConfirmError::PresenceUnavailable),
1253 }
1254 }
1255 FrameType::Error => match serde_json::from_slice::<ErrorBody>(&reply.body) {
1256 Ok(body) => Err(match body.code.as_str() {
1257 error_codes::OPERATOR_DECLINED => {
1258 let detail = body.detail.as_ref();
1259 if detail
1260 .and_then(|detail| detail.get("reason"))
1261 .and_then(serde_json::Value::as_str)
1262 == Some("backoff")
1263 {
1264 OperatorConfirmError::Backoff {
1265 retry_after_ms: detail
1266 .and_then(|detail| detail.get("retry_after_ms"))
1267 .and_then(serde_json::Value::as_u64),
1268 }
1269 } else {
1270 OperatorConfirmError::Declined
1271 }
1272 }
1273 error_codes::OPERATOR_PRESENCE_UNAVAILABLE
1274 if body
1275 .detail
1276 .as_ref()
1277 .and_then(|detail| detail.get("reason"))
1278 .and_then(serde_json::Value::as_str)
1279 == Some("unsupported_platform") =>
1280 {
1281 OperatorConfirmError::PresenceUnsupportedPlatform
1282 }
1283 error_codes::OPERATOR_SUMMARY_INVALID => OperatorConfirmError::SummaryInvalid,
1284 error_codes::OPERATOR_REQUEST_NOT_PERMITTED | "not_registered" => {
1285 OperatorConfirmError::NotPermitted
1286 }
1287 "unsupported_control_frame" => OperatorConfirmError::Unsupported,
1288 _ => OperatorConfirmError::PresenceUnavailable,
1289 }),
1290 Err(_) => Err(OperatorConfirmError::PresenceUnavailable),
1291 },
1292 _ => Err(OperatorConfirmError::PresenceUnavailable),
1293 }
1294 }
1295
1296 async fn scope_call(
1300 &self,
1301 op: &'static str,
1302 request: &ModuleControlRequestFromModule,
1303 ) -> Result<ModuleControlReply, ScopeCallError> {
1304 let body = serde_json::to_vec(request).map_err(|err| {
1305 ScopeCallError::Protocol(format!("failed to encode {op} request body: {err}"))
1306 })?;
1307 let (corr, writer, rx) = self
1308 .shared
1309 .begin_catalog_update()
1310 .map_err(ScopeCallError::from_control_error)?;
1311 let frame = Frame::build_with_version(
1312 self.shared.negotiated_ver,
1313 FrameType::Request,
1314 control_flags(),
1315 0,
1316 0,
1317 corr,
1318 body,
1319 )
1320 .map_err(|err| {
1321 self.shared.remove_pending_catalog_update(corr);
1322 ScopeCallError::Protocol(format!("failed to build {op} request frame: {err}"))
1323 })?;
1324 if writer.send(frame).await.is_err() {
1325 self.shared.remove_pending_catalog_update(corr);
1326 return Err(ScopeCallError::ConnectionClosed);
1327 }
1328 match timeout(CATALOG_UPDATE_TIMEOUT, rx).await {
1329 Ok(Ok(reply)) => reply.map_err(ScopeCallError::from_control_error),
1330 Ok(Err(_)) => Err(ScopeCallError::ConnectionClosed),
1331 Err(_) => {
1332 self.shared.remove_pending_catalog_update(corr);
1333 Err(ScopeCallError::Timeout)
1334 }
1335 }
1336 }
1337
1338 pub async fn push(
1340 &self,
1341 handle: &RouteHandle,
1342 body: Vec<u8>,
1343 admission_class: Option<AdmissionClass>,
1344 ) -> Result<(), SubcModuleError> {
1345 self.validate_route(*handle)?;
1346 let writer = self
1347 .shared
1348 .lock_inner()
1349 .writer
1350 .clone()
1351 .ok_or(SubcModuleError::WriterClosed)?;
1352 let frame = Frame::build_with_version(
1353 self.shared.negotiated_ver,
1354 FrameType::Push,
1355 data_flags().with_admission_class(admission_class.unwrap_or(AdmissionClass::Normal)),
1356 handle.channel,
1357 handle.epoch,
1358 0,
1359 body,
1360 )
1361 .map_err(SubcModuleError::FrameBuild)?;
1362 send_outbound(&writer, frame).await
1363 }
1364
1365 pub fn dropped_route_frames(&self) -> u64 {
1367 self.shared.dropped_route_frames.load(Ordering::Relaxed)
1368 }
1369
1370 fn validate_route(&self, handle: RouteHandle) -> Result<(), SubcModuleError> {
1371 if handle.connection_token() != self.shared.connection_token {
1372 return Err(SubcModuleError::StaleRouteHandle(handle));
1373 }
1374 let routes = self
1375 .shared
1376 .live_routes
1377 .lock()
1378 .map_err(|_| SubcModuleError::InFlightPoisoned)?;
1379 if routes.get(&handle.channel) == Some(&handle) {
1380 Ok(())
1381 } else {
1382 Err(SubcModuleError::StaleRouteHandle(handle))
1383 }
1384 }
1385
1386 fn install_route(&self, handle: RouteHandle) -> Result<(), SubcModuleError> {
1387 self.shared
1388 .live_routes
1389 .lock()
1390 .map_err(|_| SubcModuleError::InFlightPoisoned)?
1391 .insert(handle.channel, handle);
1392 Ok(())
1393 }
1394
1395 fn installed_route(&self, channel: u16) -> Result<Option<RouteHandle>, SubcModuleError> {
1396 Ok(self
1397 .shared
1398 .live_routes
1399 .lock()
1400 .map_err(|_| SubcModuleError::InFlightPoisoned)?
1401 .get(&channel)
1402 .copied())
1403 }
1404
1405 fn remove_route(&self, handle: RouteHandle) -> Result<bool, SubcModuleError> {
1406 let mut routes = self
1407 .shared
1408 .live_routes
1409 .lock()
1410 .map_err(|_| SubcModuleError::InFlightPoisoned)?;
1411 if routes.get(&handle.channel) == Some(&handle) {
1412 routes.remove(&handle.channel);
1413 Ok(true)
1414 } else {
1415 Ok(false)
1416 }
1417 }
1418
1419 fn validate_ingress(&self, channel: u16, epoch: u32) -> Result<bool, SubcModuleError> {
1420 let handle = RouteHandle::new(channel, epoch, self.shared.connection_token);
1421 let valid = self
1422 .shared
1423 .live_routes
1424 .lock()
1425 .map_err(|_| SubcModuleError::InFlightPoisoned)?
1426 .get(&channel)
1427 == Some(&handle);
1428 if !valid {
1429 self.shared
1430 .dropped_route_frames
1431 .fetch_add(1, Ordering::Relaxed);
1432 }
1433 Ok(valid)
1434 }
1435
1436 fn route_handle(&self, channel: u16, epoch: u32) -> RouteHandle {
1437 RouteHandle::new(channel, epoch, self.shared.connection_token)
1438 }
1439
1440 fn handle_control_reply(&self, frame: Frame) -> bool {
1441 let Some(frame) = self.handle_control_stream_frame(frame) else {
1442 return true;
1443 };
1444 let claim_reply = self
1445 .shared
1446 .lock_inner()
1447 .pending_claim_calls
1448 .remove(&frame.header.corr);
1449 if let Some(reply) = claim_reply {
1450 let _ = reply.send(frame);
1451 return true;
1452 }
1453 let config_reply = self
1454 .shared
1455 .lock_inner()
1456 .pending_config_calls
1457 .remove(&frame.header.corr);
1458 if let Some(reply) = config_reply {
1459 let _ = reply.send(frame);
1460 return true;
1461 }
1462 if let Some(reply) = self
1463 .shared
1464 .lock_inner()
1465 .pending_agent_commands
1466 .remove(&frame.header.corr)
1467 {
1468 let _ = reply.send(frame);
1469 return true;
1470 }
1471 let served_by_reply = self
1472 .shared
1473 .lock_inner()
1474 .pending_scope_served_by
1475 .remove(&frame.header.corr);
1476 if let Some(reply) = served_by_reply {
1477 let _ = reply.send(frame);
1478 return true;
1479 }
1480 let activity_reply = self
1481 .shared
1482 .lock_inner()
1483 .pending_activity_records
1484 .remove(&frame.header.corr);
1485 if let Some(reply) = activity_reply {
1486 let _ = reply.send(frame);
1487 return true;
1488 }
1489 let operator_reply = self
1490 .shared
1491 .lock_inner()
1492 .pending_operator_confirms
1493 .remove(&frame.header.corr);
1494 if let Some(reply) = operator_reply {
1495 let _ = reply.send(frame);
1496 return true;
1497 }
1498 let Some(reply) = self.shared.take_pending_catalog_update(frame.header.corr) else {
1499 return false;
1500 };
1501 let result = match frame.header.ty {
1502 FrameType::Response => serde_json::from_slice::<ModuleControlReply>(&frame.body)
1503 .map_err(|err| {
1504 CatalogUpdateError::Protocol(format!(
1505 "invalid module control response body: {err}"
1506 ))
1507 }),
1508 FrameType::Error => match serde_json::from_slice::<ErrorBody>(&frame.body) {
1509 Ok(body) => Err(match body.code.as_str() {
1510 "catalog_update_frozen_field" => CatalogUpdateError::FrozenField(body),
1511 "not_registered" => CatalogUpdateError::NotRegistered(body),
1512 _ => CatalogUpdateError::Rejected(body),
1513 }),
1514 Err(err) => Err(CatalogUpdateError::Protocol(format!(
1515 "invalid catalog.update error body: {err}"
1516 ))),
1517 },
1518 ty => Err(CatalogUpdateError::Protocol(format!(
1519 "unexpected catalog.update terminal frame: {ty:?}"
1520 ))),
1521 };
1522 let outcome = match &result {
1523 Ok(ModuleControlReply::Control(reply))
1524 if matches!(
1525 reply.as_ref(),
1526 ModuleControlResponseToModule::CatalogUpdate {}
1527 ) =>
1528 {
1529 CatalogChangeOutcome::Accepted
1530 }
1531 Err(CatalogUpdateError::FrozenField(_))
1532 | Err(CatalogUpdateError::NotRegistered(_))
1533 | Err(CatalogUpdateError::Rejected(_)) => CatalogChangeOutcome::Rejected,
1534 _ => CatalogChangeOutcome::Unknown,
1535 };
1536 self.shared
1537 .finish_catalog_change(frame.header.corr, outcome);
1538 let _ = reply.send(result);
1539 true
1540 }
1541
1542 fn close_connection(&self) {
1543 self.shared.close_connection();
1544 }
1545}
1546
1547impl ModuleHandleShared {
1548 fn begin_catalog_update(&self) -> Result<CatalogUpdateRequest, CatalogUpdateError> {
1549 let mut inner = self.lock_inner();
1550 if inner.closed {
1551 return Err(CatalogUpdateError::ConnectionClosed);
1552 }
1553 let Some(writer) = inner.writer.clone() else {
1554 drop(inner);
1555 self.close_connection();
1556 return Err(CatalogUpdateError::ConnectionClosed);
1557 };
1558 let Some(corr) = next_module_control_corr(&mut inner) else {
1559 drop(inner);
1560 self.close_connection();
1561 return Err(CatalogUpdateError::ConnectionClosed);
1562 };
1563 let (tx, rx) = oneshot::channel();
1564 inner.pending_catalog_updates.insert(corr, tx);
1565 Ok((corr, writer, rx))
1566 }
1567
1568 fn begin_operator_confirm(&self) -> Result<OperatorConfirmRequestWaiter, OperatorConfirmError> {
1569 let mut inner = self.lock_inner();
1570 if inner.closed {
1571 return Err(OperatorConfirmError::PresenceUnavailable);
1572 }
1573 let Some(writer) = inner.writer.clone() else {
1574 drop(inner);
1575 self.close_connection();
1576 return Err(OperatorConfirmError::PresenceUnavailable);
1577 };
1578 let Some(corr) = next_module_control_corr(&mut inner) else {
1579 drop(inner);
1580 self.close_connection();
1581 return Err(OperatorConfirmError::PresenceUnavailable);
1582 };
1583 let (tx, rx) = oneshot::channel();
1584 inner.pending_operator_confirms.insert(corr, tx);
1585 Ok((corr, writer, rx))
1586 }
1587
1588 fn take_pending_catalog_update(&self, corr: u64) -> Option<CatalogUpdateReply> {
1589 self.lock_inner().pending_catalog_updates.remove(&corr)
1590 }
1591
1592 fn remove_pending_catalog_update(&self, corr: u64) {
1593 let removed = self
1594 .lock_inner()
1595 .pending_catalog_updates
1596 .remove(&corr)
1597 .is_some();
1598 if removed {
1599 self.finish_catalog_change(corr, CatalogChangeOutcome::Unknown);
1600 }
1601 }
1602
1603 fn finish_catalog_change(&self, corr: u64, outcome: CatalogChangeOutcome) {
1604 let mut inner = self.lock_inner();
1605 if let Some(change) = inner
1606 .catalog_changes
1607 .iter_mut()
1608 .find(|change| change.corr == corr)
1609 {
1610 change.outcome = Some(outcome);
1611 }
1612 while inner
1615 .catalog_changes
1616 .front()
1617 .is_some_and(|change| change.outcome.is_some())
1618 {
1619 let change = inner.catalog_changes.pop_front().expect("completed change");
1620 match change.outcome.expect("completed outcome") {
1621 CatalogChangeOutcome::Accepted => inner.current_provides = Some(change.provides),
1622 CatalogChangeOutcome::Rejected => {}
1623 CatalogChangeOutcome::Unknown => inner.current_provides = None,
1624 }
1625 }
1626 }
1627
1628 fn close_connection(&self) {
1629 let pending = {
1630 let mut inner = self.lock_inner();
1631 if inner.closed {
1632 return;
1633 }
1634 inner.closed = true;
1635 inner.writer = None;
1636 inner.current_provides = None;
1637 inner.catalog_changes.clear();
1638 self.close_token.cancel();
1639 inner.pending_operator_confirms.clear();
1641 inner.pending_activity_records.clear();
1642 inner.pending_scope_served_by.clear();
1643 inner.pending_config_calls.clear();
1644 inner.pending_agent_commands.clear();
1645 inner.pending_claim_calls.clear();
1646 inner.pending_control_streams.clear();
1647 inner
1648 .pending_catalog_updates
1649 .drain()
1650 .map(|(_, reply)| reply)
1651 .collect::<Vec<_>>()
1652 };
1653 self.clear_live_routes();
1654 for reply in pending {
1655 let _ = reply.send(Err(CatalogUpdateError::ConnectionClosed));
1656 }
1657 }
1658
1659 fn clear_live_routes(&self) {
1660 if let Ok(mut routes) = self.live_routes.lock() {
1661 routes.clear();
1662 }
1663 }
1664
1665 fn lock_inner(&self) -> std::sync::MutexGuard<'_, ModuleHandleState> {
1666 self.inner
1667 .lock()
1668 .unwrap_or_else(|poisoned| poisoned.into_inner())
1669 }
1670}
1671
1672fn next_module_control_corr(inner: &mut ModuleHandleState) -> Option<u64> {
1673 let corr = inner.next_corr?;
1674 inner.next_corr = corr.checked_add(1);
1675 Some(corr)
1676}
1677
1678struct OperatorConfirmGuard {
1679 shared: Arc<ModuleHandleShared>,
1680 runtime: tokio::runtime::Handle,
1681 writer: mpsc::Sender<Frame>,
1682 corr: u64,
1683 accepted: bool,
1684 armed: bool,
1685}
1686
1687impl Drop for OperatorConfirmGuard {
1688 fn drop(&mut self) {
1689 if !self.armed {
1690 return;
1691 }
1692 self.shared
1693 .lock_inner()
1694 .pending_operator_confirms
1695 .remove(&self.corr);
1696 if !self.accepted {
1697 return;
1698 }
1699 let Ok(cancel) = Frame::build_with_version(
1700 self.shared.negotiated_ver,
1701 FrameType::Cancel,
1702 control_flags(),
1703 0,
1704 0,
1705 self.corr,
1706 Vec::new(),
1707 ) else {
1708 return;
1709 };
1710 match self.writer.try_send(cancel) {
1711 Ok(()) | Err(mpsc::error::TrySendError::Closed(_)) => {}
1712 Err(mpsc::error::TrySendError::Full(cancel)) => {
1713 let writer = self.writer.clone();
1717 self.runtime.spawn(async move {
1718 let _ = writer.send(cancel).await;
1719 });
1720 }
1721 }
1722 }
1723}
1724
1725#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1730#[non_exhaustive]
1731pub enum OperatorConfirmError {
1732 Declined,
1733 Backoff {
1736 retry_after_ms: Option<u64>,
1737 },
1738 PresenceUnavailable,
1743 PresenceUnsupportedPlatform,
1755 SummaryInvalid,
1756 OperationInvalid,
1762 NotPermitted,
1763 Unsupported,
1764}
1765
1766impl fmt::Display for OperatorConfirmError {
1767 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1768 f.write_str(match self {
1769 Self::Declined => "operator declined the write",
1770 Self::Backoff { retry_after_ms: None } => {
1771 "operator confirmation refused without asking: a recent decline holds new prompts for a short time"
1772 }
1773 Self::PresenceUnavailable => "operator confirmation is unavailable",
1774 Self::PresenceUnsupportedPlatform => {
1775 "operator confirmation has no prompt on this platform"
1776 }
1777 Self::SummaryInvalid => "operator confirmation summary is invalid",
1778 Self::OperationInvalid => "operator confirmation operation is invalid",
1779 Self::NotPermitted => "operator confirmation is not permitted for this route",
1780 Self::Unsupported => "daemon does not support operator confirmation",
1781 Self::Backoff {
1782 retry_after_ms: Some(_),
1783 } => return write!(
1784 f,
1785 "operator confirmation refused without asking: a recent decline holds new prompts for {} s",
1786 self.backoff_seconds().expect("backoff retry duration is present")
1787 ),
1788 })
1789 }
1790}
1791
1792impl OperatorConfirmError {
1793 fn backoff_seconds(&self) -> Option<u64> {
1794 match self {
1795 Self::Backoff {
1796 retry_after_ms: Some(milliseconds),
1797 } => Some(milliseconds / 1_000 + u64::from(!milliseconds.is_multiple_of(1_000))),
1798 _ => None,
1799 }
1800 }
1801}
1802
1803impl Error for OperatorConfirmError {}
1804
1805#[derive(Debug)]
1808pub enum CatalogUpdateError {
1809 NotSupported,
1810 FrozenField(ErrorBody),
1811 NotRegistered(ErrorBody),
1812 Rejected(ErrorBody),
1813 Timeout,
1814 ConnectionClosed,
1815 Protocol(String),
1816}
1817
1818impl fmt::Display for CatalogUpdateError {
1819 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1820 match self {
1821 Self::NotSupported => write!(
1822 f,
1823 "daemon HELLO_ACK did not advertise catalog.update support"
1824 ),
1825 Self::FrozenField(body) => {
1826 write!(f, "catalog.update rejected frozen field: {}", body.message)
1827 }
1828 Self::NotRegistered(body) => write!(
1829 f,
1830 "catalog.update requires a registered module: {}",
1831 body.message
1832 ),
1833 Self::Rejected(body) => write!(
1834 f,
1835 "catalog.update rejected by subc: {} ({})",
1836 body.code, body.message
1837 ),
1838 Self::Timeout => write!(f, "catalog.update timed out waiting for an ACK"),
1839 Self::ConnectionClosed => {
1840 write!(f, "subc connection closed before catalog.update completed")
1841 }
1842 Self::Protocol(message) => write!(f, "catalog.update protocol error: {message}"),
1843 }
1844 }
1845}
1846
1847impl Error for CatalogUpdateError {}
1848
1849#[derive(Debug, Clone, PartialEq, Eq)]
1858#[non_exhaustive]
1859pub enum ScopeCallError {
1860 NotSupported { op: &'static str },
1863 Refused { code: String, message: String },
1868 Timeout,
1870 ConnectionClosed,
1872 Protocol(String),
1874}
1875
1876impl ScopeCallError {
1877 pub fn code(&self) -> Option<&str> {
1879 match self {
1880 Self::Refused { code, .. } => Some(code),
1881 _ => None,
1882 }
1883 }
1884
1885 fn from_control_error(err: CatalogUpdateError) -> Self {
1890 match err {
1891 CatalogUpdateError::FrozenField(body)
1892 | CatalogUpdateError::NotRegistered(body)
1893 | CatalogUpdateError::Rejected(body) => Self::Refused {
1894 code: body.code,
1895 message: body.message,
1896 },
1897 CatalogUpdateError::Timeout => Self::Timeout,
1898 CatalogUpdateError::ConnectionClosed => Self::ConnectionClosed,
1899 CatalogUpdateError::Protocol(message) => Self::Protocol(message),
1900 CatalogUpdateError::NotSupported => {
1903 Self::Protocol("unexpected NotSupported from the control reply table".into())
1904 }
1905 }
1906 }
1907}
1908
1909impl fmt::Display for ScopeCallError {
1910 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1911 match self {
1912 Self::NotSupported { op } => {
1913 write!(f, "daemon HELLO_ACK did not advertise {op} support")
1914 }
1915 Self::Refused { code, message } => {
1916 write!(f, "subc refused the request: {code} ({message})")
1917 }
1918 Self::Timeout => write!(f, "timed out waiting for the daemon's reply"),
1919 Self::ConnectionClosed => {
1920 write!(f, "subc connection closed before the daemon replied")
1921 }
1922 Self::Protocol(message) => write!(f, "scope call protocol error: {message}"),
1923 }
1924 }
1925}
1926
1927impl Error for ScopeCallError {}
1928
1929#[async_trait]
1977pub trait ModuleHandler: Send + Sync + 'static {
1978 async fn handle(&self, ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome;
1983
1984 async fn on_hello_ack(&self, _ack: &ModuleHelloAckBody) {}
1987
1988 fn control_ops(&self) -> Vec<String> {
1994 Vec::new()
1995 }
1996
1997 async fn on_control_request(
2010 &self,
2011 _op: &str,
2012 _body: serde_json::Value,
2013 _cancel: CancellationToken,
2014 ) -> Result<serde_json::Value, ErrorBody> {
2015 Err(ErrorBody::new(
2016 "unsupported_op",
2017 "no control handler implemented",
2018 ))
2019 }
2020
2021 fn config_declaration(&self) -> Option<config::ConfigDeclareRequest> {
2025 None
2026 }
2027
2028 async fn config_review(
2031 &self,
2032 _request: config::ConfigReviewRequest,
2033 ) -> config::ConfigReviewReply {
2034 config::ConfigReviewReply::new(
2035 vec![],
2036 vec![config::ConfigReviewRefusal::new(
2037 "value_invalid",
2038 vec![],
2039 vec!["module has no config review implementation".into()],
2040 )],
2041 )
2042 }
2043
2044 async fn config_apply(&self, _view: config::ConfigView) -> Result<(), Vec<String>> {
2048 Err(vec!["module has no config apply implementation".into()])
2049 }
2050
2051 async fn config_files(&self) {}
2054
2055 async fn on_config_state(&self, _state: config::ConfigState) {}
2057
2058 async fn on_config_error(&self, error: config::ConfigError) {
2061 tracing::warn!(%error, "module configuration request failed");
2062 }
2063
2064 async fn on_bind(&self, _req: &RouteBindRequest) -> BindDecision {
2066 BindDecision::accept()
2067 }
2068
2069 async fn on_bound(&self, _handle: &RouteHandle) {}
2071
2072 async fn health(&self) -> HealthReport {
2102 HealthReport {
2115 detail: Some("no health implementation; inherited default".to_string()),
2116 ..HealthReport::ok()
2117 }
2118 }
2119
2120 async fn on_route_gone(&self, _handle: &RouteHandle) {}
2122
2123 fn request_pools(&self) -> RequestPools {
2155 RequestPools::new()
2156 }
2157
2158 async fn on_draining(&self, _reason: RouteCloseReason, _deadline: SystemTime) {}
2176
2177 async fn on_connection_end(&self, _end: ConnectionEnd) {}
2184}
2185
2186#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2191#[non_exhaustive]
2192pub enum ConnectionEnd {
2193 Goodbye,
2195 Eof,
2197 Reset,
2200 Closed,
2202}
2203
2204impl ConnectionEnd {
2205 pub fn as_str(self) -> &'static str {
2207 match self {
2208 Self::Goodbye => "goodbye",
2209 Self::Eof => "eof",
2210 Self::Reset => "reset",
2211 Self::Closed => "closed",
2212 }
2213 }
2214}
2215
2216impl std::fmt::Display for ConnectionEnd {
2217 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2218 f.write_str(self.as_str())
2219 }
2220}
2221
2222#[derive(Debug, Clone, PartialEq)]
2224pub enum HandlerOutcome {
2225 Response(Vec<u8>),
2227 Error { code: String, message: String },
2229 ErrorWithDetail {
2231 code: String,
2232 message: String,
2233 detail: serde_json::Value,
2234 },
2235 Streamed,
2238}
2239
2240#[derive(Clone)]
2243pub struct RequestCtx {
2244 handle: RouteHandle,
2245 corr: u64,
2246 ver: u8,
2247 egress: mpsc::Sender<Frame>,
2248 module_handle: ModuleHandle,
2249 cancelled: CancellationToken,
2250 binary_reply: Arc<AtomicBool>,
2251 request_flags: Flags,
2252 request_body: Arc<[u8]>,
2253}
2254
2255impl RequestCtx {
2256 pub fn module_handle(&self) -> ModuleHandle {
2258 self.module_handle.clone()
2259 }
2260
2261 pub fn request_payload(
2266 &self,
2267 ) -> Result<Option<(serde_json::Value, &[u8])>, subc_protocol::framed_body::FramedBodyError>
2268 {
2269 if !self.request_flags.is_binary() {
2270 return Ok(None);
2271 }
2272 subc_protocol::framed_body::decode(&self.request_body).map(Some)
2273 }
2274
2275 pub fn route_handle(&self) -> RouteHandle {
2277 self.handle
2278 }
2279
2280 pub fn corr(&self) -> u64 {
2282 self.corr
2283 }
2284
2285 pub fn mark_binary_reply(&self) {
2297 self.binary_reply.store(true, Ordering::Relaxed);
2298 }
2299
2300 pub async fn emit(&self, body: Vec<u8>) -> Result<(), SubcModuleError> {
2303 self.emit_with_admission(body, None).await
2304 }
2305
2306 pub async fn emit_with_admission(
2308 &self,
2309 body: Vec<u8>,
2310 admission_class: Option<AdmissionClass>,
2311 ) -> Result<(), SubcModuleError> {
2312 self.module_handle.validate_route(self.handle)?;
2313 if self.cancelled.is_cancelled() {
2314 return Ok(());
2315 }
2316 self.send_frame(
2317 FrameType::StreamData,
2318 data_flags().with_admission_class(admission_class.unwrap_or(AdmissionClass::Normal)),
2319 body,
2320 )
2321 .await
2322 }
2323
2324 pub fn cancelled(&self) -> WaitForCancellationFuture<'_> {
2327 self.cancelled.cancelled()
2328 }
2329
2330 pub fn cancellation_token(&self) -> CancellationToken {
2332 self.cancelled.clone()
2333 }
2334
2335 async fn send_frame(
2336 &self,
2337 frame_type: FrameType,
2338 flags: Flags,
2339 body: Vec<u8>,
2340 ) -> Result<(), SubcModuleError> {
2341 self.module_handle.validate_route(self.handle)?;
2342 let (frame_type, flags, body) = bounded_terminal(frame_type, flags, body)?;
2343 let frame = Frame::build_with_version(
2344 self.ver,
2345 frame_type,
2346 flags,
2347 self.handle.channel,
2348 self.handle.epoch,
2349 self.corr,
2350 body,
2351 )
2352 .map_err(SubcModuleError::FrameBuild)?;
2353 send_outbound(&self.egress, frame).await
2354 }
2355}
2356
2357fn checked_increment(counter: &AtomicU64) -> Option<u64> {
2362 let mut current = counter.load(Ordering::Relaxed);
2363 loop {
2364 let next = current.checked_add(1)?;
2365 match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
2366 Ok(previous) => return Some(previous),
2367 Err(actual) => current = actual,
2368 }
2369 }
2370}
2371
2372#[derive(Debug, Clone)]
2379#[non_exhaustive]
2380pub struct RouteBindRequest {
2381 pub handle: RouteHandle,
2382 pub target: RouteTarget,
2383 pub identity: BindIdentity,
2384 pub principal: Option<Principal>,
2385 pub consumer_capabilities: Option<Vec<String>>,
2392 pub role_versions: Option<std::collections::BTreeMap<String, String>>,
2399 pub admission_facts: Option<serde_json::Value>,
2401 pub scope: Option<ScopeStamp>,
2410 pub call_mode: CallMode,
2413 pub peer: subc_protocol::peer::RouteBindPeer,
2441}
2442
2443impl RouteBindRequest {
2444 pub fn new(handle: RouteHandle, target: RouteTarget, identity: BindIdentity) -> Self {
2449 Self {
2450 handle,
2451 target,
2452 identity,
2453 principal: None,
2454 consumer_capabilities: None,
2455 role_versions: None,
2456 admission_facts: None,
2457 scope: None,
2458 call_mode: derive_call_mode(None),
2459 peer: subc_protocol::peer::RouteBindPeer::new(),
2460 }
2461 }
2462
2463 pub fn with_peer(mut self, peer: subc_protocol::peer::RouteBindPeer) -> Self {
2466 self.peer = peer;
2467 self
2468 }
2469
2470 pub fn peer_build(&self) -> Option<&str> {
2473 self.peer.peer_build.as_deref()
2474 }
2475
2476 pub fn with_principal(mut self, principal: Principal) -> Self {
2478 self.principal = Some(principal);
2479 self
2480 }
2481
2482 pub fn with_consumer_capabilities(mut self, consumer_capabilities: Vec<String>) -> Self {
2484 self.consumer_capabilities = Some(consumer_capabilities);
2485 self
2486 }
2487
2488 pub fn with_role_versions(
2490 mut self,
2491 role_versions: std::collections::BTreeMap<String, String>,
2492 ) -> Self {
2493 self.role_versions = Some(role_versions);
2494 self
2495 }
2496
2497 pub fn with_admission_facts(mut self, admission_facts: serde_json::Value) -> Self {
2499 self.admission_facts = Some(admission_facts);
2500 self
2501 }
2502
2503 pub fn with_scope(mut self, scope: ScopeStamp) -> Self {
2506 self.call_mode = derive_call_mode(Some(&scope));
2507 self.scope = Some(scope);
2508 self
2509 }
2510
2511 pub fn with_call_mode(mut self, call_mode: CallMode) -> Self {
2513 self.call_mode = call_mode;
2514 self
2515 }
2516}
2517
2518#[derive(Debug, Clone)]
2520pub struct BindDecision {
2521 kind: BindDecisionKind,
2522}
2523
2524impl BindDecision {
2525 pub fn accept() -> Self {
2527 Self {
2528 kind: BindDecisionKind::Accept,
2529 }
2530 }
2531
2532 pub fn reject(code: impl Into<String>, message: impl Into<String>) -> Self {
2534 Self {
2535 kind: BindDecisionKind::Reject {
2536 code: code.into(),
2537 message: message.into(),
2538 detail: None,
2539 },
2540 }
2541 }
2542
2543 pub fn reject_with_detail(
2550 code: impl Into<String>,
2551 message: impl Into<String>,
2552 detail: serde_json::Value,
2553 ) -> Self {
2554 Self {
2555 kind: BindDecisionKind::Reject {
2556 code: code.into(),
2557 message: message.into(),
2558 detail: Some(detail),
2559 },
2560 }
2561 }
2562}
2563
2564#[derive(Debug, Clone)]
2565enum BindDecisionKind {
2566 Accept,
2567 Reject {
2568 code: String,
2569 message: String,
2570 detail: Option<serde_json::Value>,
2571 },
2572}
2573
2574pub async fn serve<H>(manifest: ModuleManifest, handler: H) -> Result<(), SubcModuleError>
2583where
2584 H: ModuleHandler,
2585{
2586 ServeOptions::default().serve(manifest, handler).await
2587}
2588
2589#[derive(Clone, Debug, Default)]
2593pub struct ModuleStopHandle {
2594 token: CancellationToken,
2595}
2596
2597impl ModuleStopHandle {
2598 pub fn new() -> Self {
2599 Self::default()
2600 }
2601
2602 pub fn stop(&self) {
2605 self.token.cancel();
2606 }
2607}
2608
2609#[derive(Clone, Debug)]
2612#[non_exhaustive]
2613pub struct ServeOptions {
2614 registration_channel: bool,
2615 prefix_child: bool,
2616 writer_drain_timeout: Duration,
2617 stop_handle: ModuleStopHandle,
2618}
2619
2620impl Default for ServeOptions {
2621 fn default() -> Self {
2622 Self {
2623 writer_drain_timeout: DEFAULT_WRITER_DRAIN_TIMEOUT,
2624 registration_channel: false,
2625 prefix_child: false,
2626 stop_handle: ModuleStopHandle::new(),
2627 }
2628 }
2629}
2630
2631impl ServeOptions {
2632 pub fn with_writer_drain_timeout(mut self, timeout: Duration) -> Self {
2642 self.writer_drain_timeout = timeout;
2643 self
2644 }
2645
2646 pub fn with_stop_handle(mut self, handle: ModuleStopHandle) -> Self {
2649 self.stop_handle = handle;
2650 self
2651 }
2652
2653 pub async fn serve<H: ModuleHandler>(
2655 self,
2656 mut manifest: ModuleManifest,
2657 handler: H,
2658 ) -> Result<(), SubcModuleError> {
2659 let connection_file = parse_subc_arg(env::args_os().skip(1))?;
2660 if let Some(module_id) = module_id_from_env()? {
2661 manifest.module_id = module_id;
2662 }
2663 self.serve_with(&connection_file, manifest, handler).await
2664 }
2665
2666 pub async fn serve_with<H: ModuleHandler>(
2668 self,
2669 connection_file: &Path,
2670 manifest: ModuleManifest,
2671 handler: H,
2672 ) -> Result<(), SubcModuleError> {
2673 if let Some((_handle, future)) = self
2674 .serve_with_handle(connection_file, manifest, handler)
2675 .await?
2676 {
2677 future.await?;
2678 }
2679 Ok(())
2680 }
2681
2682 pub async fn serve_from_env_with_handle<H: ModuleHandler>(
2696 self,
2697 mut manifest: ModuleManifest,
2698 handler: H,
2699 ) -> Result<Option<(ModuleHandle, ModuleServeFuture)>, SubcModuleError> {
2700 let connection_file = parse_subc_arg(env::args_os().skip(1))?;
2701 if let Some(module_id) = module_id_from_env()? {
2702 manifest.module_id = module_id;
2703 }
2704 self.serve_with_handle(&connection_file, manifest, handler)
2705 .await
2706 }
2707
2708 pub async fn serve_with_handle<H: ModuleHandler>(
2720 self,
2721 connection_file: &Path,
2722 manifest: ModuleManifest,
2723 handler: H,
2724 ) -> Result<Option<(ModuleHandle, ModuleServeFuture)>, SubcModuleError> {
2725 prepare_connection(connection_file, manifest, handler, self).await
2726 }
2727}
2728
2729pub async fn serve_from_env_with_handle<H>(
2744 manifest: ModuleManifest,
2745 handler: H,
2746) -> Result<(ModuleHandle, ModuleServeFuture), SubcModuleError>
2747where
2748 H: ModuleHandler,
2749{
2750 ServeOptions::default()
2751 .serve_from_env_with_handle(manifest, handler)
2752 .await?
2753 .ok_or(SubcModuleError::ConnectionClosedBeforeHelloAck)
2754}
2755
2756pub async fn serve_with<H>(
2759 connection_file: &Path,
2760 manifest: ModuleManifest,
2761 handler: H,
2762) -> Result<(), SubcModuleError>
2763where
2764 H: ModuleHandler,
2765{
2766 ServeOptions::default()
2767 .serve_with(connection_file, manifest, handler)
2768 .await
2769}
2770
2771pub async fn serve_with_handle<H>(
2774 connection_file: &Path,
2775 manifest: ModuleManifest,
2776 handler: H,
2777) -> Result<(ModuleHandle, ModuleServeFuture), SubcModuleError>
2778where
2779 H: ModuleHandler,
2780{
2781 ServeOptions::default()
2782 .serve_with_handle(connection_file, manifest, handler)
2783 .await?
2784 .ok_or(SubcModuleError::ConnectionClosedBeforeHelloAck)
2785}
2786
2787async fn prepare_connection<H: ModuleHandler>(
2788 connection_file: &Path,
2789 manifest: ModuleManifest,
2790 handler: H,
2791 mut options: ServeOptions,
2792) -> Result<Option<(ModuleHandle, ModuleServeFuture)>, SubcModuleError> {
2793 options.registration_channel = registration::adopt(&manifest.module_id)?;
2794 let stream = tokio::select! {
2795 biased;
2796 () = options.stop_handle.token.cancelled() => return Ok(None),
2797 stream = async {
2798 let claim = if registration::supervised() && !options.registration_channel {
2799 registration::prefix_claim(&manifest.module_id).await
2800 .map_err(|error| registration::serve_claim_error(&manifest.module_id, error))?
2801 } else {
2802 None
2803 };
2804 let mut stream = connect_to_subc(connection_file, options.registration_channel).await?;
2805 if let Some(claim) = claim {
2806 registration::present_claim(&mut stream, claim, AUTH_DEADLINE).await
2807 .map_err(|error| registration::serve_claim_error(&manifest.module_id, error))?;
2808 options.prefix_child = true;
2809 }
2810 Ok::<_, SubcModuleError>(stream)
2811 } => stream?,
2812 };
2813 register_connection(stream, manifest, handler, options).await
2814}
2815
2816async fn register_connection<S, H>(
2817 stream: S,
2818 manifest: ModuleManifest,
2819 handler: H,
2820 options: ServeOptions,
2821) -> Result<Option<(ModuleHandle, ModuleServeFuture)>, SubcModuleError>
2822where
2823 S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
2824 H: ModuleHandler,
2825{
2826 let control_ops = control_ops::validate(handler.control_ops())?;
2827 let (mut read_half, write_half) = tokio::io::split(stream);
2828 let (tx, rx) = mpsc::channel::<Frame>(EGRESS_BUFFER);
2829 let connection_runtime = tokio::runtime::Handle::current();
2830 let mut writer = tokio::spawn(drain_writer(write_half, rx));
2831 let handler = Arc::new(handler);
2832 let declaration = if options.prefix_child {
2833 None
2834 } else {
2835 handler.config_declaration()
2836 };
2837
2838 let mut hello_ack = None;
2839 let provides = manifest.provides.clone();
2840 let handshake = async {
2841 send_hello(&tx, manifest, declaration.is_some(), &control_ops).await?;
2842 hello_ack = Some(expect_hello_ack(&mut read_half).await?);
2845 handler
2846 .on_hello_ack(hello_ack.as_ref().expect("validated HELLO_ACK"))
2847 .await;
2848 Ok::<_, SubcModuleError>(())
2849 };
2850 let mut writer_result = None;
2851 let handshake_result = tokio::select! {
2852 biased;
2853 () = options.stop_handle.token.cancelled() => None,
2854 result = handshake => Some(result),
2855 result = &mut writer => {
2856 writer_result = Some(result);
2857 Some(Ok(()))
2858 }
2859 };
2860 match handshake_result {
2861 Some(Ok(())) => {}
2862 other => {
2863 let result = match other {
2864 None => {
2865 let deadline = Instant::now() + options.writer_drain_timeout;
2866 let result = timeout_at(deadline, send_goodbye(&tx)).await;
2867 drop(tx);
2868 let flush = finish_writer_and_drain(writer, &mut read_half, deadline).await;
2869 handler.on_connection_end(ConnectionEnd::Closed).await;
2870 match result.unwrap_or(Ok(())) {
2871 Ok(()) | Err(SubcModuleError::WriterClosed) => {}
2872 Err(error) => return Err(error),
2873 }
2874 match flush.0 {
2875 Err(SubcModuleError::FrameIo(FrameIoError::Io(error)))
2876 if is_disconnect(&error) => {}
2877 result => result?,
2878 }
2879 return Ok(None);
2880 }
2881 Some(Err(error)) => error,
2882 Some(Ok(_)) => unreachable!(),
2883 };
2884 drop(tx);
2885 let deadline = Instant::now() + options.writer_drain_timeout;
2886 let _ = finish_writer_and_drain(writer, &mut read_half, deadline).await;
2887 return Err(result);
2888 }
2889 }
2890 let mut ack = match hello_ack {
2891 Some(ack) => ack,
2892 None => {
2893 return Err(writer_ended(
2894 writer_result.expect("writer ended before HELLO_ACK"),
2895 ))
2896 }
2897 };
2898 if options.prefix_child {
2899 ack.subc_ops.retain(|op| {
2902 matches!(
2903 op.as_str(),
2904 "catalog.update" | "scope.describe" | "supervisor.live_roots" | "activity.record"
2905 )
2906 });
2907 }
2908 let writer_end = writer_result.map(writer_connection_end);
2911
2912 let Some(connection_token) = checked_increment(&NEXT_MODULE_CONNECTION_TOKEN) else {
2913 drop(tx);
2914 let deadline = Instant::now() + options.writer_drain_timeout;
2915 if writer_end.is_none() {
2916 let _ = finish_writer_and_drain(writer, &mut read_half, deadline).await;
2917 }
2918 return Err(SubcModuleError::ConnectionTokenExhausted);
2919 };
2920 let close_token = CancellationToken::new();
2921 let handle = ModuleHandle::new(
2922 &ack,
2923 tx.clone(),
2924 connection_token,
2925 close_token,
2926 connection_runtime,
2927 );
2928 handle.shared.lock_inner().current_provides = Some(provides);
2929 handle
2930 .shared
2931 .registration
2932 .prefix_child
2933 .store(options.prefix_child, Ordering::Release);
2934 let _ = handle.shared.served_control_ops.set(control_ops);
2935 if options.registration_channel {
2936 registration::install_owner(&handle)?;
2937 }
2938 if let Some(declaration) = declaration {
2939 let _ = handle.shared.config.declaration.set(declaration);
2940 }
2941 let serve_future = connection_serve_future_with_options(
2942 read_half,
2943 tx,
2944 handler,
2945 handle.clone(),
2946 writer,
2947 options,
2948 writer_end,
2949 );
2950 Ok(Some((handle, serve_future)))
2951}
2952
2953#[cfg(any(test, feature = "test-support"))]
2956fn connection_serve_future<R, H>(
2957 read_half: R,
2958 tx: mpsc::Sender<Frame>,
2959 handler: Arc<H>,
2960 handle: ModuleHandle,
2961 writer: tokio::task::JoinHandle<Result<(), WriterFailure>>,
2962) -> ModuleServeFuture
2963where
2964 R: AsyncRead + Unpin + Send + 'static,
2965 H: ModuleHandler,
2966{
2967 connection_serve_future_with_options(
2968 read_half,
2969 tx,
2970 handler,
2971 handle,
2972 writer,
2973 ServeOptions::default(),
2974 None,
2975 )
2976}
2977
2978fn registration_channel_exit_code(result: &Result<ConnectionEnd, SubcModuleError>) -> i32 {
2979 if matches!(result, Ok(ConnectionEnd::Goodbye)) {
2980 0
2981 } else {
2982 1
2983 }
2984}
2985
2986fn connection_serve_future_with_options<R, H>(
2987 mut read_half: R,
2988 tx: mpsc::Sender<Frame>,
2989 handler: Arc<H>,
2990 handle: ModuleHandle,
2991 mut writer: tokio::task::JoinHandle<Result<(), WriterFailure>>,
2992 options: ServeOptions,
2993 writer_end: Option<Result<ConnectionEnd, SubcModuleError>>,
2994) -> ModuleServeFuture
2995where
2996 R: AsyncRead + Unpin + Send + 'static,
2997 H: ModuleHandler,
2998{
2999 let serve_handle = handle.clone();
3000 handle.install_dispatch_pools(handler.request_pools());
3004 Box::pin(async move {
3005 let dispatcher = RequestDispatcher::with_pools(serve_handle.dispatch_pools());
3009 if !options.prefix_child {
3010 config::start(serve_handle.clone(), Arc::clone(&handler), &dispatcher);
3011 }
3012 let mut writer_finished = writer_end.is_some();
3013 let mut loop_result = if let Some(end) = writer_end {
3014 end
3015 } else {
3016 tokio::select! {
3017 biased;
3018 result = module_loop(&mut read_half, &tx, Arc::clone(&handler), serve_handle.clone(), &dispatcher, &options.stop_handle) => result,
3019 result = &mut writer => {
3020 writer_finished = true;
3021 writer_connection_end(result)
3022 }
3023 }
3024 };
3025 if matches!(loop_result, Err(SubcModuleError::WriterClosed)) && !writer_finished {
3029 writer_finished = true;
3030 loop_result = writer_connection_end((&mut writer).await);
3031 }
3032 let deadline = Instant::now() + options.writer_drain_timeout;
3033 serve_handle.shared.config.stop.cancel();
3034 dispatcher.wait_for_draining_hooks_to_start().await;
3035 dispatcher.tasks.close();
3036 if matches!(loop_result, Ok(ConnectionEnd::Closed))
3037 && options.stop_handle.token.is_cancelled()
3038 {
3039 let _ = timeout_at(deadline, dispatcher.tasks.wait()).await;
3041 }
3042 let orderly = loop_result.is_ok();
3043 if let Some(error) = dispatcher.take_failure() {
3044 if !orderly || !matches!(error, SubcModuleError::WriterClosed) {
3045 loop_result = Err(error);
3046 }
3047 }
3048 dispatcher.cancel_all();
3049 let _ = timeout_at(deadline, dispatcher.tasks.wait()).await;
3050 let abandoned_replies = dispatcher
3051 .in_flight
3052 .lock()
3053 .unwrap_or_else(|e| e.into_inner())
3054 .len();
3055 if let Some(error) = dispatcher.take_failure() {
3056 if !orderly || !matches!(error, SubcModuleError::WriterClosed) {
3057 loop_result = Err(error);
3058 }
3059 }
3060 dispatcher.abort_all();
3061 dispatcher.tasks.wait().await;
3062 let daemon_eof = matches!(loop_result, Ok(ConnectionEnd::Eof));
3063 let locally_closed = matches!(loop_result, Ok(ConnectionEnd::Closed));
3064 let module_ended =
3069 !writer_finished && matches!(loop_result, Ok(ConnectionEnd::Closed) | Err(_));
3070 let deadline = Instant::now() + options.writer_drain_timeout;
3071 if matches!(loop_result, Ok(ConnectionEnd::Closed))
3072 && options.stop_handle.token.is_cancelled()
3073 {
3074 match timeout_at(deadline, send_goodbye(&tx)).await {
3075 Ok(Ok(())) | Err(_) => {}
3076 Ok(Err(SubcModuleError::WriterClosed)) => {}
3077 Ok(Err(error)) => loop_result = Err(error),
3078 }
3079 }
3080 serve_handle.close_connection();
3081 drop(tx);
3082 let loop_result = match loop_result {
3083 Ok(end) => {
3084 handler.on_connection_end(end).await;
3085 Ok(end)
3086 }
3087 Err(error) => Err(error),
3088 };
3089
3090 let (writer_result, undelivered_replies) = if writer_finished {
3091 (Ok(()), 0)
3092 } else if module_ended {
3093 finish_writer_and_drain(writer, &mut read_half, deadline).await
3094 } else {
3095 finish_writer(writer, deadline).await
3096 };
3097 let undelivered_replies = undelivered_replies as u64
3098 + dispatcher.undelivered_replies.load(Ordering::Relaxed)
3099 + abandoned_replies as u64;
3100 if daemon_eof && undelivered_replies > 0 {
3101 tracing::warn!(
3102 undelivered_replies,
3103 "daemon EOF: replies could not be delivered during drain"
3104 );
3105 }
3106 if options.registration_channel && !locally_closed {
3107 std::process::exit(registration_channel_exit_code(&loop_result));
3112 }
3113 match (loop_result.map(|_| ()), writer_result) {
3114 (Err(SubcModuleError::WriterClosed), Err(writer_error)) => Err(writer_error),
3115 (Err(loop_err), _) => Err(loop_err),
3116 (Ok(()), Ok(())) => Ok(()),
3117 (Ok(()), Err(SubcModuleError::FrameIo(FrameIoError::Io(err))))
3122 if is_disconnect(&err) =>
3123 {
3124 Ok(())
3125 }
3126 (Ok(()), Err(error)) => Err(error),
3127 }
3128 })
3129}
3130
3131async fn module_loop<R, H>(
3132 mut reader: R,
3133 egress: &mpsc::Sender<Frame>,
3134 handler: Arc<H>,
3135 module_handle: ModuleHandle,
3136 dispatcher: &RequestDispatcher,
3137 stop: &ModuleStopHandle,
3138) -> Result<ConnectionEnd, SubcModuleError>
3139where
3140 R: AsyncRead + Unpin,
3141 H: ModuleHandler,
3142{
3143 serve_frames(
3144 &mut reader,
3145 egress,
3146 &handler,
3147 &module_handle,
3148 dispatcher,
3149 stop,
3150 )
3151 .await
3152}
3153
3154async fn serve_frames<R, H>(
3155 reader: &mut R,
3156 egress: &mpsc::Sender<Frame>,
3157 handler: &Arc<H>,
3158 module_handle: &ModuleHandle,
3159 dispatcher: &RequestDispatcher,
3160 stop: &ModuleStopHandle,
3161) -> Result<ConnectionEnd, SubcModuleError>
3162where
3163 R: AsyncRead + Unpin,
3164 H: ModuleHandler,
3165{
3166 loop {
3167 let read = tokio::select! {
3168 biased;
3169 () = dispatcher.failed.cancelled() => return Err(dispatcher.take_failure().expect("failure signalled")),
3170 () = stop.token.cancelled() => return Ok(ConnectionEnd::Closed),
3171 () = module_handle.shared.close_token.cancelled() => return Ok(ConnectionEnd::Closed),
3172 read = read_frame(reader) => read,
3173 };
3174 let frame = match read {
3175 Ok(Some(frame)) => frame,
3176 Ok(None) => return Ok(ConnectionEnd::Eof),
3178 Err(FrameIoError::Io(error)) if is_disconnect(&error) => {
3181 return Ok(ConnectionEnd::Reset);
3182 }
3183 Err(err) => return Err(SubcModuleError::FrameIo(err)),
3184 };
3185 if !handle_frame(
3186 frame,
3187 egress,
3188 Arc::clone(handler),
3189 dispatcher.clone(),
3190 module_handle.clone(),
3191 )
3192 .await?
3193 {
3194 return Ok(ConnectionEnd::Goodbye);
3196 }
3197 }
3198}
3199
3200async fn handle_frame<H>(
3201 frame: Frame,
3202 egress: &mpsc::Sender<Frame>,
3203 handler: Arc<H>,
3204 dispatcher: RequestDispatcher,
3205 module_handle: ModuleHandle,
3206) -> Result<bool, SubcModuleError>
3207where
3208 H: ModuleHandler,
3209{
3210 if frame.header.channel != 0
3211 && !module_handle.validate_ingress(frame.header.channel, frame.header.epoch)?
3212 {
3213 return Ok(true);
3214 }
3215 match frame.header.ty {
3216 FrameType::Ping if frame.header.channel == 0 => {
3217 let pong = Frame::build_with_version(
3218 frame.header.ver,
3219 FrameType::Pong,
3220 frame.header.flags,
3221 0,
3222 0,
3223 frame.header.corr,
3224 Vec::new(),
3225 )
3226 .map_err(SubcModuleError::FrameBuild)?;
3227 send_outbound(egress, pong).await?;
3228 Ok(true)
3229 }
3230 FrameType::Goodbye if frame.header.channel == 0 => Ok(false),
3231 FrameType::Goodbye => {
3232 let handle = module_handle.route_handle(frame.header.channel, frame.header.epoch);
3233 if module_handle.remove_route(handle)? {
3234 cancel_handle(&dispatcher.in_flight, handle)?;
3235 handler.on_route_gone(&handle).await;
3236 }
3237 Ok(true)
3238 }
3239 FrameType::Response if frame.header.channel == 0 => {
3240 let _ = module_handle.handle_control_reply(frame);
3241 Ok(true)
3242 }
3243 FrameType::Error | FrameType::StreamData | FrameType::StreamEnd
3244 if frame.header.channel == 0 =>
3245 {
3246 let _ = module_handle.handle_control_reply(frame);
3247 Ok(true)
3248 }
3249 FrameType::Cancel => {
3250 handle_cancel(frame, &dispatcher.in_flight)?;
3251 Ok(true)
3252 }
3253 FrameType::Request if frame.header.channel == 0 => {
3254 handle_control_request(frame, egress, handler, dispatcher, module_handle.clone())
3255 .await?;
3256 Ok(true)
3257 }
3258 FrameType::Request => {
3259 spawn_data_request(frame, egress.clone(), handler, dispatcher, module_handle)?;
3260 Ok(true)
3261 }
3262 FrameType::Push if frame.header.channel == 0 => {
3263 if config::changed(&frame, &module_handle) {
3264 return Ok(true);
3265 }
3266 handle_control_push(&frame, handler, &dispatcher);
3267 Ok(true)
3268 }
3269 _ => Ok(true),
3270 }
3271}
3272
3273fn handle_control_push<H>(frame: &Frame, handler: Arc<H>, dispatcher: &RequestDispatcher)
3278where
3279 H: ModuleHandler,
3280{
3281 match serde_json::from_slice::<ModuleControlCommand>(&frame.body) {
3282 Ok(ModuleControlCommand::Draining {
3283 reason,
3284 deadline_ms,
3285 }) => {
3286 let started = spawn_draining_hook(
3287 handler,
3288 sdk_route_close_reason(reason),
3289 drain_deadline(deadline_ms),
3290 dispatcher,
3291 );
3292 dispatcher
3293 .draining_hooks
3294 .lock()
3295 .unwrap_or_else(|poisoned| poisoned.into_inner())
3296 .push(started);
3297 }
3298 Err(error) => {
3299 if !dispatcher
3300 .undecodable_push_logged
3301 .swap(true, Ordering::Relaxed)
3302 {
3303 let op = serde_json::from_slice::<serde_json::Value>(&frame.body)
3304 .ok()
3305 .and_then(|body| body.get("op")?.as_str().map(str::to_string));
3306 tracing::warn!(
3307 op = op.as_deref().unwrap_or("<none>"),
3308 error = %error,
3309 "ignoring a channel-0 push from subc that this SDK cannot decode; \
3310 further undecodable pushes on this connection are ignored without logging"
3311 );
3312 }
3313 }
3314 }
3315}
3316
3317fn spawn_draining_hook<H>(
3321 handler: Arc<H>,
3322 reason: RouteCloseReason,
3323 deadline: SystemTime,
3324 dispatcher: &RequestDispatcher,
3325) -> oneshot::Receiver<()>
3326where
3327 H: ModuleHandler,
3328{
3329 let (started_tx, started_rx) = oneshot::channel();
3330 dispatcher.spawn(async move {
3331 let mut hook = handler.on_draining(reason, deadline);
3332 let mut started_tx = Some(started_tx);
3333 std::future::poll_fn(|cx| {
3334 let poll = hook.as_mut().poll(cx);
3335 if let Some(started) = started_tx.take() {
3336 let _ = started.send(());
3337 }
3338 poll
3339 })
3340 .await;
3341 });
3342 started_rx
3343}
3344
3345fn sdk_route_close_reason(reason: subc_protocol::RouteCloseReason) -> RouteCloseReason {
3349 match serde_json::to_value(reason) {
3350 Ok(serde_json::Value::String(wire)) => RouteCloseReason::from_wire(&wire),
3351 _ => RouteCloseReason::Unknown(format!("{reason:?}")),
3352 }
3353}
3354
3355fn drain_deadline(deadline_ms: u64) -> SystemTime {
3359 UNIX_EPOCH
3360 .checked_add(Duration::from_millis(deadline_ms))
3361 .unwrap_or_else(SystemTime::now)
3362}
3363
3364fn spawn_data_request<H>(
3365 frame: Frame,
3366 egress: mpsc::Sender<Frame>,
3367 handler: Arc<H>,
3368 dispatcher: RequestDispatcher,
3369 module_handle: ModuleHandle,
3370) -> Result<(), SubcModuleError>
3371where
3372 H: ModuleHandler,
3373{
3374 let handle = module_handle.route_handle(frame.header.channel, frame.header.epoch);
3375 let corr = frame.header.corr;
3376 let cancellation = CancellationToken::new();
3377 {
3378 let mut guard = lock_in_flight(&dispatcher.in_flight)?;
3379 guard.insert((handle.channel, handle.epoch, corr), cancellation.clone());
3380 }
3381
3382 let ctx = RequestCtx {
3383 handle,
3384 corr,
3385 ver: frame.header.ver,
3386 egress,
3387 module_handle,
3388 cancelled: cancellation,
3389 binary_reply: Arc::new(AtomicBool::new(false)),
3390 request_flags: frame.header.flags,
3391 request_body: Arc::from(frame.body.as_slice()),
3392 };
3393 let body = frame.body;
3394 let in_flight = Arc::clone(&dispatcher.in_flight);
3395 let (pool, op) = dispatcher.pools.classify(&body);
3398 let pool = pool.clone();
3399 let wait = pool.start_waiting((handle.channel, handle.epoch, corr));
3400 dispatcher.clone().spawn(async move {
3401 let waiting = WaitingRequest {
3402 op,
3403 channel: handle.channel,
3404 corr,
3405 };
3406 let permit = match wait_for_permit(&pool, wait, waiting, &ctx.cancelled).await {
3407 PermitWaitOutcome::Acquired(permit) => Some(permit),
3408 PermitWaitOutcome::Cancelled => None,
3411 PermitWaitOutcome::Closed => {
3412 dispatcher
3414 .undelivered_replies
3415 .fetch_add(1, Ordering::Relaxed);
3416 if let Ok(mut guard) = in_flight.lock() {
3417 guard.remove(&(handle.channel, handle.epoch, corr));
3418 }
3419 return;
3420 }
3421 };
3422 let outcome = if permit.is_none() || ctx.cancelled.is_cancelled() {
3425 Ok(HandlerOutcome::Error {
3426 code: "cancelled".to_string(),
3427 message: "request cancelled".to_string(),
3428 })
3429 } else {
3430 catch_handler_panic(async { handler.handle(ctx.clone(), body).await }).await
3431 };
3432 let panicked = outcome.is_err();
3433 let outcome = outcome.unwrap_or_else(|()| HandlerOutcome::Error {
3434 code: "internal".to_string(),
3435 message: "module handler panicked".to_string(),
3436 });
3437 match send_handler_outcome(&ctx, outcome).await {
3438 Err(SubcModuleError::StaleRouteHandle(_)) if !panicked => {}
3442 Err(error) => {
3443 if matches!(error, SubcModuleError::WriterClosed) {
3444 dispatcher
3445 .undelivered_replies
3446 .fetch_add(1, Ordering::Relaxed);
3447 }
3448 dispatcher.fail(error);
3449 }
3450 Ok(()) if panicked => dispatcher.fail(SubcModuleError::HandlerPanic),
3451 Ok(()) => {}
3452 }
3453 if let Ok(mut guard) = in_flight.lock() {
3454 guard.remove(&(handle.channel, handle.epoch, corr));
3455 }
3456 });
3457 Ok(())
3458}
3459
3460fn spawn_health_request<H>(
3461 frame: Frame,
3462 egress: mpsc::Sender<Frame>,
3463 handler: Arc<H>,
3464 dispatcher: RequestDispatcher,
3465) -> Result<(), SubcModuleError>
3466where
3467 H: ModuleHandler,
3468{
3469 let channel = frame.header.channel;
3470 let epoch = frame.header.epoch;
3471 let corr = frame.header.corr;
3472 let ver = frame.header.ver;
3473 let cancellation = CancellationToken::new();
3474 {
3475 let mut guard = lock_in_flight(&dispatcher.in_flight)?;
3476 guard.insert((channel, epoch, corr), cancellation.clone());
3477 }
3478
3479 let in_flight = Arc::clone(&dispatcher.in_flight);
3480 dispatcher.clone().spawn(async move {
3481 if !cancellation.is_cancelled() {
3484 let result = async {
3485 let report = catch_handler_panic(async { handler.health().await }).await;
3486 let panicked = report.is_err();
3487 let (ty, body) = match report {
3488 Ok(report) => (
3489 FrameType::Response,
3490 serde_json::to_vec(&ModuleControlResponse::from(
3491 dispatcher.fold_health(report),
3492 ))?,
3493 ),
3494 Err(()) => (
3495 FrameType::Error,
3496 serde_json::to_vec(&ErrorBody::new("internal", "module handler panicked"))?,
3497 ),
3498 };
3499 let (ty, flags, body) = bounded_terminal(ty, control_flags(), body)?;
3500 let frame = Frame::build_with_version(ver, ty, flags, channel, epoch, corr, body)
3501 .map_err(SubcModuleError::FrameBuild)?;
3502 send_outbound(&egress, frame).await?;
3503 if panicked {
3504 return Err(SubcModuleError::HandlerPanic);
3505 }
3506 Ok(())
3507 }
3508 .await;
3509 if let Err(error) = result {
3510 if matches!(error, SubcModuleError::WriterClosed) {
3511 dispatcher
3512 .undelivered_replies
3513 .fetch_add(1, Ordering::Relaxed);
3514 }
3515 dispatcher.fail(error);
3516 }
3517 }
3518 if let Ok(mut guard) = in_flight.lock() {
3519 guard.remove(&(channel, epoch, corr));
3520 }
3521 });
3522 Ok(())
3523}
3524
3525async fn send_handler_outcome(
3526 ctx: &RequestCtx,
3527 outcome: HandlerOutcome,
3528) -> Result<(), SubcModuleError> {
3529 match outcome {
3530 HandlerOutcome::Response(body) => {
3531 let flags = Flags::new(
3532 ctx.binary_reply.load(Ordering::Relaxed),
3533 Priority::Interactive,
3534 false,
3535 );
3536 ctx.send_frame(FrameType::Response, flags, body).await
3537 }
3538 HandlerOutcome::Error { code, message } => {
3539 let body = serde_json::to_vec(&ErrorBody::new(code, message))
3540 .map_err(SubcModuleError::Json)?;
3541 ctx.send_frame(FrameType::Error, data_flags(), body).await
3542 }
3543 HandlerOutcome::ErrorWithDetail {
3544 code,
3545 message,
3546 detail,
3547 } => {
3548 let body = serde_json::to_vec(&ErrorBody::new(code, message).with_detail(detail))
3549 .map_err(SubcModuleError::Json)?;
3550 ctx.send_frame(FrameType::Error, data_flags(), body).await
3551 }
3552 HandlerOutcome::Streamed => {
3553 ctx.send_frame(FrameType::StreamEnd, data_flags(), Vec::new())
3554 .await
3555 }
3556 }
3557}
3558
3559async fn catch_handler_panic<F: Future>(future: F) -> Result<F::Output, ()> {
3562 let mut future = std::pin::pin!(future);
3563 std::future::poll_fn(
3564 |cx| match catch_unwind(AssertUnwindSafe(|| future.as_mut().poll(cx))) {
3565 Ok(poll) => poll.map(Ok),
3566 Err(_) => std::task::Poll::Ready(Err(())),
3567 },
3568 )
3569 .await
3570}
3571
3572fn bounded_terminal(
3573 ty: FrameType,
3574 flags: Flags,
3575 body: Vec<u8>,
3576) -> Result<(FrameType, Flags, Vec<u8>), SubcModuleError> {
3577 let size = subc_protocol::HEADER_LEN + body.len();
3578 let limit = subc_protocol::MAX_DELIVERED_FRAME_LEN;
3579 if matches!(ty, FrameType::Response | FrameType::Error) && size > limit {
3580 let body = serde_json::to_vec(
3581 &ErrorBody::new("frame_too_large", "module reply exceeds frame limit")
3582 .with_detail(serde_json::json!({ "size": size, "limit": limit })),
3583 )?;
3584 return Ok((FrameType::Error, flags, body));
3586 }
3587 Ok((ty, flags, body))
3588}
3589
3590fn handle_cancel(frame: Frame, in_flight: &InFlight) -> Result<(), SubcModuleError> {
3591 let guard = lock_in_flight(in_flight)?;
3592 if let Some(cancellation) =
3593 guard.get(&(frame.header.channel, frame.header.epoch, frame.header.corr))
3594 {
3595 cancellation.cancel();
3596 }
3597 Ok(())
3598}
3599
3600fn cancel_handle(in_flight: &InFlight, handle: RouteHandle) -> Result<(), SubcModuleError> {
3601 let cancelled = {
3602 let mut guard = lock_in_flight(in_flight)?;
3603 let keys = guard
3604 .keys()
3605 .copied()
3606 .filter(|(channel, epoch, _)| *channel == handle.channel && *epoch == handle.epoch)
3607 .collect::<Vec<_>>();
3608 keys.into_iter()
3609 .filter_map(|key| guard.remove(&key))
3610 .collect::<Vec<_>>()
3611 };
3612 for cancellation in cancelled {
3613 cancellation.cancel();
3614 }
3615 Ok(())
3616}
3617
3618async fn handle_control_request<H>(
3619 frame: Frame,
3620 egress: &mpsc::Sender<Frame>,
3621 handler: Arc<H>,
3622 dispatcher: RequestDispatcher,
3623 module_handle: ModuleHandle,
3624) -> Result<(), SubcModuleError>
3625where
3626 H: ModuleHandler,
3627{
3628 if control_ops::request(
3629 &frame,
3630 egress,
3631 Arc::clone(&handler),
3632 &dispatcher,
3633 &module_handle,
3634 )
3635 .await?
3636 {
3637 return Ok(());
3638 }
3639 if config::review(
3640 &frame,
3641 egress,
3642 Arc::clone(&handler),
3643 &dispatcher,
3644 &module_handle,
3645 ) {
3646 return Ok(());
3647 }
3648 let request = match serde_json::from_slice::<ModuleControlRequest>(&frame.body) {
3654 Ok(request) => request,
3655 Err(error) => {
3656 let body = serde_json::to_vec(&ErrorBody::new(
3657 "invalid_request",
3658 format!("control request could not be decoded by this module: {error}"),
3659 ))
3660 .map_err(SubcModuleError::Json)?;
3661 let refusal = Frame::build_with_version(
3662 frame.header.ver,
3663 FrameType::Error,
3664 control_flags(),
3665 0,
3666 0,
3667 frame.header.corr,
3668 body,
3669 )
3670 .map_err(SubcModuleError::FrameBuild)?;
3671 send_outbound(egress, refusal).await?;
3672 return Ok(());
3673 }
3674 };
3675 match request {
3676 ModuleControlRequest::RouteBind {
3677 route_channel,
3678 epoch,
3679 target,
3680 identity,
3681 principal,
3682 consumer_capabilities,
3683 role_versions,
3684 admission_facts,
3685 scope,
3686 call_mode,
3687 } => {
3688 if let Some(stale) = module_handle.installed_route(route_channel)? {
3695 if epoch <= stale.epoch {
3696 let body = serde_json::to_vec(&ErrorBody::new(
3697 "route_rejected",
3698 format!(
3699 "route.bind epoch {epoch} does not supersede installed epoch {} on channel {route_channel}",
3700 stale.epoch
3701 ),
3702 ))
3703 .map_err(SubcModuleError::Json)?;
3704 let reject = Frame::build_with_version(
3705 frame.header.ver,
3706 FrameType::Error,
3707 control_flags(),
3708 0,
3709 0,
3710 frame.header.corr,
3711 body,
3712 )
3713 .map_err(SubcModuleError::FrameBuild)?;
3714 send_outbound(egress, reject).await?;
3715 return Ok(());
3716 }
3717 if module_handle.remove_route(stale)? {
3718 cancel_handle(&dispatcher.in_flight, stale)?;
3719 handler.on_route_gone(&stale).await;
3720 }
3721 }
3722 let handle = module_handle.route_handle(route_channel, epoch);
3723 let peer =
3727 match serde_json::from_slice::<subc_protocol::peer::RouteBindPeer>(&frame.body) {
3728 Ok(peer) => peer,
3729 Err(error) => {
3730 tracing::warn!(
3731 channel = route_channel,
3732 epoch,
3733 %error,
3734 "route.bind peer keys did not decode; treating them as absent"
3735 );
3736 subc_protocol::peer::RouteBindPeer::new()
3737 }
3738 };
3739 let req = RouteBindRequest {
3740 handle,
3741 target,
3742 identity,
3743 principal,
3744 consumer_capabilities,
3745 role_versions,
3746 admission_facts,
3747 call_mode: call_mode.unwrap_or_else(|| derive_call_mode(scope.as_ref())),
3748 scope,
3749 peer,
3750 };
3751 let decision = handler.on_bind(&req).await;
3752 match decision.kind {
3753 BindDecisionKind::Accept => {
3754 let response = match serde_json::to_vec(&ModuleControlResponse::RouteBindAck {})
3755 .map_err(SubcModuleError::Json)
3756 .and_then(|body| {
3757 Frame::build_with_version(
3758 frame.header.ver,
3759 FrameType::Response,
3760 control_flags(),
3761 0,
3762 0,
3763 frame.header.corr,
3764 body,
3765 )
3766 .map_err(SubcModuleError::FrameBuild)
3767 }) {
3768 Ok(response) => response,
3769 Err(err) => {
3770 handler.on_route_gone(&handle).await;
3771 return Err(err);
3772 }
3773 };
3774 if let Err(err) = send_outbound(egress, response).await {
3775 handler.on_route_gone(&handle).await;
3776 return Err(err);
3777 }
3778 if let Err(err) = module_handle.install_route(handle) {
3779 handler.on_route_gone(&handle).await;
3780 return Err(err);
3781 }
3782 handler.on_bound(&handle).await;
3783 }
3784 BindDecisionKind::Reject {
3785 code,
3786 message,
3787 detail,
3788 } => {
3789 let body = match detail {
3790 Some(detail) => ErrorBody::new(code, message).with_detail(detail),
3791 None => ErrorBody::new(code, message),
3792 };
3793 let result = serde_json::to_vec(&body)
3794 .map_err(SubcModuleError::Json)
3795 .and_then(|body| {
3796 Frame::build_with_version(
3797 frame.header.ver,
3798 FrameType::Error,
3799 control_flags(),
3800 0,
3801 0,
3802 frame.header.corr,
3803 body,
3804 )
3805 .map_err(SubcModuleError::FrameBuild)
3806 });
3807 let result = match result {
3808 Ok(response) => send_outbound(egress, response).await,
3809 Err(err) => Err(err),
3810 };
3811 handler.on_route_gone(&handle).await;
3812 result?;
3813 }
3814 }
3815 }
3816 ModuleControlRequest::HealthCheck {} => {
3817 spawn_health_request(frame, egress.clone(), handler, dispatcher)?;
3818 }
3819 }
3820 Ok(())
3821}
3822
3823async fn send_hello(
3824 egress: &mpsc::Sender<Frame>,
3825 mut manifest: ModuleManifest,
3826 config_gate: bool,
3827 extra_control_ops: &[String],
3828) -> Result<(), SubcModuleError> {
3829 let launch_nonce = if registration::supervised() {
3830 None
3831 } else {
3832 match retained_launch_nonce() {
3833 Some(retained) => Some(retained),
3834 None => launch_nonce()
3835 .map_err(SubcModuleError::LaunchNonce)?
3836 .map(|nonce| nonce.value().to_string()),
3837 }
3838 };
3839 stamp_launch_nonce_source(&mut manifest, launch_nonce.as_deref());
3840 let mut control_ops = vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()];
3841 if config_gate {
3842 control_ops.push(config::CONFIG_REVIEW_OP.to_owned());
3843 }
3844 control_ops.extend_from_slice(extra_control_ops);
3845 let body = serde_json::to_vec(&ModuleHelloBody {
3846 manifest,
3847 protocol_ver: PROTOCOL_VERSION,
3848 control_ops: Some(control_ops),
3849 launch_nonce,
3850 })
3851 .map_err(SubcModuleError::Json)?;
3852 let frame = Frame::build(FrameType::Hello, control_flags(), 0, 0, HELLO_CORR, body)
3853 .map_err(SubcModuleError::FrameBuild)?;
3854 send_outbound(egress, frame).await
3855}
3856
3857async fn expect_hello_ack<R>(reader: &mut R) -> Result<ModuleHelloAckBody, SubcModuleError>
3858where
3859 R: AsyncRead + Unpin,
3860{
3861 let Some(frame) = read_frame(reader).await.map_err(SubcModuleError::FrameIo)? else {
3862 return Err(SubcModuleError::ConnectionClosedBeforeHelloAck);
3863 };
3864 match frame.header.ty {
3865 FrameType::HelloAck => serde_json::from_slice(&frame.body).map_err(SubcModuleError::Json),
3866 FrameType::Error => {
3867 let body =
3868 serde_json::from_slice::<ErrorBody>(&frame.body).map_err(SubcModuleError::Json)?;
3869 Err(SubcModuleError::HelloRejected { body })
3870 }
3871 ty => Err(SubcModuleError::UnexpectedHelloAck { ty }),
3872 }
3873}
3874
3875pub trait ModuleStream: AsyncRead + AsyncWrite + Unpin + Send {}
3877impl<T: AsyncRead + AsyncWrite + Unpin + Send> ModuleStream for T {}
3878
3879async fn connect_to_subc(
3880 connection_file_path: &Path,
3881 registration_channel: bool,
3882) -> Result<Box<dyn ModuleStream>, SubcModuleError> {
3883 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
3884 SubcModuleError::ConnectionFile {
3885 path: connection_file_path.to_path_buf(),
3886 source,
3887 }
3888 })?;
3889 if let Some(mut stream) = if registration_channel {
3890 registration::take_stream()?
3891 } else {
3892 None
3893 } {
3894 authenticate_client(&mut stream, &conn, AUTH_DEADLINE)
3895 .await
3896 .map_err(|source| SubcModuleError::Auth {
3897 path: connection_file_path.to_path_buf(),
3898 endpoint: "registration channel".into(),
3899 source,
3900 })?;
3901 return Ok(stream);
3902 }
3903 let endpoint = conn
3904 .endpoints
3905 .first()
3906 .ok_or_else(|| SubcModuleError::NoEndpoint {
3907 path: connection_file_path.to_path_buf(),
3908 })?;
3909 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
3910 let mut stream = connect_endpoint(&endpoint.host, endpoint.port)
3911 .await
3912 .map_err(|source| SubcModuleError::Connect {
3913 path: connection_file_path.to_path_buf(),
3914 endpoint: endpoint_label.clone(),
3915 source,
3916 })?;
3917 let _ = stream.set_nodelay(true);
3927 authenticate_client(&mut stream, &conn, AUTH_DEADLINE)
3928 .await
3929 .map_err(|source| SubcModuleError::Auth {
3930 path: connection_file_path.to_path_buf(),
3931 endpoint: endpoint_label,
3932 source,
3933 })?;
3934 Ok(Box::new(stream))
3935}
3936
3937async fn connect_endpoint(host: &str, port: u16) -> io::Result<TcpStream> {
3938 let host = host
3939 .strip_prefix('[')
3940 .and_then(|h| h.strip_suffix(']'))
3941 .unwrap_or(host);
3942 TcpStream::connect((host, port)).await
3943}
3944
3945async fn send_goodbye(tx: &mpsc::Sender<Frame>) -> Result<(), SubcModuleError> {
3946 let frame = Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 0, Vec::new())
3947 .map_err(SubcModuleError::FrameBuild)?;
3948 send_outbound(tx, frame).await
3949}
3950
3951fn is_disconnect(error: &io::Error) -> bool {
3952 matches!(
3953 error.kind(),
3954 io::ErrorKind::ConnectionReset
3955 | io::ErrorKind::ConnectionAborted
3956 | io::ErrorKind::BrokenPipe
3957 )
3958}
3959
3960fn writer_ended(
3961 result: Result<Result<(), WriterFailure>, tokio::task::JoinError>,
3962) -> SubcModuleError {
3963 match result {
3964 Ok(Ok(())) => SubcModuleError::WriterClosed,
3965 Ok(Err(error)) => SubcModuleError::FrameIo(error.error),
3966 Err(error) => SubcModuleError::WriterTask(error),
3967 }
3968}
3969
3970fn writer_connection_end(
3971 result: Result<Result<(), WriterFailure>, tokio::task::JoinError>,
3972) -> Result<ConnectionEnd, SubcModuleError> {
3973 match writer_ended(result) {
3974 SubcModuleError::FrameIo(FrameIoError::Io(error)) if is_disconnect(&error) => {
3975 Ok(ConnectionEnd::Reset)
3976 }
3977 error => Err(error),
3978 }
3979}
3980
3981async fn finish_writer(
3982 mut writer: tokio::task::JoinHandle<Result<(), WriterFailure>>,
3983 deadline: Instant,
3984) -> (Result<(), SubcModuleError>, usize) {
3985 match timeout_at(deadline, &mut writer).await {
3986 Ok(Ok(Ok(()))) => (Ok(()), 0),
3987 Ok(Ok(Err(failure))) => (
3988 Err(SubcModuleError::FrameIo(failure.error)),
3989 failure.undelivered_replies,
3990 ),
3991 Ok(Err(error)) => (Err(SubcModuleError::WriterTask(error)), 0),
3992 Err(_) => {
3993 writer.abort();
3994 let _ = writer.await;
3995 (Ok(()), 0)
3996 }
3997 }
3998}
3999
4000struct WriterFailure {
4001 error: FrameIoError,
4002 undelivered_replies: usize,
4003}
4004
4005async fn finish_writer_and_drain<R>(
4010 writer: tokio::task::JoinHandle<Result<(), WriterFailure>>,
4011 reader: &mut R,
4012 deadline: Instant,
4013) -> (Result<(), SubcModuleError>, usize)
4014where
4015 R: AsyncRead + Unpin,
4016{
4017 let flushed = finish_writer(writer, deadline).await;
4018 if flushed.0.is_ok() {
4019 drain_until_peer_closes(reader, deadline).await;
4020 }
4021 flushed
4022}
4023
4024async fn drain_until_peer_closes<R>(reader: &mut R, deadline: Instant)
4042where
4043 R: AsyncRead + Unpin,
4044{
4045 if Instant::now() >= deadline {
4046 return;
4047 }
4048 let mut discard = [0u8; 4096];
4049 let _ = timeout_at(deadline, async {
4050 while let Ok(read) = reader.read(&mut discard).await {
4051 if read == 0 {
4052 break;
4053 }
4054 }
4055 })
4056 .await;
4057}
4058
4059fn is_reply(frame: &Frame) -> bool {
4060 matches!(
4061 frame.header.ty,
4062 FrameType::Response | FrameType::Error | FrameType::StreamEnd
4063 )
4064}
4065
4066async fn drain_writer<W>(write_half: W, mut rx: mpsc::Receiver<Frame>) -> Result<(), WriterFailure>
4067where
4068 W: AsyncWrite + Unpin,
4069{
4070 let mut writer = BufWriter::new(write_half);
4071 let mut pending_replies = 0;
4072 let result = async {
4073 while let Some(frame) = rx.recv().await {
4074 pending_replies += usize::from(is_reply(&frame));
4075 write_frame(&mut writer, &frame).await?;
4076 while let Ok(frame) = rx.try_recv() {
4077 pending_replies += usize::from(is_reply(&frame));
4078 write_frame(&mut writer, &frame).await?;
4079 }
4080 writer.flush().await.map_err(FrameIoError::Io)?;
4081 pending_replies = 0;
4082 }
4083 writer.flush().await.map_err(FrameIoError::Io)?;
4084 let _ = writer.shutdown().await;
4093 Ok(())
4094 }
4095 .await;
4096 result.map_err(|error| {
4097 rx.close();
4098 while let Ok(frame) = rx.try_recv() {
4099 pending_replies += usize::from(is_reply(&frame));
4100 }
4101 WriterFailure {
4102 error,
4103 undelivered_replies: pending_replies,
4104 }
4105 })
4106}
4107
4108async fn send_outbound(egress: &mpsc::Sender<Frame>, frame: Frame) -> Result<(), SubcModuleError> {
4109 egress
4110 .send(frame)
4111 .await
4112 .map_err(|_| SubcModuleError::WriterClosed)
4113}
4114
4115fn parse_subc_arg(args: impl IntoIterator<Item = OsString>) -> Result<PathBuf, SubcModuleError> {
4116 let mut args = args.into_iter();
4117 while let Some(arg) = args.next() {
4118 if arg == "--subc" {
4119 let value = args.next().ok_or(SubcModuleError::MissingSubcValue)?;
4120 return Ok(PathBuf::from(value));
4121 }
4122 if let Some(raw) = arg.to_str().and_then(|arg| arg.strip_prefix("--subc=")) {
4123 if raw.is_empty() {
4124 return Err(SubcModuleError::MissingSubcValue);
4125 }
4126 return Ok(PathBuf::from(raw));
4127 }
4128 }
4129 Err(SubcModuleError::MissingSubcArg)
4130}
4131
4132fn module_id_from_env() -> Result<Option<String>, SubcModuleError> {
4133 match env::var(SUBC_MODULE_ID_ENV) {
4134 Ok(value) if !value.trim().is_empty() => Ok(Some(value)),
4135 Ok(_) => Err(SubcModuleError::EmptyModuleIdEnv),
4136 Err(env::VarError::NotPresent) => Ok(None),
4137 Err(env::VarError::NotUnicode(value)) => {
4138 Err(SubcModuleError::NonUnicodeModuleIdEnv { value })
4139 }
4140 }
4141}
4142
4143fn lock_in_flight(
4144 in_flight: &InFlight,
4145) -> Result<std::sync::MutexGuard<'_, HashMap<RequestKey, CancellationToken>>, SubcModuleError> {
4146 in_flight
4147 .lock()
4148 .map_err(|_| SubcModuleError::InFlightPoisoned)
4149}
4150
4151fn control_flags() -> Flags {
4152 Flags::new(false, Priority::Passive, false)
4153}
4154
4155fn data_flags() -> Flags {
4156 Flags::new(false, Priority::Interactive, false)
4157}
4158
4159#[derive(Debug)]
4160#[non_exhaustive]
4161pub enum SubcModuleError {
4162 InvalidControlOp(String),
4164 DuplicateControlCorrelation(u64),
4166 RegistrationChannel(io::Error),
4167 MissingSubcArg,
4168 MissingSubcValue,
4169 EmptyModuleIdEnv,
4170 NonUnicodeModuleIdEnv {
4171 value: OsString,
4172 },
4173 ConnectionFile {
4174 path: PathBuf,
4175 source: ConnectionFileError,
4176 },
4177 NoEndpoint {
4178 path: PathBuf,
4179 },
4180 Connect {
4181 path: PathBuf,
4182 endpoint: String,
4183 source: io::Error,
4184 },
4185 Auth {
4186 path: PathBuf,
4187 endpoint: String,
4188 source: AuthError,
4189 },
4190 FrameIo(FrameIoError),
4191 FrameBuild(FrameBuildError),
4192 Json(serde_json::Error),
4193 WriterClosed,
4194 HandlerPanic,
4196 StaleRouteHandle(RouteHandle),
4197 ConnectionTokenExhausted,
4198 WriterTask(tokio::task::JoinError),
4199 InFlightPoisoned,
4200 ConnectionClosedBeforeHelloAck,
4201 UnexpectedHelloAck {
4202 ty: FrameType,
4203 },
4204 HelloRejected {
4205 body: ErrorBody,
4206 },
4207 LaunchNonce(launch_nonce::LaunchNonceError),
4210}
4211
4212impl fmt::Display for SubcModuleError {
4213 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4214 match self {
4215 Self::InvalidControlOp(op) => {
4216 write!(f, "invalid or reserved control operation: {op:?}")
4217 }
4218 Self::DuplicateControlCorrelation(corr) => {
4219 write!(f, "duplicate in-flight control correlation: {corr}")
4220 }
4221 Self::RegistrationChannel(error) => write!(f, "registration channel: {error}"),
4222 Self::MissingSubcArg => write!(f, "missing required --subc <connection-file> argument"),
4223 Self::MissingSubcValue => write!(f, "--subc requires a connection-file path value"),
4224 Self::EmptyModuleIdEnv => write!(f, "{SUBC_MODULE_ID_ENV} must not be empty when set"),
4225 Self::NonUnicodeModuleIdEnv { value } => write!(
4226 f,
4227 "{SUBC_MODULE_ID_ENV} must be valid UTF-8, got '{}'",
4228 value.to_string_lossy()
4229 ),
4230 Self::ConnectionFile { path, source } => write!(
4231 f,
4232 "failed to read subc connection file '{}': {source}",
4233 path.display()
4234 ),
4235 Self::NoEndpoint { path } => write!(
4236 f,
4237 "subc connection file '{}' has no endpoints",
4238 path.display()
4239 ),
4240 Self::Connect {
4241 path,
4242 endpoint,
4243 source,
4244 } => write!(
4245 f,
4246 "failed to connect to subc endpoint {endpoint} from '{}': {source}",
4247 path.display()
4248 ),
4249 Self::Auth {
4250 path,
4251 endpoint,
4252 source,
4253 } => write!(
4254 f,
4255 "failed to authenticate to subc endpoint {endpoint} from '{}': {source}",
4256 path.display()
4257 ),
4258 Self::FrameIo(err) => fmt::Display::fmt(err, f),
4259 Self::FrameBuild(err) => write!(f, "frame build error: {err}"),
4260 Self::Json(err) => write!(f, "JSON error: {err}"),
4261 Self::WriterClosed => write!(f, "module writer task closed"),
4262 Self::HandlerPanic => write!(f, "module handler panicked"),
4263 Self::StaleRouteHandle(handle) => write!(f, "stale route handle: {handle:?}"),
4264 Self::ConnectionTokenExhausted => write!(f, "module connection token exhausted"),
4265 Self::WriterTask(err) => write!(f, "module writer task failed: {err}"),
4266 Self::InFlightPoisoned => write!(f, "in-flight registry lock poisoned"),
4267 Self::ConnectionClosedBeforeHelloAck => write!(f, "connection closed before HELLO_ACK"),
4268 Self::UnexpectedHelloAck { ty } => write!(f, "expected HELLO_ACK, got {ty:?}"),
4269 Self::HelloRejected { body } => write!(
4270 f,
4271 "HELLO rejected by subc: {} ({})",
4272 body.code, body.message
4273 ),
4274 Self::LaunchNonce(err) => write!(f, "launch nonce unavailable: {err}"),
4275 }
4276 }
4277}
4278
4279impl Error for SubcModuleError {
4280 fn source(&self) -> Option<&(dyn Error + 'static)> {
4281 match self {
4282 Self::RegistrationChannel(error) => Some(error),
4283 Self::ConnectionFile { source, .. } => Some(source),
4284 Self::Connect { source, .. } => Some(source),
4285 Self::Auth { source, .. } => Some(source),
4286 Self::FrameIo(err) => Some(err),
4287 Self::FrameBuild(err) => Some(err),
4288 Self::Json(err) => Some(err),
4289 Self::LaunchNonce(err) => Some(err),
4290 Self::WriterTask(err) => Some(err),
4291 Self::MissingSubcArg
4292 | Self::InvalidControlOp(_)
4293 | Self::DuplicateControlCorrelation(_)
4294 | Self::MissingSubcValue
4295 | Self::EmptyModuleIdEnv
4296 | Self::NonUnicodeModuleIdEnv { .. }
4297 | Self::NoEndpoint { .. }
4298 | Self::WriterClosed
4299 | Self::HandlerPanic
4300 | Self::StaleRouteHandle(_)
4301 | Self::ConnectionTokenExhausted
4302 | Self::InFlightPoisoned
4303 | Self::ConnectionClosedBeforeHelloAck
4304 | Self::UnexpectedHelloAck { .. }
4305 | Self::HelloRejected { .. } => None,
4306 }
4307 }
4308}
4309
4310impl From<serde_json::Error> for SubcModuleError {
4311 fn from(err: serde_json::Error) -> Self {
4312 Self::Json(err)
4313 }
4314}
4315
4316#[cfg(test)]
4317mod call_mode_tests;
4318
4319#[cfg(test)]
4320mod tests {
4321 use std::sync::atomic::{AtomicUsize, Ordering};
4322
4323 use serde_json::json;
4324
4325 use subc_protocol::manifest::{Concurrency, ExecutionMode, IdentityScope, Tool};
4326 use tokio::sync::Semaphore;
4327 use tokio::time::{advance, timeout};
4328
4329 use super::*;
4330
4331 struct EchoHandler;
4332
4333 #[async_trait]
4334 impl ModuleHandler for EchoHandler {
4335 async fn handle(&self, _ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome {
4336 HandlerOutcome::Response(body)
4337 }
4338 }
4339
4340 struct BlockingHandler {
4341 entered: Arc<AtomicUsize>,
4342 release: Semaphore,
4343 report: HealthReport,
4344 }
4345
4346 #[async_trait]
4347 impl ModuleHandler for BlockingHandler {
4348 async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
4349 self.entered.fetch_add(1, Ordering::SeqCst);
4350 self.release.acquire().await.unwrap().forget();
4351 HandlerOutcome::Streamed
4352 }
4353
4354 async fn health(&self) -> HealthReport {
4355 self.report.clone()
4356 }
4357 }
4358
4359 fn health_request(corr: u64) -> Frame {
4360 Frame::build(
4361 FrameType::Request,
4362 control_flags(),
4363 0,
4364 0,
4365 corr,
4366 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).unwrap(),
4367 )
4368 .unwrap()
4369 }
4370
4371 fn data_request(channel: u16, corr: u64) -> Frame {
4372 Frame::build(
4373 FrameType::Request,
4374 data_flags(),
4375 channel,
4376 1,
4377 corr,
4378 b"opaque".to_vec(),
4379 )
4380 .unwrap()
4381 }
4382
4383 fn catalog_update_response(corr: u64) -> Frame {
4384 Frame::build(
4385 FrameType::Response,
4386 control_flags(),
4387 0,
4388 0,
4389 corr,
4390 serde_json::to_vec(&ModuleControlResponseToModule::CatalogUpdate {}).unwrap(),
4391 )
4392 .unwrap()
4393 }
4394
4395 fn test_module_handle(subc_ops: &[&str]) -> (ModuleHandle, mpsc::Receiver<Frame>) {
4396 let (tx, rx) = mpsc::channel(4);
4397 let ack = ModuleHelloAckBody {
4398 negotiated_ver: PROTOCOL_VERSION,
4399 subc_ops: subc_ops.iter().map(|op| (*op).to_string()).collect(),
4400 subc_capabilities: Vec::new(),
4401 storage: None,
4402 machine_id: None,
4403 };
4404 (
4405 ModuleHandle::new(
4406 &ack,
4407 tx,
4408 1,
4409 CancellationToken::new(),
4410 tokio::runtime::Handle::current(),
4411 ),
4412 rx,
4413 )
4414 }
4415
4416 #[test]
4417 fn module_handle_exposes_the_acked_machine_id_and_none_otherwise() {
4418 let runtime = tokio::runtime::Runtime::new().unwrap();
4419 let handle_for = |machine_id: Option<&str>| {
4420 let (tx, _rx) = mpsc::channel(1);
4421 let ack = ModuleHelloAckBody {
4422 negotiated_ver: PROTOCOL_VERSION,
4423 subc_ops: Vec::new(),
4424 subc_capabilities: Vec::new(),
4425 storage: None,
4426 machine_id: machine_id.map(str::to_owned),
4427 };
4428 ModuleHandle::new(
4429 &ack,
4430 tx,
4431 1,
4432 CancellationToken::new(),
4433 runtime.handle().clone(),
4434 )
4435 };
4436 assert_eq!(
4437 handle_for(Some("0123456789abcdef0123456789abcdef"))
4438 .machine_id()
4439 .map(MachineId::as_str),
4440 Some("0123456789abcdef0123456789abcdef")
4441 );
4442 assert_eq!(handle_for(None).machine_id(), None);
4444 assert_eq!(handle_for(Some("not-a-machine-id")).machine_id(), None);
4445 }
4446
4447 fn test_provider_role(tool_names: &[&str]) -> ProviderRole {
4448 ProviderRole::ToolProvider {
4449 tools: tool_names
4450 .iter()
4451 .map(|name| Tool {
4452 name: (*name).to_string(),
4453 description: None,
4454 execution_mode: ExecutionMode::Pure,
4455 schema: json!({"type": "object"}),
4456 })
4457 .collect(),
4458 identity_scope: vec![IdentityScope::Project],
4459 concurrency: Concurrency::ModuleManaged,
4460 emits_push: false,
4461 sub_supervises: false,
4462 }
4463 }
4464
4465 #[tokio::test]
4466 async fn catalog_update_fails_fast_when_hello_ack_does_not_advertise_support() {
4467 let (handle, mut rx) = test_module_handle(&[]);
4468
4469 let error = handle
4470 .catalog_update(vec![test_provider_role(&["a"])])
4471 .await
4472 .unwrap_err();
4473 assert!(matches!(error, CatalogUpdateError::NotSupported));
4474 assert!(matches!(
4475 handle.set_ready(true).await,
4476 Err(CatalogUpdateError::NotSupported)
4477 ));
4478 assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
4479 }
4480
4481 #[tokio::test]
4482 async fn catalog_update_demuxes_multiple_in_flight_requests() {
4483 let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
4484 let first_handle = handle.clone();
4485 let second_handle = handle.clone();
4486 let first = tokio::spawn(async move {
4487 first_handle
4488 .catalog_update(vec![test_provider_role(&["a"])])
4489 .await
4490 });
4491 let second = tokio::spawn(async move {
4492 second_handle
4493 .catalog_update(vec![test_provider_role(&["b"])])
4494 .await
4495 });
4496
4497 let first_frame = timeout(Duration::from_secs(1), rx.recv())
4498 .await
4499 .unwrap()
4500 .unwrap();
4501 let second_frame = timeout(Duration::from_secs(1), rx.recv())
4502 .await
4503 .unwrap()
4504 .unwrap();
4505 assert_eq!(first_frame.header.ty, FrameType::Request);
4506 assert_eq!(first_frame.header.channel, 0);
4507 assert_eq!(second_frame.header.ty, FrameType::Request);
4508 assert_eq!(second_frame.header.channel, 0);
4509 assert_ne!(first_frame.header.corr, second_frame.header.corr);
4510
4511 let (earlier, later) =
4512 if json_body(&first_frame.body)["provides"] == json!([test_provider_role(&["a"])]) {
4513 (first, second)
4514 } else {
4515 (second, first)
4516 };
4517 assert!(handle.handle_control_reply(catalog_update_response(second_frame.header.corr)));
4518 assert!(later.await.unwrap().is_ok());
4519 assert!(handle.handle_control_reply(catalog_update_response(first_frame.header.corr)));
4520 assert!(earlier.await.unwrap().is_ok());
4521
4522 let expected = json_body(&second_frame.body)["provides"].clone();
4523 let (result, ()) = tokio::join!(handle.set_ready(true), async {
4524 let frame = rx.recv().await.unwrap();
4525 assert_eq!(json_body(&frame.body)["provides"], expected);
4526 assert_eq!(json_body(&frame.body)["ready"], true);
4527 handle.handle_control_reply(catalog_update_response(frame.header.corr));
4528 });
4529 result.unwrap();
4530 }
4531
4532 fn json_body(body: &[u8]) -> serde_json::Value {
4533 serde_json::from_slice(body).unwrap()
4534 }
4535
4536 #[tokio::test]
4537 async fn role_declarations_update_requires_its_own_advertised_op() {
4538 for ops in [vec![], vec![MODULE_TO_SUBC_OP_CATALOG_UPDATE]] {
4539 let (handle, mut rx) = test_module_handle(&ops);
4540 assert!(matches!(
4541 handle.update_role_declarations(BTreeMap::new()).await,
4542 Err(CatalogUpdateError::NotSupported)
4543 ));
4544 assert!(matches!(
4545 rx.try_recv(),
4546 Err(mpsc::error::TryRecvError::Empty)
4547 ));
4548 }
4549 }
4550
4551 #[tokio::test]
4552 async fn role_declarations_update_sends_map_and_preserves_refusal() {
4553 let (handle, mut rx) =
4554 test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS]);
4555 let initial_roles = vec![test_provider_role(&["original"])];
4556 handle.shared.lock_inner().current_provides = Some(initial_roles.clone());
4557 for declarations in [
4558 json!({"media-provider/v1": {"engines": ["image"]}}),
4559 json!({}),
4560 ] {
4561 let map = serde_json::from_value(declarations.clone()).unwrap();
4562 let (result, ()) = tokio::join!(handle.update_role_declarations(map), async {
4563 let frame = rx.recv().await.unwrap();
4564 assert_eq!(frame.header.ty, FrameType::Request);
4565 assert_eq!(frame.header.channel, 0);
4566 assert_eq!(
4567 json_body(&frame.body),
4568 json!({"op": "catalog.update_role_declarations", "role_declarations": declarations})
4569 );
4570 handle.handle_control_reply(control_reply(
4571 FrameType::Response,
4572 frame.header.corr,
4573 serde_json::to_vec(&json!({"op": "catalog.update_role_declarations"})).unwrap(),
4574 ));
4575 });
4576 result.unwrap();
4577 }
4578 let (result, ()) = tokio::join!(handle.update_role_declarations(BTreeMap::new()), async {
4579 let frame = rx.recv().await.unwrap();
4580 handle.handle_control_reply(control_reply(FrameType::Error, frame.header.corr,
4581 serde_json::to_vec(&json!({"code": "invalid_role_declarations", "message": "role_declarations key must use capability-name grammar"})).unwrap()));
4582 });
4583 let Err(CatalogUpdateError::Rejected(body)) = result else {
4584 panic!("expected typed refusal")
4585 };
4586 assert_eq!(body.code, "invalid_role_declarations");
4587 assert_eq!(
4588 body.message,
4589 "role_declarations key must use capability-name grammar"
4590 );
4591 assert_eq!(
4592 handle.shared.lock_inner().current_provides,
4593 Some(initial_roles)
4594 );
4595 }
4596
4597 async fn ack_catalog_update(
4598 handle: &ModuleHandle,
4599 rx: &mut mpsc::Receiver<Frame>,
4600 provides: &[ProviderRole],
4601 ready: Option<bool>,
4602 capabilities: Option<&CapabilityDeclarations>,
4603 ) {
4604 let frame = rx.recv().await.unwrap();
4605 let mut expected = json!({"op": "catalog.update", "provides": provides});
4606 if let Some(ready) = ready {
4607 expected["ready"] = json!(ready);
4608 }
4609 if let Some(capabilities) = capabilities {
4610 expected["capabilities"] = json!(capabilities);
4611 }
4612 assert_eq!(json_body(&frame.body), expected);
4613 let mut wire = format!(
4614 r#"{{"op":"catalog.update","provides":{}"#,
4615 serde_json::to_string(provides).unwrap()
4616 );
4617 if let Some(capabilities) = capabilities {
4618 wire.push_str(&format!(
4619 r#","capabilities":{}"#,
4620 serde_json::to_string(capabilities).unwrap()
4621 ));
4622 }
4623 if let Some(ready) = ready {
4624 wire.push_str(&format!(r#","ready":{ready}"#));
4625 }
4626 wire.push('}');
4627 assert_eq!(frame.body, wire.as_bytes());
4628 assert!(handle.handle_control_reply(catalog_update_response(frame.header.corr)));
4629 }
4630
4631 #[tokio::test]
4632 async fn catalog_update_set_ready_preserves_registered_and_replaced_roles() {
4633 let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
4634 let initial = vec![test_provider_role(&["initial", "second"])];
4635 handle.shared.lock_inner().current_provides = Some(initial.clone());
4636 let (result, ()) = tokio::join!(
4637 handle.set_ready(true),
4638 ack_catalog_update(&handle, &mut rx, &initial, Some(true), None)
4639 );
4640 result.unwrap();
4641
4642 let replaced = vec![test_provider_role(&["replacement"])];
4643 let (result, ()) = tokio::join!(
4644 handle.catalog_update(replaced.clone()),
4645 ack_catalog_update(&handle, &mut rx, &replaced, None, None)
4646 );
4647 result.unwrap();
4648 let (result, ()) = tokio::join!(
4649 handle.set_ready(false),
4650 ack_catalog_update(&handle, &mut rx, &replaced, Some(false), None)
4651 );
4652 result.unwrap();
4653
4654 let attested = vec![test_provider_role(&["attested"])];
4655 let capabilities = CapabilityDeclarations {
4656 provides: vec!["test.capability".into()],
4657 requires: Vec::new(),
4658 must_never_reach: Vec::new(),
4659 };
4660 let (result, ()) = tokio::join!(
4661 handle.catalog_update_with_capabilities(attested.clone(), capabilities.clone()),
4662 ack_catalog_update(&handle, &mut rx, &attested, None, Some(&capabilities))
4663 );
4664 result.unwrap();
4665 let (result, ()) = tokio::join!(
4666 handle.set_ready(true),
4667 ack_catalog_update(&handle, &mut rx, &attested, Some(true), None)
4668 );
4669 result.unwrap();
4670 }
4671
4672 #[tokio::test]
4673 async fn catalog_update_set_ready_refuses_unknown_or_in_flight_roles() {
4674 let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
4675 assert!(matches!(
4676 handle.set_ready(true).await,
4677 Err(CatalogUpdateError::Protocol(_))
4678 ));
4679 assert!(rx.try_recv().is_err());
4680
4681 let updating = handle.clone();
4682 let update = tokio::spawn(async move {
4683 updating
4684 .catalog_update(vec![test_provider_role(&["new"])])
4685 .await
4686 });
4687 let frame = rx.recv().await.unwrap();
4688 assert!(matches!(
4689 handle.set_ready(true).await,
4690 Err(CatalogUpdateError::Protocol(_))
4691 ));
4692 assert!(rx.try_recv().is_err());
4693 update.abort();
4694 assert!(update.await.unwrap_err().is_cancelled());
4695 assert!(!handle.handle_control_reply(catalog_update_response(frame.header.corr)));
4696 assert!(matches!(
4697 handle.set_ready(true).await,
4698 Err(CatalogUpdateError::Protocol(_))
4699 ));
4700
4701 let roles = vec![test_provider_role(&["recovered"])];
4702 let (result, ()) = tokio::join!(
4703 handle.catalog_update(roles.clone()),
4704 ack_catalog_update(&handle, &mut rx, &roles, None, None)
4705 );
4706 result.unwrap();
4707 let (result, ()) = tokio::join!(
4708 handle.set_ready(true),
4709 ack_catalog_update(&handle, &mut rx, &roles, Some(true), None)
4710 );
4711 result.unwrap();
4712 }
4713
4714 #[tokio::test(start_paused = true)]
4715 async fn catalog_update_set_ready_timeout_does_not_replay_stale_roles() {
4716 let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
4717 handle.shared.lock_inner().current_provides = Some(vec![test_provider_role(&["old"])]);
4718 assert!(matches!(
4719 handle
4720 .catalog_update(vec![test_provider_role(&["new"])])
4721 .await,
4722 Err(CatalogUpdateError::Timeout)
4723 ));
4724 let frame = rx.recv().await.unwrap();
4725 assert!(!handle.handle_control_reply(catalog_update_response(frame.header.corr)));
4726 assert!(matches!(
4727 handle.set_ready(true).await,
4728 Err(CatalogUpdateError::Protocol(_))
4729 ));
4730 assert!(rx.try_recv().is_err());
4731 }
4732
4733 #[tokio::test]
4734 async fn catalog_update_set_ready_rejection_retains_last_accepted_roles() {
4735 let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
4736 let initial = vec![test_provider_role(&["kept"])];
4737 handle.shared.lock_inner().current_provides = Some(initial.clone());
4738 let (result, ()) = tokio::join!(
4739 handle.catalog_update(vec![test_provider_role(&["rejected"])]),
4740 async {
4741 let frame = rx.recv().await.unwrap();
4742 handle.handle_control_reply(control_reply(
4743 FrameType::Error,
4744 frame.header.corr,
4745 br#"{"code":"catalog_update_frozen_field","message":"frozen"}"#.to_vec(),
4746 ));
4747 }
4748 );
4749 assert!(matches!(result, Err(CatalogUpdateError::FrozenField(_))));
4750 let (result, ()) = tokio::join!(
4751 handle.set_ready(true),
4752 ack_catalog_update(&handle, &mut rx, &initial, Some(true), None)
4753 );
4754 result.unwrap();
4755 }
4756
4757 fn control_reply(ty: FrameType, corr: u64, body: Vec<u8>) -> Frame {
4758 Frame::build(ty, control_flags(), 0, 0, corr, body).unwrap()
4759 }
4760
4761 async fn authorize_for_test(
4762 handle: &ModuleHandle,
4763 ) -> Result<RouteAuthorizeResponse, ScopeCallError> {
4764 handle
4765 .route_authorize(
4766 Principal::Direct,
4767 RouteAuthorizeScope::new(
4768 Principal::Reserved {
4769 module_id: "owner".into(),
4770 },
4771 "stored-scope",
4772 7,
4773 ),
4774 "target".into(),
4775 RouteAuthorizeExpected::new(
4776 subc_protocol::scope::ScopeKind::Head,
4777 &subc_protocol::scope::ScopeAttributes::default(),
4778 ),
4779 )
4780 .await
4781 }
4782
4783 #[tokio::test]
4784 async fn route_authorize_requires_advertised_operation() {
4785 let (tx, mut rx) = mpsc::channel(4);
4786 let ack = ModuleHelloAckBody {
4787 negotiated_ver: PROTOCOL_VERSION,
4788 subc_ops: vec![],
4789 subc_capabilities: vec![subc_protocol::scope::ROUTE_AUTHORIZE_CAPABILITY.into()],
4791 storage: None,
4792 machine_id: None,
4793 };
4794 let handle = ModuleHandle::new(
4795 &ack,
4796 tx,
4797 1,
4798 CancellationToken::new(),
4799 tokio::runtime::Handle::current(),
4800 );
4801 let result = timeout(Duration::from_millis(100), authorize_for_test(&handle))
4802 .await
4803 .expect("an unsupported operation must return without sending or awaiting a reply");
4804 assert_eq!(
4805 result,
4806 Err(ScopeCallError::NotSupported {
4807 op: subc_protocol::scope::ROUTE_AUTHORIZE_OP,
4808 })
4809 );
4810 assert!(
4811 rx.try_recv().is_err(),
4812 "unsupported calls must send nothing"
4813 );
4814 }
4815
4816 #[tokio::test]
4817 async fn route_authorize_decodes_every_answer_and_permission_refusal() {
4818 use RouteAuthorizeOutcome::*;
4819 let (handle, mut rx) = test_module_handle(&[subc_protocol::scope::ROUTE_AUTHORIZE_OP]);
4820 for (wire, outcome, code) in [
4821 ("authorized", Authorized, None),
4822 ("scope_ended", ScopeEnded, Some("scope_not_live")),
4823 ("indeterminate", Indeterminate, Some("scope_not_synced")),
4824 ("epoch_changed", EpochChanged, Some("scope_ended")),
4825 ("claim_changed", ClaimChanged, None),
4826 (
4827 "proof_required",
4828 ProofRequired,
4829 Some("scope_bind_proof_required"),
4830 ),
4831 ("not_authorized", NotAuthorized, Some("scope_not_carrier")),
4832 ("target_absent", TargetAbsent, None),
4833 ] {
4834 let caller = handle.clone();
4835 let pending = tokio::spawn(async move { authorize_for_test(&caller).await });
4836 let request = rx.recv().await.unwrap();
4837 assert_eq!(request.header.ty, FrameType::Request);
4838 assert_eq!(request.header.channel, 0);
4839 assert_eq!(request.header.ver, PROTOCOL_VERSION);
4840 assert_eq!(
4841 serde_json::from_slice::<serde_json::Value>(&request.body).unwrap(),
4842 serde_json::json!({
4843 "op": "route.authorize", "principal": {"kind": "direct"},
4844 "scope": {"owner": {"kind": "reserved", "module_id": "owner"},
4845 "ref": "stored-scope", "epoch": 7},
4846 "target": "target", "expected": {"kind": "head"}
4847 })
4848 );
4849 let content_version = (outcome == Authorized).then_some("sha256:content");
4850 let mut body = serde_json::json!({
4851 "op": "route.authorize", "outcome": wire, "message": "judgment"
4852 });
4853 if let Some(code) = code {
4854 body["route_open_code"] = code.into();
4855 }
4856 if let Some(version) = content_version {
4857 body["content_version"] = version.into();
4858 }
4859 assert!(handle.handle_control_reply(control_reply(
4860 FrameType::Response,
4861 request.header.corr,
4862 serde_json::to_vec(&body).unwrap(),
4863 )));
4864 let reply = pending.await.unwrap().unwrap();
4865 assert_eq!(reply.outcome, outcome);
4866 assert_eq!(reply.message, "judgment");
4867 assert_eq!(reply.route_open_code.as_deref(), code);
4868 assert_eq!(reply.content_version.as_deref(), content_version);
4869 }
4870
4871 for (code, message) in [
4872 ("route_authorize_not_permitted", "not the configured owner"),
4873 ("not_registered", "connection is not registered"),
4874 ("unexpected", "a future daemon call refusal"),
4875 ] {
4876 let caller = handle.clone();
4877 let pending = tokio::spawn(async move { authorize_for_test(&caller).await });
4878 let request = rx.recv().await.unwrap();
4879 assert!(handle.handle_control_reply(control_reply(
4880 FrameType::Error,
4881 request.header.corr,
4882 serde_json::to_vec(&serde_json::json!({"code": code, "message": message})).unwrap(),
4883 )));
4884 assert_eq!(
4885 pending.await.unwrap(),
4886 Err(ScopeCallError::Refused {
4887 code: code.into(),
4888 message: message.into(),
4889 })
4890 );
4891 }
4892 }
4893
4894 #[tokio::test]
4895 async fn route_authorize_rejects_malformed_and_unexpected_replies() {
4896 let (handle, mut rx) = test_module_handle(&[subc_protocol::scope::ROUTE_AUTHORIZE_OP]);
4897 for (ty, body) in [
4898 (FrameType::Response, r#"{"op":"catalog.update"}"#),
4899 (
4900 FrameType::Response,
4901 r#"{"op":"route.authorize","outcome":"authorized"}"#,
4902 ),
4903 (
4904 FrameType::Response,
4905 r#"{"op":"route.authorize","outcome":"future","message":"new"}"#,
4906 ),
4907 (FrameType::Error, "not json"),
4908 (FrameType::Request, "{}"),
4909 ] {
4910 let caller = handle.clone();
4911 let pending = tokio::spawn(async move { authorize_for_test(&caller).await });
4912 let request = rx.recv().await.unwrap();
4913 assert!(handle.handle_control_reply(control_reply(
4914 ty,
4915 request.header.corr,
4916 body.as_bytes().to_vec(),
4917 )));
4918 assert!(
4919 matches!(pending.await.unwrap(), Err(ScopeCallError::Protocol(_))),
4920 "{body}"
4921 );
4922 }
4923 }
4924
4925 #[tokio::test]
4926 async fn scope_served_by_requires_capability_and_decodes_open_replies() {
4927 let (unsupported, mut rx) = test_module_handle(&[subc_control::SCOPE_SERVED_BY_OP]);
4928 assert_eq!(
4929 unsupported
4930 .scope_served_by("s".into(), 1, "tool".into())
4931 .await,
4932 Err(ScopeCallError::NotSupported {
4933 op: subc_control::SCOPE_SERVED_BY_OP
4934 })
4935 );
4936 assert!(rx.try_recv().is_err());
4937 let (tx, mut rx) = mpsc::channel(4);
4938 let ack = ModuleHelloAckBody {
4939 negotiated_ver: PROTOCOL_VERSION,
4940 subc_ops: vec![],
4941 subc_capabilities: vec![subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.into()],
4942 storage: None,
4943 machine_id: None,
4944 };
4945 let handle = ModuleHandle::new(
4946 &ack,
4947 tx,
4948 1,
4949 CancellationToken::new(),
4950 tokio::runtime::Handle::current(),
4951 );
4952 for answer in ["served", "not_served", "unknown", "future"] {
4953 let caller = handle.clone();
4954 let pending =
4955 tokio::spawn(
4956 async move { caller.scope_served_by("s".into(), 7, "tool".into()).await },
4957 );
4958 let request = rx.recv().await.unwrap();
4959 let decoded: subc_control::ScopeServedByRequest =
4960 serde_json::from_slice(&request.body).unwrap();
4961 assert_eq!(
4962 (
4963 decoded.scope_ref.as_str(),
4964 decoded.scope_epoch,
4965 decoded.module_id.as_str()
4966 ),
4967 ("s", 7, "tool")
4968 );
4969 assert!(handle.handle_control_reply(control_reply(FrameType::Response, request.header.corr,
4970 serde_json::to_vec(&serde_json::json!({"op":"scope.served_by", "answer":answer, "daemon_incarnation":"d"})).unwrap())));
4971 let reply = pending.await.unwrap().unwrap();
4972 assert_eq!(reply.daemon_incarnation, "d");
4973 assert_eq!(serde_json::to_value(reply.answer).unwrap(), answer);
4974 }
4975 let caller = handle.clone();
4976 let pending =
4977 tokio::spawn(async move { caller.scope_served_by("s".into(), 7, "tool".into()).await });
4978 let request = rx.recv().await.unwrap();
4979 assert!(handle.handle_control_reply(control_reply(
4980 FrameType::Error,
4981 request.header.corr,
4982 serde_json::to_vec(&serde_json::json!({"code":"not_registered", "message":"gone"}))
4983 .unwrap()
4984 )));
4985 assert_eq!(
4986 pending.await.unwrap().unwrap_err().code(),
4987 Some("not_registered")
4988 );
4989 }
4990
4991 fn owner() -> Principal {
4992 Principal::Reserved {
4993 module_id: "owner".to_string(),
4994 }
4995 }
4996
4997 mod scope_watch {
4998 use super::*;
4999 include!("module_subscription_tests.rs");
5000 }
5001
5002 #[tokio::test]
5006 async fn scope_ops_fail_fast_when_hello_ack_does_not_advertise_them() {
5007 let (handle, mut rx) = test_module_handle(&[]);
5008 assert_eq!(
5009 handle.scope_sync(1, Vec::new()).await.unwrap_err(),
5010 ScopeCallError::NotSupported { op: SCOPE_SYNC_OP }
5011 );
5012 assert_eq!(
5013 handle
5014 .scope_describe(owner(), "s".to_string())
5015 .await
5016 .unwrap_err(),
5017 ScopeCallError::NotSupported {
5018 op: SCOPE_DESCRIBE_OP
5019 }
5020 );
5021 assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
5022
5023 let (describe_only, mut rx) = test_module_handle(&[SCOPE_DESCRIBE_OP]);
5024 assert_eq!(
5025 describe_only.scope_sync(1, Vec::new()).await.unwrap_err(),
5026 ScopeCallError::NotSupported { op: SCOPE_SYNC_OP }
5027 );
5028 assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
5029
5030 let (sync_only, mut rx) = test_module_handle(&[SCOPE_SYNC_OP]);
5031 assert_eq!(
5032 sync_only
5033 .scope_describe(owner(), "s".to_string())
5034 .await
5035 .unwrap_err(),
5036 ScopeCallError::NotSupported {
5037 op: SCOPE_DESCRIBE_OP
5038 }
5039 );
5040 assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
5041 }
5042
5043 #[tokio::test]
5044 async fn scope_apply_sends_nothing_without_its_own_advertised_op() {
5045 for ops in [&[][..], &[SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP][..]] {
5046 let (handle, mut rx) = test_module_handle(ops);
5047 assert_eq!(
5048 handle.scope_apply(2, Vec::new(), Vec::new()).await,
5049 Err(ScopeCallError::NotSupported { op: SCOPE_APPLY_OP })
5050 );
5051 assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
5052 }
5053 }
5054
5055 #[tokio::test]
5056 async fn scope_apply_sends_the_delta_and_preserves_per_entry_outcomes() {
5057 let (handle, mut rx) = test_module_handle(&[SCOPE_APPLY_OP]);
5058 let apply = tokio::spawn({
5059 let handle = handle.clone();
5060 async move {
5061 handle
5062 .scope_apply(
5063 2,
5064 vec![ScopeRecord::new(
5065 "run",
5066 1,
5067 subc_protocol::scope::ScopeKind::Ephemeral,
5068 )
5069 .with_expires_at_ms(Some(12345))
5070 .with_workdir(Some("/session/folder".into()))
5071 .with_project_id(Some("project:1".into()))],
5072 vec![ScopeEnd::new("old", 3), ScopeEnd::new("missing", 1)],
5073 )
5074 .await
5075 }
5076 });
5077 let request = timeout(Duration::from_secs(1), rx.recv())
5078 .await
5079 .unwrap()
5080 .unwrap();
5081 assert_eq!(request.header.ty, FrameType::Request);
5082 assert_eq!(request.header.channel, 0);
5083 assert_eq!(
5084 serde_json::from_slice::<serde_json::Value>(&request.body).unwrap(),
5085 serde_json::json!({
5086 "op": "scope.apply", "generation": 2,
5087 "upsert": [{"ref": "run", "scope_epoch": 1, "kind": "ephemeral", "expires_at_ms": 12345,
5088 "workdir": "/session/folder", "project_id": "project:1"}],
5089 "end": [{"ref": "old", "scope_epoch": 3}, {"ref": "missing", "scope_epoch": 1}]
5090 })
5091 );
5092 let body = serde_json::to_vec(&serde_json::json!({
5093 "op": "scope.apply", "generation": 2,
5094 "results": [{"ref": "run", "scope_epoch": 1, "outcome": "refused", "code": "scope_expired"}],
5095 "end_results": [
5096 {"ref": "old", "scope_epoch": 3, "outcome": "ended"},
5097 {"ref": "missing", "scope_epoch": 1, "outcome": "not_live"}
5098 ],
5099 "ended": [{"ref": "old", "scope_epoch": 3}]
5100 }))
5101 .unwrap();
5102 assert!(handle.handle_control_reply(control_reply(
5103 FrameType::Response,
5104 request.header.corr,
5105 body
5106 )));
5107 let reply = apply.await.unwrap().unwrap();
5108 assert_eq!(reply.generation, 2);
5109 assert_eq!(reply.results.len(), 1);
5110 assert_eq!(reply.results[0].scope_ref, "run");
5111 assert_eq!(
5112 reply.results[0].outcome,
5113 subc_protocol::scope::ScopeRecordOutcome::Refused
5114 );
5115 assert_eq!(reply.results[0].code.as_deref(), Some("scope_expired"));
5116 assert_eq!(reply.end_results.len(), 2);
5117 assert_eq!(reply.end_results[0].scope_ref, "old");
5118 assert_eq!(reply.end_results[0].scope_epoch, 3);
5119 assert_eq!(
5120 reply.end_results[0].outcome,
5121 subc_protocol::scope::ScopeEndOutcome::Ended
5122 );
5123 assert_eq!(reply.end_results[1].scope_ref, "missing");
5124 assert_eq!(
5125 reply.end_results[1].outcome,
5126 subc_protocol::scope::ScopeEndOutcome::NotLive
5127 );
5128 assert_eq!(reply.ended.len(), 1);
5129 assert_eq!(reply.ended[0].scope_ref, "old");
5130 assert_eq!(reply.ended[0].scope_epoch, 3);
5131 }
5132
5133 #[tokio::test]
5134 async fn scope_apply_rejects_a_reply_for_another_op() {
5135 let (handle, mut rx) = test_module_handle(&[SCOPE_APPLY_OP]);
5136 let apply = tokio::spawn({
5137 let handle = handle.clone();
5138 async move { handle.scope_apply(2, Vec::new(), Vec::new()).await }
5139 });
5140 let request = timeout(Duration::from_secs(1), rx.recv())
5141 .await
5142 .unwrap()
5143 .unwrap();
5144 let body = serde_json::to_vec(&ModuleControlResponseToModule::ScopeSync {
5145 generation: 2,
5146 results: Vec::new(),
5147 ended: Vec::new(),
5148 })
5149 .unwrap();
5150 assert!(handle.handle_control_reply(control_reply(
5151 FrameType::Response,
5152 request.header.corr,
5153 body
5154 )));
5155 assert!(matches!(
5156 apply.await.unwrap(),
5157 Err(ScopeCallError::Protocol(_))
5158 ));
5159 }
5160
5161 #[tokio::test]
5162 async fn scope_apply_preserves_whole_call_refusal_codes() {
5163 let (handle, mut rx) = test_module_handle(&[SCOPE_APPLY_OP]);
5164 for code in [
5165 subc_protocol::error_codes::SCOPE_SYNC_REQUIRED,
5166 subc_protocol::error_codes::SCOPE_SYNC_STALE,
5167 subc_protocol::error_codes::SCOPE_SYNC_NOT_AUTHORITY,
5168 "not_registered",
5169 ] {
5170 let apply = tokio::spawn({
5171 let handle = handle.clone();
5172 async move { handle.scope_apply(2, Vec::new(), Vec::new()).await }
5173 });
5174 let request = timeout(Duration::from_secs(1), rx.recv())
5175 .await
5176 .unwrap()
5177 .unwrap();
5178 let body = serde_json::to_vec(&ErrorBody::new(code, "why")).unwrap();
5179 assert!(handle.handle_control_reply(control_reply(
5180 FrameType::Error,
5181 request.header.corr,
5182 body
5183 )));
5184 let error = apply.await.unwrap().unwrap_err();
5185 assert_eq!(
5186 error,
5187 ScopeCallError::Refused {
5188 code: code.to_string(),
5189 message: "why".to_string()
5190 }
5191 );
5192 assert_eq!(error.code(), Some(code));
5193 }
5194 }
5195
5196 #[tokio::test]
5198 async fn a_scope_reply_of_another_op_is_a_protocol_error() {
5199 let (handle, mut rx) = test_module_handle(&[SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP]);
5200 let sync = tokio::spawn({
5201 let handle = handle.clone();
5202 async move { handle.scope_sync(1, Vec::new()).await }
5203 });
5204 let request = timeout(Duration::from_secs(1), rx.recv())
5205 .await
5206 .unwrap()
5207 .unwrap();
5208 let describe_body = serde_json::to_vec(&ModuleControlResponseToModule::ScopeDescribe {
5209 status: ScopeStatus::NotLive,
5210 scope_epoch: None,
5211 daemon_incarnation: "i".to_string(),
5212 owner_synced: false,
5213 owner_configured: false,
5214 scope: None,
5215 })
5216 .unwrap();
5217 assert!(handle.handle_control_reply(control_reply(
5218 FrameType::Response,
5219 request.header.corr,
5220 describe_body
5221 )));
5222 assert!(matches!(
5223 sync.await.unwrap(),
5224 Err(ScopeCallError::Protocol(_))
5225 ));
5226
5227 let describe = tokio::spawn({
5228 let handle = handle.clone();
5229 async move { handle.scope_describe(owner(), "s".to_string()).await }
5230 });
5231 let request = timeout(Duration::from_secs(1), rx.recv())
5232 .await
5233 .unwrap()
5234 .unwrap();
5235 let sync_body = serde_json::to_vec(&ModuleControlResponseToModule::ScopeSync {
5236 generation: 1,
5237 results: Vec::new(),
5238 ended: Vec::new(),
5239 })
5240 .unwrap();
5241 assert!(handle.handle_control_reply(control_reply(
5242 FrameType::Response,
5243 request.header.corr,
5244 sync_body
5245 )));
5246 assert!(matches!(
5247 describe.await.unwrap(),
5248 Err(ScopeCallError::Protocol(_))
5249 ));
5250 }
5251
5252 #[tokio::test]
5256 async fn a_scope_error_frame_keeps_the_daemons_code_and_message() {
5257 let (handle, mut rx) = test_module_handle(&[SCOPE_SYNC_OP]);
5258 for code in [
5259 subc_protocol::error_codes::SCOPE_SYNC_STALE,
5260 subc_protocol::error_codes::SCOPE_SYNC_NOT_AUTHORITY,
5261 "not_registered",
5262 ] {
5263 let sync = tokio::spawn({
5264 let handle = handle.clone();
5265 async move { handle.scope_sync(1, Vec::new()).await }
5266 });
5267 let request = timeout(Duration::from_secs(1), rx.recv())
5268 .await
5269 .unwrap()
5270 .unwrap();
5271 let body = serde_json::to_vec(&ErrorBody::new(code, "why")).unwrap();
5272 assert!(handle.handle_control_reply(control_reply(
5273 FrameType::Error,
5274 request.header.corr,
5275 body
5276 )));
5277 let error = sync.await.unwrap().unwrap_err();
5278 assert_eq!(
5279 error,
5280 ScopeCallError::Refused {
5281 code: code.to_string(),
5282 message: "why".to_string()
5283 }
5284 );
5285 assert_eq!(error.code(), Some(code));
5286 }
5287 }
5288
5289 #[tokio::test]
5293 async fn an_undecodable_control_request_is_refused_and_the_module_keeps_serving() {
5294 let (tx, mut rx) = mpsc::channel(4);
5295 let handler = Arc::new(EchoHandler);
5296 let dispatcher = RequestDispatcher::new();
5297 let (module_handle, _unused_rx) = test_module_handle(&[]);
5298 let mut body: serde_json::Value =
5301 serde_json::from_slice(&route_bind_frame(9, 1, 41).body).unwrap();
5302 body["scope"] = serde_json::json!({
5303 "owner": { "kind": "reserved", "module_id": "prefrontal-core" },
5304 "ref": "s",
5305 "scope_epoch": 1,
5306 "kind": "head",
5307 "attributes": { "a_field_from_a_newer_daemon": "x" },
5308 "owner_authorized": true
5309 });
5310 let body = serde_json::to_vec(&body).unwrap();
5311 let bind = Frame::build(FrameType::Request, control_flags(), 0, 0, 41, body).unwrap();
5312
5313 let kept_serving = handle_frame(
5314 bind,
5315 &tx,
5316 Arc::clone(&handler),
5317 dispatcher.clone(),
5318 module_handle.clone(),
5319 )
5320 .await
5321 .expect("an undecodable control request must not end the module");
5322 assert!(kept_serving);
5323 let refusal = timeout(Duration::from_secs(1), rx.recv())
5324 .await
5325 .unwrap()
5326 .unwrap();
5327 assert_eq!(refusal.header.ty, FrameType::Error);
5328 assert_eq!(refusal.header.corr, 41);
5329 let error: ErrorBody = serde_json::from_slice(&refusal.body).unwrap();
5330 assert_eq!(error.code, "invalid_request");
5331 assert!(
5334 error.message.contains("a_field_from_a_newer_daemon"),
5335 "{}",
5336 error.message
5337 );
5338
5339 assert!(
5340 handle_frame(health_request(42), &tx, handler, dispatcher, module_handle)
5341 .await
5342 .unwrap()
5343 );
5344 let health = timeout(Duration::from_secs(1), rx.recv())
5345 .await
5346 .unwrap()
5347 .unwrap();
5348 assert_eq!(health.header.ty, FrameType::Response);
5349 assert_eq!(health.header.corr, 42);
5350 }
5351
5352 #[tokio::test]
5353 async fn default_health_check_answers_ok() {
5354 let (tx, mut rx) = mpsc::channel(4);
5355 let handler = Arc::new(EchoHandler);
5356 let dispatcher = RequestDispatcher::new();
5357 let (module_handle, _unused_rx) = test_module_handle(&[]);
5358
5359 assert!(
5360 handle_frame(health_request(77), &tx, handler, dispatcher, module_handle)
5361 .await
5362 .unwrap()
5363 );
5364
5365 let response = timeout(Duration::from_secs(1), rx.recv())
5366 .await
5367 .unwrap()
5368 .unwrap();
5369 assert_eq!(response.header.ty, FrameType::Response);
5370 assert_eq!(response.header.channel, 0);
5371 assert_eq!(response.header.corr, 77);
5372
5373 let parsed = serde_json::from_slice::<ModuleControlResponse>(&response.body).unwrap();
5378 let ModuleControlResponse::HealthCheck { status, detail, .. } = parsed else {
5379 panic!("expected a health.check response");
5380 };
5381 assert_eq!(status, HealthStatus::Ok);
5384 assert!(
5388 detail
5389 .as_deref()
5390 .is_some_and(|d| d.contains("no health implementation")),
5391 "the inherited default must identify itself, got {detail:?}"
5392 );
5393 }
5394
5395 struct DispatchHarness {
5396 tx: mpsc::Sender<Frame>,
5397 rx: mpsc::Receiver<Frame>,
5398 handler: Arc<BlockingHandler>,
5399 dispatcher: RequestDispatcher,
5400 module_handle: ModuleHandle,
5401 }
5402
5403 impl DispatchHarness {
5404 fn new(report: HealthReport) -> Self {
5405 let (tx, rx) = mpsc::channel(HANDLER_TASK_CAPACITY + 4);
5406 let (module_handle, _unused_rx) = test_module_handle(&[]);
5407 module_handle
5408 .install_route(RouteHandle::new(7, 1, 1))
5409 .unwrap();
5410 Self {
5411 tx,
5412 rx,
5413 handler: Arc::new(BlockingHandler {
5414 entered: Arc::new(AtomicUsize::new(0)),
5415 release: Semaphore::new(0),
5416 report,
5417 }),
5418 dispatcher: RequestDispatcher::new(),
5419 module_handle,
5420 }
5421 }
5422
5423 async fn request(&self, corr: u64) {
5424 handle_frame(
5425 data_request(7, corr),
5426 &self.tx,
5427 Arc::clone(&self.handler),
5428 self.dispatcher.clone(),
5429 self.module_handle.clone(),
5430 )
5431 .await
5432 .unwrap();
5433 }
5434
5435 async fn wait_for_entered(&self, count: usize) {
5436 timeout(Duration::from_secs(1), async {
5437 while self.handler.entered.load(Ordering::SeqCst) != count {
5438 tokio::task::yield_now().await;
5439 }
5440 })
5441 .await
5442 .expect("data handlers should acquire the available slots");
5443 }
5444
5445 async fn saturate(&self) {
5446 for corr in 1..=HANDLER_TASK_CAPACITY as u64 {
5447 self.request(corr).await;
5448 }
5449 self.wait_for_entered(HANDLER_TASK_CAPACITY).await;
5450 self.request(HANDLER_TASK_CAPACITY as u64 + 1).await;
5451 tokio::task::yield_now().await;
5452 assert_eq!(self.dispatcher.pools.default.permits.available_permits(), 0);
5453 assert_eq!(
5454 self.dispatcher.pools.default.waiting.lock().unwrap().len(),
5455 1
5456 );
5457 assert_eq!(
5458 self.handler.entered.load(Ordering::SeqCst),
5459 HANDLER_TASK_CAPACITY
5460 );
5461 }
5462
5463 async fn health(&mut self, corr: u64) -> HealthReport {
5464 handle_frame(
5465 health_request(corr),
5466 &self.tx,
5467 Arc::clone(&self.handler),
5468 self.dispatcher.clone(),
5469 self.module_handle.clone(),
5470 )
5471 .await
5472 .unwrap();
5473 let frame = timeout(Duration::from_millis(100), self.rx.recv())
5474 .await
5475 .expect("health.check must answer within 100 ms while data slots are busy")
5476 .unwrap();
5477 assert_eq!(frame.header.ty, FrameType::Response);
5478 assert_eq!(frame.header.channel, 0);
5479 assert_eq!(frame.header.corr, corr);
5480 serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5481 .unwrap()
5482 .health_report()
5483 .unwrap()
5484 }
5485
5486 async fn release(&mut self, count: usize) {
5487 self.handler.release.add_permits(count);
5488 for _ in 0..count {
5489 let frame = timeout(Duration::from_secs(1), self.rx.recv())
5490 .await
5491 .unwrap()
5492 .unwrap();
5493 assert_eq!(frame.header.ty, FrameType::StreamEnd);
5494 assert_eq!(frame.header.channel, 7);
5495 }
5496 assert!(self
5497 .dispatcher
5498 .pools
5499 .default
5500 .waiting
5501 .lock()
5502 .unwrap()
5503 .is_empty());
5504 assert!(self.dispatcher.in_flight.lock().unwrap().is_empty());
5505 assert_eq!(
5506 self.dispatcher.pools.default.permits.available_permits(),
5507 HANDLER_TASK_CAPACITY
5508 );
5509 }
5510 }
5511
5512 #[tokio::test(start_paused = true)]
5513 async fn health_check_answers_degraded_under_saturation_and_recovers() {
5514 let own_report = HealthReport {
5515 status: HealthStatus::Ok,
5516 detail: Some("module gauges healthy".to_string()),
5517 metrics: Some(json!({"transforms": 64})),
5518 };
5519 let mut harness = DispatchHarness::new(own_report.clone());
5520 harness.saturate().await;
5521 assert_eq!(harness.health(899).await, own_report);
5523 advance(Duration::from_millis(7300)).await;
5526 let report = harness.health(900).await;
5527 assert_eq!(report.status, HealthStatus::Degraded);
5528 assert_eq!(report.metrics, own_report.metrics);
5529 assert_eq!(
5530 report.detail.as_deref(),
5531 Some("request dispatch saturated: 64/64 in use, oldest waiting 7.3 s; module gauges healthy")
5532 );
5533
5534 harness.release(HANDLER_TASK_CAPACITY + 1).await;
5535 assert_eq!(harness.health(901).await, own_report);
5536 }
5537
5538 #[tokio::test(start_paused = true)]
5539 async fn health_check_preserves_failing_under_saturation() {
5540 let mut harness = DispatchHarness::new(HealthReport {
5541 status: HealthStatus::Failing,
5542 detail: Some("module dispatch heartbeat stale".to_string()),
5543 metrics: Some(json!({"heartbeat_age_s": 12})),
5544 });
5545 harness.saturate().await;
5546 advance(Duration::from_millis(7300)).await;
5547 let report = harness.health(900).await;
5548 assert_eq!(report.status, HealthStatus::Failing);
5549 assert_eq!(report.metrics, Some(json!({"heartbeat_age_s": 12})));
5550 assert_eq!(
5551 report.detail.as_deref(),
5552 Some("request dispatch saturated: 64/64 in use, oldest waiting 7.3 s; module dispatch heartbeat stale")
5553 );
5554 harness.release(HANDLER_TASK_CAPACITY + 1).await;
5555 }
5556
5557 #[tokio::test(start_paused = true)]
5558 async fn health_check_passes_module_report_through_with_free_permits() {
5559 for status in [
5560 HealthStatus::Ok,
5561 HealthStatus::Degraded,
5562 HealthStatus::Failing,
5563 ] {
5564 let own_report = HealthReport {
5565 status,
5566 detail: Some("module's own assessment".to_string()),
5567 metrics: Some(json!({"gauge": 42})),
5568 };
5569 let mut harness = DispatchHarness::new(own_report.clone());
5570 harness.request(1).await;
5571 harness.wait_for_entered(1).await;
5572 advance(Duration::from_millis(7300)).await;
5573 assert_eq!(harness.health(900).await, own_report);
5574 harness.release(1).await;
5575 }
5576 }
5577
5578 struct CorrBlockingHandler {
5579 entered: Arc<Mutex<Vec<u64>>>,
5580 release_first: Arc<Semaphore>,
5581 }
5582
5583 #[async_trait]
5584 impl ModuleHandler for CorrBlockingHandler {
5585 async fn handle(&self, ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
5586 let corr = ctx.corr();
5587 self.entered.lock().unwrap().push(corr);
5588 if corr == 1 {
5589 self.release_first.acquire().await.unwrap().forget();
5590 }
5591 HandlerOutcome::Response(Vec::new())
5592 }
5593 }
5594
5595 #[tokio::test]
5599 async fn cancelled_capacity_queued_data_request_is_answered_without_waiting_for_a_slot() {
5600 let (tx, mut rx) = mpsc::channel(4);
5601 let entered = Arc::new(Mutex::new(Vec::new()));
5602 let release_first = Arc::new(Semaphore::new(0));
5603 let handler = Arc::new(CorrBlockingHandler {
5604 entered: Arc::clone(&entered),
5605 release_first: Arc::clone(&release_first),
5606 });
5607 let dispatcher = RequestDispatcher::with_pools(Arc::new(
5608 DispatchPools::with_default_limit(1, RequestPools::new()),
5609 ));
5610 let (module_handle, _unused_rx) = test_module_handle(&[]);
5611 module_handle
5612 .install_route(RouteHandle::new(7, 1, 1))
5613 .unwrap();
5614
5615 handle_frame(
5616 data_request(7, 1),
5617 &tx,
5618 Arc::clone(&handler),
5619 dispatcher.clone(),
5620 module_handle.clone(),
5621 )
5622 .await
5623 .unwrap();
5624 timeout(Duration::from_secs(1), async {
5625 while entered.lock().unwrap().as_slice() != [1] {
5626 tokio::task::yield_now().await;
5627 }
5628 })
5629 .await
5630 .unwrap();
5631
5632 handle_frame(
5633 data_request(7, 2),
5634 &tx,
5635 Arc::clone(&handler),
5636 dispatcher.clone(),
5637 module_handle.clone(),
5638 )
5639 .await
5640 .unwrap();
5641 timeout(Duration::from_secs(1), async {
5642 while !dispatcher
5643 .in_flight
5644 .lock()
5645 .unwrap()
5646 .contains_key(&(7, 1, 2))
5647 {
5648 tokio::task::yield_now().await;
5649 }
5650 })
5651 .await
5652 .unwrap();
5653 handle_frame(
5654 Frame::build(FrameType::Cancel, data_flags(), 7, 1, 2, Vec::new()).unwrap(),
5655 &tx,
5656 Arc::clone(&handler),
5657 dispatcher.clone(),
5658 module_handle,
5659 )
5660 .await
5661 .unwrap();
5662 assert!(
5663 dispatcher
5664 .in_flight
5665 .lock()
5666 .unwrap()
5667 .get(&(7, 1, 2))
5668 .unwrap()
5669 .is_cancelled(),
5670 "cancel must land while the second request waits for handler capacity"
5671 );
5672
5673 let cancelled = timeout(Duration::from_secs(1), rx.recv())
5676 .await
5677 .expect("a request cancelled while waiting must be answered without waiting for a slot")
5678 .unwrap();
5679 assert_eq!(cancelled.header.ty, FrameType::Error);
5680 assert_eq!(cancelled.header.channel, 7);
5681 assert_eq!(cancelled.header.epoch, 1);
5682 assert_eq!(cancelled.header.corr, 2);
5683 assert_eq!(
5684 serde_json::from_slice::<ErrorBody>(&cancelled.body).unwrap(),
5685 ErrorBody {
5686 code: "cancelled".to_string(),
5687 message: "request cancelled".to_string(),
5688 detail: None,
5689 }
5690 );
5691 timeout(Duration::from_secs(1), async {
5692 while dispatcher
5693 .in_flight
5694 .lock()
5695 .unwrap()
5696 .contains_key(&(7, 1, 2))
5697 {
5698 tokio::task::yield_now().await;
5699 }
5700 })
5701 .await
5702 .unwrap();
5703 assert_eq!(dispatcher.pools.default.permits.available_permits(), 0);
5704 assert!(dispatcher.pools.default.waiting.lock().unwrap().is_empty());
5705 assert_eq!(*entered.lock().unwrap(), vec![1]);
5706
5707 release_first.add_permits(1);
5708 let response = timeout(Duration::from_secs(1), rx.recv())
5709 .await
5710 .unwrap()
5711 .unwrap();
5712 assert_eq!(response.header.ty, FrameType::Response);
5713 assert_eq!(response.header.corr, 1);
5714 timeout(Duration::from_secs(1), async {
5715 while !dispatcher.in_flight.lock().unwrap().is_empty() {
5716 tokio::task::yield_now().await;
5717 }
5718 })
5719 .await
5720 .unwrap();
5721 assert_eq!(*entered.lock().unwrap(), vec![1]);
5722 assert_eq!(dispatcher.pools.default.permits.available_permits(), 1);
5723 assert!(timeout(Duration::from_millis(50), rx.recv()).await.is_err());
5724 }
5725
5726 #[tokio::test]
5727 async fn cancelled_terminal_is_not_sent_after_route_teardown() {
5728 let (tx, mut rx) = mpsc::channel(1);
5729 let (module_handle, _unused_rx) = test_module_handle(&[]);
5730 let handle = RouteHandle::new(7, 1, 1);
5731 module_handle.install_route(handle).unwrap();
5732 let ctx = RequestCtx {
5733 handle,
5734 corr: 2,
5735 ver: PROTOCOL_VERSION,
5736 egress: tx,
5737 module_handle: module_handle.clone(),
5738 cancelled: CancellationToken::new(),
5739 binary_reply: Arc::new(AtomicBool::new(false)),
5740 request_flags: Flags::new(false, Priority::Interactive, false),
5741 request_body: Arc::from([]),
5742 };
5743 assert!(module_handle.remove_route(handle).unwrap());
5744
5745 let result = send_handler_outcome(
5746 &ctx,
5747 HandlerOutcome::Error {
5748 code: "cancelled".to_string(),
5749 message: "request cancelled".to_string(),
5750 },
5751 )
5752 .await;
5753
5754 assert!(matches!(
5755 result,
5756 Err(SubcModuleError::StaleRouteHandle(stale)) if stale == handle
5757 ));
5758 assert!(rx.try_recv().is_err());
5759 }
5760
5761 struct CountingHandler {
5762 calls: Arc<AtomicUsize>,
5763 }
5764
5765 #[async_trait]
5766 impl ModuleHandler for CountingHandler {
5767 async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
5768 self.calls.fetch_add(1, Ordering::SeqCst);
5769 HandlerOutcome::Response(Vec::new())
5770 }
5771 }
5772
5773 #[tokio::test]
5774 async fn endpoint_validation_drops_stale_request_before_handler_dispatch() {
5775 let (tx, mut rx) = mpsc::channel(4);
5776 let calls = Arc::new(AtomicUsize::new(0));
5777 let handler = Arc::new(CountingHandler {
5778 calls: Arc::clone(&calls),
5779 });
5780 let dispatcher = RequestDispatcher::new();
5781 let (module_handle, _unused_rx) = test_module_handle(&[]);
5782 module_handle
5783 .install_route(RouteHandle::new(7, 2, 1))
5784 .unwrap();
5785 let stale = Frame::build(FrameType::Request, data_flags(), 7, 1, 55, Vec::new()).unwrap();
5786
5787 assert!(
5788 handle_frame(stale, &tx, handler, dispatcher, module_handle.clone())
5789 .await
5790 .unwrap()
5791 );
5792 tokio::task::yield_now().await;
5793 assert_eq!(calls.load(Ordering::SeqCst), 0);
5794 assert_eq!(module_handle.dropped_route_frames(), 1);
5795 assert!(rx.try_recv().is_err());
5796 }
5797
5798 struct BindOrderingHandler {
5799 module_handle: ModuleHandle,
5800 bind_emit_rejected: Arc<AtomicUsize>,
5801 bound: Arc<AtomicUsize>,
5802 cleanup: Arc<AtomicUsize>,
5803 }
5804
5805 #[async_trait]
5806 impl ModuleHandler for BindOrderingHandler {
5807 async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
5808 HandlerOutcome::Response(Vec::new())
5809 }
5810
5811 async fn on_bind(&self, req: &RouteBindRequest) -> BindDecision {
5812 if matches!(
5813 self.module_handle
5814 .push(&req.handle, b"too-early".to_vec(), None)
5815 .await,
5816 Err(SubcModuleError::StaleRouteHandle(_))
5817 ) {
5818 self.bind_emit_rejected.fetch_add(1, Ordering::SeqCst);
5819 }
5820 BindDecision::accept()
5821 }
5822
5823 async fn on_bound(&self, handle: &RouteHandle) {
5824 self.bound.fetch_add(1, Ordering::SeqCst);
5825 self.module_handle
5826 .push(handle, b"bound".to_vec(), Some(AdmissionClass::Expedite))
5827 .await
5828 .unwrap();
5829 }
5830
5831 async fn on_route_gone(&self, _handle: &RouteHandle) {
5832 self.cleanup.fetch_add(1, Ordering::SeqCst);
5833 }
5834 }
5835
5836 fn route_bind_frame(channel: u16, epoch: u32, corr: u64) -> Frame {
5837 let body = serde_json::to_vec(&ModuleControlRequest::RouteBind {
5838 route_channel: channel,
5839 epoch,
5840 target: RouteTarget::ToolProvider {
5841 module_id: "provider".to_string(),
5842 },
5843 identity: BindIdentity::new("test".to_string(), "bind".to_string()),
5844 principal: None,
5845 consumer_capabilities: None,
5846 role_versions: None,
5847 admission_facts: None,
5848 scope: None,
5849 call_mode: None,
5850 })
5851 .unwrap();
5852 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5853 }
5854
5855 #[tokio::test]
5856 async fn on_bound_runs_only_after_ack_queue_and_handle_install() {
5857 let (module_handle, mut rx) = test_module_handle(&[]);
5858 let tx = module_handle.shared.lock_inner().writer.clone().unwrap();
5859 let bind_emit_rejected = Arc::new(AtomicUsize::new(0));
5860 let bound = Arc::new(AtomicUsize::new(0));
5861 let cleanup = Arc::new(AtomicUsize::new(0));
5862 let handler = Arc::new(BindOrderingHandler {
5863 module_handle: module_handle.clone(),
5864 bind_emit_rejected: Arc::clone(&bind_emit_rejected),
5865 bound: Arc::clone(&bound),
5866 cleanup: Arc::clone(&cleanup),
5867 });
5868
5869 assert!(handle_frame(
5870 route_bind_frame(8, 4, 90),
5871 &tx,
5872 handler,
5873 RequestDispatcher::new(),
5874 module_handle.clone(),
5875 )
5876 .await
5877 .unwrap());
5878 assert_eq!(bind_emit_rejected.load(Ordering::SeqCst), 1);
5879 assert_eq!(bound.load(Ordering::SeqCst), 1);
5880 assert_eq!(cleanup.load(Ordering::SeqCst), 0);
5881
5882 let ack = rx.recv().await.unwrap();
5883 let push = rx.recv().await.unwrap();
5884 assert_eq!(ack.header.ty, FrameType::Response);
5885 assert_eq!(ack.header.channel, 0);
5886 assert_eq!(push.header.ty, FrameType::Push);
5887 assert_eq!((push.header.channel, push.header.epoch), (8, 4));
5888 assert_eq!(
5889 push.header.flags.admission_class(),
5890 Some(AdmissionClass::Expedite)
5891 );
5892
5893 let captured = RouteHandle::new(8, 4, 1);
5894 assert!(module_handle.remove_route(captured).unwrap());
5895 let stale = module_handle
5896 .push(&captured, Vec::new(), None)
5897 .await
5898 .unwrap_err();
5899 assert!(matches!(stale, SubcModuleError::StaleRouteHandle(_)));
5900 assert!(rx.try_recv().is_err());
5901 }
5902
5903 struct RejectingHandler {
5904 bound: Arc<AtomicUsize>,
5905 cleanup: Arc<AtomicUsize>,
5906 }
5907
5908 #[async_trait]
5909 impl ModuleHandler for RejectingHandler {
5910 async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
5911 HandlerOutcome::Response(Vec::new())
5912 }
5913
5914 async fn on_bind(&self, _req: &RouteBindRequest) -> BindDecision {
5915 BindDecision::reject("no", "rejected")
5916 }
5917
5918 async fn on_bound(&self, _handle: &RouteHandle) {
5919 self.bound.fetch_add(1, Ordering::SeqCst);
5920 }
5921
5922 async fn on_route_gone(&self, _handle: &RouteHandle) {
5923 self.cleanup.fetch_add(1, Ordering::SeqCst);
5924 }
5925 }
5926
5927 #[tokio::test]
5928 async fn rejected_bind_cleans_up_without_installing_or_calling_on_bound() {
5929 let (module_handle, mut rx) = test_module_handle(&[]);
5930 let tx = module_handle.shared.lock_inner().writer.clone().unwrap();
5931 let bound = Arc::new(AtomicUsize::new(0));
5932 let cleanup = Arc::new(AtomicUsize::new(0));
5933 let handler = Arc::new(RejectingHandler {
5934 bound: Arc::clone(&bound),
5935 cleanup: Arc::clone(&cleanup),
5936 });
5937 handle_frame(
5938 route_bind_frame(6, 3, 91),
5939 &tx,
5940 handler,
5941 RequestDispatcher::new(),
5942 module_handle.clone(),
5943 )
5944 .await
5945 .unwrap();
5946 let refusal = rx.recv().await.unwrap();
5947 assert_eq!(refusal.header.ty, FrameType::Error);
5948 assert_eq!(refusal.body, br#"{"code":"no","message":"rejected"}"#);
5950 assert_eq!(bound.load(Ordering::SeqCst), 0);
5951 assert_eq!(cleanup.load(Ordering::SeqCst), 1);
5952 assert!(matches!(
5953 module_handle.validate_route(RouteHandle::new(6, 3, 1)),
5954 Err(SubcModuleError::StaleRouteHandle(_))
5955 ));
5956 }
5957
5958 struct BusyHandler;
5959
5960 #[async_trait]
5961 impl ModuleHandler for BusyHandler {
5962 async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
5963 HandlerOutcome::Response(Vec::new())
5964 }
5965
5966 async fn on_bind(&self, _req: &RouteBindRequest) -> BindDecision {
5967 BindDecision::reject_with_detail(
5968 "bind_busy",
5969 "a read is still running on this root",
5970 serde_json::json!({"reason": "reader", "retry_after_ms": 2000}),
5971 )
5972 }
5973 }
5974
5975 #[tokio::test]
5976 async fn rejected_bind_carries_its_detail_to_the_daemon() {
5977 let (module_handle, mut rx) = test_module_handle(&[]);
5978 let tx = module_handle.shared.lock_inner().writer.clone().unwrap();
5979 handle_frame(
5980 route_bind_frame(6, 3, 92),
5981 &tx,
5982 Arc::new(BusyHandler),
5983 RequestDispatcher::new(),
5984 module_handle.clone(),
5985 )
5986 .await
5987 .unwrap();
5988 let refusal = rx.recv().await.unwrap();
5989 assert_eq!(refusal.header.ty, FrameType::Error);
5990 let body: ErrorBody = serde_json::from_slice(&refusal.body).unwrap();
5991 assert_eq!(body.code, "bind_busy");
5992 assert_eq!(
5993 body.detail,
5994 Some(serde_json::json!({"reason": "reader", "retry_after_ms": 2000}))
5995 );
5996 }
5997
5998 struct RebindCountingHandler {
5999 bound: Arc<AtomicUsize>,
6000 cleanup: Arc<AtomicUsize>,
6001 }
6002
6003 #[async_trait]
6004 impl ModuleHandler for RebindCountingHandler {
6005 async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
6006 HandlerOutcome::Response(Vec::new())
6007 }
6008
6009 async fn on_bound(&self, _handle: &RouteHandle) {
6010 self.bound.fetch_add(1, Ordering::SeqCst);
6011 }
6012
6013 async fn on_route_gone(&self, _handle: &RouteHandle) {
6014 self.cleanup.fetch_add(1, Ordering::SeqCst);
6015 }
6016 }
6017
6018 #[tokio::test]
6024 async fn rebind_on_installed_channel_replaces_on_higher_epoch_only() {
6025 let (module_handle, mut rx) = test_module_handle(&[]);
6026 let tx = module_handle.shared.lock_inner().writer.clone().unwrap();
6027 let bound = Arc::new(AtomicUsize::new(0));
6028 let cleanup = Arc::new(AtomicUsize::new(0));
6029 let handler = Arc::new(RebindCountingHandler {
6030 bound: Arc::clone(&bound),
6031 cleanup: Arc::clone(&cleanup),
6032 });
6033
6034 handle_frame(
6036 route_bind_frame(8, 4, 90),
6037 &tx,
6038 Arc::clone(&handler),
6039 RequestDispatcher::new(),
6040 module_handle.clone(),
6041 )
6042 .await
6043 .unwrap();
6044 assert_eq!(rx.recv().await.unwrap().header.ty, FrameType::Response);
6045 assert_eq!(
6046 (bound.load(Ordering::SeqCst), cleanup.load(Ordering::SeqCst)),
6047 (1, 0)
6048 );
6049
6050 handle_frame(
6052 route_bind_frame(8, 4, 91),
6053 &tx,
6054 Arc::clone(&handler),
6055 RequestDispatcher::new(),
6056 module_handle.clone(),
6057 )
6058 .await
6059 .unwrap();
6060 let reject = rx.recv().await.unwrap();
6061 assert_eq!(reject.header.ty, FrameType::Error);
6062 assert_eq!(
6063 (bound.load(Ordering::SeqCst), cleanup.load(Ordering::SeqCst)),
6064 (1, 0)
6065 );
6066 module_handle
6067 .validate_route(RouteHandle::new(8, 4, 1))
6068 .expect("epoch-4 install must survive the rejected rebind");
6069
6070 handle_frame(
6072 route_bind_frame(8, 3, 92),
6073 &tx,
6074 Arc::clone(&handler),
6075 RequestDispatcher::new(),
6076 module_handle.clone(),
6077 )
6078 .await
6079 .unwrap();
6080 assert_eq!(rx.recv().await.unwrap().header.ty, FrameType::Error);
6081 assert_eq!(
6082 (bound.load(Ordering::SeqCst), cleanup.load(Ordering::SeqCst)),
6083 (1, 0)
6084 );
6085
6086 handle_frame(
6089 route_bind_frame(8, 5, 93),
6090 &tx,
6091 Arc::clone(&handler),
6092 RequestDispatcher::new(),
6093 module_handle.clone(),
6094 )
6095 .await
6096 .unwrap();
6097 assert_eq!(rx.recv().await.unwrap().header.ty, FrameType::Response);
6098 assert_eq!(
6099 (bound.load(Ordering::SeqCst), cleanup.load(Ordering::SeqCst)),
6100 (2, 1)
6101 );
6102 assert!(matches!(
6103 module_handle.validate_route(RouteHandle::new(8, 4, 1)),
6104 Err(SubcModuleError::StaleRouteHandle(_))
6105 ));
6106 module_handle
6107 .validate_route(RouteHandle::new(8, 5, 1))
6108 .expect("epoch-5 install must be live after implicit replace");
6109 }
6110
6111 #[tokio::test]
6112 async fn module_control_corr_is_monotonic_and_exhausts_without_wrap() {
6113 let (module_handle, _rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
6114 let mut inner = module_handle.shared.lock_inner();
6115 inner.next_corr = Some(u64::MAX);
6116 assert_eq!(next_module_control_corr(&mut inner), Some(u64::MAX));
6117 assert_eq!(next_module_control_corr(&mut inner), None);
6118 }
6119}
6120
6121#[cfg(test)]
6124mod module_close_tests {
6125 use std::time::Instant;
6126
6127 use tokio::{io::DuplexStream, sync::Notify};
6128
6129 use super::stand_in::{serve_against_stand_in_with_ops, Served};
6130 use super::*;
6131
6132 #[derive(Clone, Copy)]
6134 enum Hold {
6135 UntilCancelled,
6137 IgnoringCancellation,
6139 }
6140
6141 struct HoldingHandler {
6142 hold: Hold,
6143 entered: Arc<Notify>,
6144 token: Arc<Mutex<Option<CancellationToken>>>,
6145 }
6146
6147 #[async_trait]
6148 impl ModuleHandler for HoldingHandler {
6149 async fn handle(&self, ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
6150 *self.token.lock().unwrap() = Some(ctx.cancellation_token());
6151 self.entered.notify_one();
6152 match self.hold {
6153 Hold::UntilCancelled => {
6154 ctx.cancelled().await;
6155 HandlerOutcome::Error {
6156 code: "cancelled".to_string(),
6157 message: "request cancelled".to_string(),
6158 }
6159 }
6160 Hold::IgnoringCancellation => std::future::pending().await,
6161 }
6162 }
6163 }
6164
6165 async fn serve_against_stand_in<H: ModuleHandler>(handler: H) -> Served {
6167 serve_against_stand_in_with_ops(|_| handler, &[]).await
6168 }
6169
6170 async fn send(daemon: &mut DuplexStream, frame: Frame) {
6171 write_frame(daemon, &frame).await.unwrap();
6172 daemon.flush().await.unwrap();
6173 }
6174
6175 async fn hold_one_request(served: &mut Served, entered: &Notify) {
6178 let bind = serde_json::to_vec(&ModuleControlRequest::RouteBind {
6179 route_channel: 7,
6180 epoch: 1,
6181 target: RouteTarget::ToolProvider {
6182 module_id: "close-test".to_string(),
6183 },
6184 identity: BindIdentity::new("test".to_string(), "close".to_string()),
6185 principal: None,
6186 consumer_capabilities: None,
6187 role_versions: None,
6188 admission_facts: None,
6189 scope: None,
6190 call_mode: None,
6191 })
6192 .unwrap();
6193 send(
6194 &mut served.daemon,
6195 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, bind).unwrap(),
6196 )
6197 .await;
6198 loop {
6199 let frame = read_frame(&mut served.daemon).await.unwrap().unwrap();
6200 if frame.header.channel == 0 && frame.header.corr == 2 {
6201 assert_eq!(
6202 frame.header.ty,
6203 FrameType::Response,
6204 "route.bind must be acked"
6205 );
6206 break;
6207 }
6208 }
6209 send(
6210 &mut served.daemon,
6211 Frame::build(FrameType::Request, data_flags(), 7, 1, 10, b"hold".to_vec()).unwrap(),
6212 )
6213 .await;
6214 timeout(Duration::from_secs(2), entered.notified())
6215 .await
6216 .expect("the handler must receive the request");
6217 }
6218
6219 fn goodbye() -> Frame {
6220 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 0, Vec::new()).unwrap()
6221 }
6222
6223 fn holding(
6224 hold: Hold,
6225 ) -> (
6226 HoldingHandler,
6227 Arc<Notify>,
6228 Arc<Mutex<Option<CancellationToken>>>,
6229 ) {
6230 let entered = Arc::new(Notify::new());
6231 let token = Arc::new(Mutex::new(None));
6232 let handler = HoldingHandler {
6233 hold,
6234 entered: Arc::clone(&entered),
6235 token: Arc::clone(&token),
6236 };
6237 (handler, entered, token)
6238 }
6239
6240 #[tokio::test]
6241 async fn goodbye_cancels_an_open_request_and_the_serve_future_completes() {
6242 let (handler, entered, token) = holding(Hold::UntilCancelled);
6243 let mut served = serve_against_stand_in(handler).await;
6244 hold_one_request(&mut served, &entered).await;
6245
6246 let started = Instant::now();
6247 send(&mut served.daemon, goodbye()).await;
6248 let result = timeout(Duration::from_secs(5), &mut served.serve)
6249 .await
6250 .expect("a held request must not keep the serve future alive after GOODBYE");
6251 result.unwrap().unwrap();
6252 assert!(
6253 token.lock().unwrap().as_ref().unwrap().is_cancelled(),
6254 "GOODBYE must cancel the request's token"
6255 );
6256 assert!(
6259 started.elapsed() < DEFAULT_WRITER_DRAIN_TIMEOUT,
6260 "closing took {:?}",
6261 started.elapsed()
6262 );
6263 }
6264
6265 #[tokio::test]
6266 async fn a_handler_ignoring_cancellation_cannot_keep_the_serve_future_alive() {
6267 let (handler, entered, _token) = holding(Hold::IgnoringCancellation);
6268 let mut served = serve_against_stand_in(handler).await;
6269 hold_one_request(&mut served, &entered).await;
6270
6271 let started = Instant::now();
6272 send(&mut served.daemon, goodbye()).await;
6273 let result = timeout(
6274 DEFAULT_WRITER_DRAIN_TIMEOUT + Duration::from_secs(3),
6275 &mut served.serve,
6276 )
6277 .await
6278 .expect("the writer drain must be bounded");
6279 result.unwrap().unwrap();
6280 assert!(started.elapsed() >= DEFAULT_WRITER_DRAIN_TIMEOUT);
6281 }
6282
6283 #[tokio::test]
6284 async fn closed_resolves_after_goodbye() {
6285 let served = serve_against_stand_in(EchoHandler).await;
6286 let mut served = served;
6287 assert!(!served.handle.is_closed());
6288 let closed = served.handle.closed();
6289 send(&mut served.daemon, goodbye()).await;
6290 timeout(Duration::from_secs(2), closed)
6291 .await
6292 .expect("closed() must resolve after GOODBYE");
6293 assert!(served.handle.is_closed());
6294 }
6295
6296 #[tokio::test]
6297 async fn closed_resolves_after_eof() {
6298 let Served {
6299 daemon,
6300 handle,
6301 serve: _serve,
6302 } = serve_against_stand_in(EchoHandler).await;
6303 assert!(!handle.is_closed());
6304 drop(daemon);
6305 timeout(Duration::from_secs(2), handle.closed())
6306 .await
6307 .expect("closed() must resolve after EOF");
6308 assert!(handle.is_closed());
6309 }
6310
6311 struct EchoHandler;
6312
6313 #[async_trait]
6314 impl ModuleHandler for EchoHandler {
6315 async fn handle(&self, _ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome {
6316 HandlerOutcome::Response(body)
6317 }
6318 }
6319
6320 #[derive(Clone, Default)]
6323 struct EndRecorder(Arc<Mutex<Vec<ConnectionEnd>>>);
6324
6325 #[async_trait]
6326 impl ModuleHandler for EndRecorder {
6327 async fn handle(&self, _ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome {
6328 HandlerOutcome::Response(body)
6329 }
6330 async fn on_connection_end(&self, end: ConnectionEnd) {
6331 self.0.lock().unwrap().push(end);
6332 }
6333 }
6334
6335 async fn reported_end(recorder: &EndRecorder, served: &mut Served) -> Vec<ConnectionEnd> {
6336 timeout(Duration::from_secs(5), &mut served.serve)
6337 .await
6338 .expect("serving must end")
6339 .unwrap()
6340 .unwrap();
6341 recorder.0.lock().unwrap().clone()
6342 }
6343
6344 #[tokio::test]
6345 async fn a_daemon_goodbye_is_reported_as_goodbye() {
6346 let recorder = EndRecorder::default();
6347 let mut served = serve_against_stand_in(recorder.clone()).await;
6348 send(&mut served.daemon, goodbye()).await;
6349 assert_eq!(
6350 reported_end(&recorder, &mut served).await,
6351 vec![ConnectionEnd::Goodbye]
6352 );
6353 }
6354
6355 #[tokio::test]
6356 async fn a_daemon_closing_without_goodbye_is_reported_as_eof() {
6357 let recorder = EndRecorder::default();
6358 let mut served = serve_against_stand_in(recorder.clone()).await;
6359 served.daemon.shutdown().await.unwrap();
6360 assert_eq!(
6361 reported_end(&recorder, &mut served).await,
6362 vec![ConnectionEnd::Eof]
6363 );
6364 }
6365
6366 #[derive(Debug, Clone, PartialEq)]
6368 enum Seen {
6369 Draining(RouteCloseReason, SystemTime),
6370 End(ConnectionEnd),
6371 }
6372
6373 #[derive(Clone, Default)]
6376 struct DrainRecorder {
6377 seen: Arc<Mutex<Vec<Seen>>>,
6378 park: bool,
6379 }
6380
6381 #[async_trait]
6382 impl ModuleHandler for DrainRecorder {
6383 async fn handle(&self, _ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome {
6384 HandlerOutcome::Response(body)
6385 }
6386 async fn on_draining(&self, reason: RouteCloseReason, deadline: SystemTime) {
6387 self.seen
6388 .lock()
6389 .unwrap()
6390 .push(Seen::Draining(reason, deadline));
6391 if self.park {
6392 std::future::pending::<()>().await;
6393 }
6394 }
6395 async fn on_connection_end(&self, end: ConnectionEnd) {
6396 self.seen.lock().unwrap().push(Seen::End(end));
6397 }
6398 }
6399
6400 fn control_push(body: &[u8]) -> Frame {
6401 Frame::build(FrameType::Push, control_flags(), 0, 0, 0, body.to_vec()).unwrap()
6402 }
6403
6404 fn draining_push(reason: subc_protocol::RouteCloseReason, deadline_ms: u64) -> Frame {
6405 control_push(
6406 &serde_json::to_vec(&ModuleControlCommand::Draining {
6407 reason,
6408 deadline_ms,
6409 })
6410 .unwrap(),
6411 )
6412 }
6413
6414 async fn ping_is_answered(served: &mut Served, corr: u64) {
6417 send(
6418 &mut served.daemon,
6419 Frame::build(FrameType::Ping, control_flags(), 0, 0, corr, Vec::new()).unwrap(),
6420 )
6421 .await;
6422 let pong = timeout(Duration::from_secs(2), read_frame(&mut served.daemon))
6423 .await
6424 .expect("the module must answer PING")
6425 .unwrap()
6426 .expect("the module must not close the connection");
6427 assert_eq!(pong.header.ty, FrameType::Pong);
6428 assert_eq!(pong.header.corr, corr);
6429 }
6430
6431 #[tokio::test]
6435 async fn a_drain_notice_calls_on_draining_before_the_goodbye_that_follows_it() {
6436 let recorder = DrainRecorder::default();
6437 let mut served = serve_against_stand_in(recorder.clone()).await;
6438 let deadline_ms = 4_102_444_800_000;
6441 write_frame(
6442 &mut served.daemon,
6443 &draining_push(subc_protocol::RouteCloseReason::Restart, deadline_ms),
6444 )
6445 .await
6446 .unwrap();
6447 write_frame(&mut served.daemon, &goodbye()).await.unwrap();
6448 served.daemon.flush().await.unwrap();
6449
6450 timeout(Duration::from_secs(5), &mut served.serve)
6451 .await
6452 .expect("serving must end after GOODBYE")
6453 .unwrap()
6454 .unwrap();
6455 assert_eq!(
6456 *recorder.seen.lock().unwrap(),
6457 vec![
6458 Seen::Draining(
6459 RouteCloseReason::Restart,
6460 UNIX_EPOCH + Duration::from_millis(deadline_ms)
6461 ),
6462 Seen::End(ConnectionEnd::Goodbye),
6463 ]
6464 );
6465 }
6466
6467 #[tokio::test]
6470 async fn a_drain_hook_that_never_returns_does_not_stop_the_frame_reader() {
6471 let recorder = DrainRecorder {
6472 park: true,
6473 ..DrainRecorder::default()
6474 };
6475 let mut served = serve_against_stand_in(recorder.clone()).await;
6476 send(
6477 &mut served.daemon,
6478 draining_push(subc_protocol::RouteCloseReason::Reload, 1),
6479 )
6480 .await;
6481 ping_is_answered(&mut served, 41).await;
6482 assert!(matches!(
6483 recorder.seen.lock().unwrap().as_slice(),
6484 [Seen::Draining(RouteCloseReason::Reload, _)]
6485 ));
6486
6487 send(&mut served.daemon, goodbye()).await;
6488 timeout(Duration::from_secs(5), &mut served.serve)
6489 .await
6490 .expect("a parked drain hook must not keep the serve future alive")
6491 .unwrap()
6492 .unwrap();
6493 }
6494
6495 #[tokio::test]
6499 async fn an_undecodable_channel_zero_push_is_ignored_and_the_connection_stays_up() {
6500 let recorder = DrainRecorder::default();
6501 let mut served = serve_against_stand_in(recorder.clone()).await;
6502 send(
6503 &mut served.daemon,
6504 control_push(br#"{"op":"module.some_future_command","x":1}"#),
6505 )
6506 .await;
6507 send(&mut served.daemon, control_push(b"not json")).await;
6508 send(
6509 &mut served.daemon,
6510 control_push(br#"{"op":"module.draining","reason":"restart"}"#),
6511 )
6512 .await;
6513 ping_is_answered(&mut served, 42).await;
6514 assert!(!served.handle.is_closed());
6515 assert!(!served.serve.is_finished());
6516 assert!(recorder.seen.lock().unwrap().is_empty());
6517 }
6518
6519 #[tokio::test]
6520 async fn a_module_closing_its_own_connection_is_reported_as_closed() {
6521 let recorder = EndRecorder::default();
6522 let mut served = serve_against_stand_in(recorder.clone()).await;
6523 served.handle.close_connection();
6524 assert_eq!(
6525 reported_end(&recorder, &mut served).await,
6526 vec![ConnectionEnd::Closed]
6527 );
6528 }
6529}
6530
6531#[cfg(test)]
6532mod operator_confirm_tests;
6533
6534#[cfg(test)]
6535mod binary_reply_tests;
6536
6537#[cfg(test)]
6538mod serve_lifecycle_tests;
6539
6540#[cfg(test)]
6541mod request_pool_tests;
6542
6543#[cfg(test)]
6544mod detailed_error_body_tests {
6545 use serde_json::json;
6546
6547 use subc_protocol::ErrorBody;
6548
6549 #[test]
6553 fn detailed_error_body_keeps_code_message_and_detail() {
6554 let body = ErrorBody::new("bad_request", "invalid envelope")
6555 .with_detail(json!({"reason": "missing_server"}));
6556
6557 assert_eq!(
6558 serde_json::to_value(body).unwrap(),
6559 json!({
6560 "code": "bad_request",
6561 "message": "invalid envelope",
6562 "detail": {"reason": "missing_server"},
6563 })
6564 );
6565 }
6566
6567 #[test]
6570 fn detail_less_error_body_is_byte_identical_to_the_pre_detail_wire() {
6571 let old_wire = r#"{"code":"cancelled","message":"caller cancelled"}"#;
6572 let parsed: ErrorBody = serde_json::from_str(old_wire).unwrap();
6573 assert_eq!(parsed.detail, None);
6574 assert_eq!(serde_json::to_string(&parsed).unwrap(), old_wire);
6575 }
6576}
6577
6578static RETAINED_LAUNCH_NONCE: std::sync::OnceLock<String> = std::sync::OnceLock::new();
6581
6582pub fn retain_launch_nonce_for_hello(launch_nonce: String) -> Result<(), String> {
6585 RETAINED_LAUNCH_NONCE
6586 .set(launch_nonce)
6587 .map_err(|_| "launch nonce was already retained for this process".to_string())
6588}
6589
6590fn retained_launch_nonce() -> Option<String> {
6591 RETAINED_LAUNCH_NONCE.get().cloned()
6592}
6593
6594pub fn launch_nonce_source() -> Option<LaunchNonceSource> {
6600 launch_nonce()
6601 .ok()
6602 .flatten()
6603 .map(|nonce| LaunchNonceSource::from_wire_name(nonce.source().as_str()))
6604}
6605
6606fn stamp_launch_nonce_source(manifest: &mut ModuleManifest, sent: Option<&str>) {
6614 let Some(provenance) = manifest.provenance.as_mut() else {
6615 return;
6616 };
6617 if provenance.launch_nonce_source.is_some() {
6618 return;
6619 }
6620 if let Ok(Some(nonce)) = launch_nonce() {
6621 let launched = registration::is_launched_module(&manifest.module_id).unwrap_or(false);
6624 if launched && (registration::supervised() || sent == Some(nonce.value())) {
6625 provenance.launch_nonce_source =
6626 Some(LaunchNonceSource::from_wire_name(nonce.source().as_str()));
6627 }
6628 }
6629}
6630
6631#[cfg(test)]
6632mod checked_increment_tests {
6633 use super::checked_increment;
6634 use std::sync::atomic::{AtomicU64, Ordering};
6635
6636 #[test]
6637 fn returns_the_previous_value_and_stops_at_the_maximum() {
6638 let counter = AtomicU64::new(7);
6639 assert_eq!(checked_increment(&counter), Some(7));
6640 assert_eq!(counter.load(Ordering::Relaxed), 8);
6641
6642 let full = AtomicU64::new(u64::MAX);
6643 assert_eq!(checked_increment(&full), None);
6644 assert_eq!(
6645 full.load(Ordering::Relaxed),
6646 u64::MAX,
6647 "an exhausted counter is left unchanged"
6648 );
6649 }
6650}