1#![forbid(unsafe_code)]
2
3pub mod activity;
4mod agent_commands;
5pub mod config;
6mod control_ops;
7pub use agent_commands::AgentCommandsError;
8pub use subc_protocol::session::{AgentCommandFinished, AgentCommandRoot};
9pub mod consumer;
10mod operator_app;
11pub mod policy_cache;
12#[cfg(any(test, feature = "test-support"))]
13mod stand_in;
14#[cfg(feature = "test-support")]
15pub mod test_support;
16pub use activity::*;
17pub use consumer::{
18 is_retryable_route_open_code, AppOpenError, CallError, CallOptions, CallReply, CancelHandle,
19 CancelOutcome, CatalogList, CatalogSubscription, CloseRouteOptions, ConnectionFacts,
20 ConnectionState, ConsumerCapabilityError, ConsumerError, ConsumerOptions, ControlPush,
21 CorrelationId, HealthProbeResponse, NotSentCause, OutcomeUnknownCause, PendingReply, PushEvent,
22 RequestBody, RequestReservation, RetryBackoff, ReverseRequestContext, ReverseRequestError,
23 ReverseRequestRegistrationError, ReverseRequestRegistry, RouteCloseDisposition,
24 RouteCloseReason, RouteEndReason, RouteOpenTarget, RouteOpenTargetKind, RoutePollResult,
25 ScopeBindProofRefusal, ScopeSelector, ServerDescription, SpawnStreamError, SpawnSubscription,
26 StderrTailResponse, SubcConsumer, SubscribeOptions, Subscription, SubscriptionClosed,
27 SubscriptionEnd, SupervisorAck, SupervisorEntry, SupervisorHealthResponse,
28 SupervisorListResponse, SupervisorProvenanceResponse, ACCEPTED_CLOSE_REASONS,
29 DEFAULT_CALL_TIMEOUT, DEFAULT_LIVENESS_PROBE_WINDOW, DEFAULT_ROUTE_RETRY_DEADLINE,
30 SPAWN_CURSOR_INCARNATION_MISMATCH, SPAWN_CURSOR_TOO_OLD, SPAWN_SUBSCRIBER_LAGGED,
31};
32pub use operator_app::{operator_app_candidates, operator_app_path, OperatorAppPathError};
33pub use policy_cache::{
34 PolicyResolveError, PolicyResolver, PolicyResolverConfig, PolicyResolverFootprint,
35 PolicyVerdict, ProjectRef, Subject, DEFAULT_POLICY_RESOLVER_MODULE_ID,
36};
37pub use subc_control::{AppOpenReply, AppOpenStatus, CatalogOrigin, CatalogWatchResponse};
38pub use subc_control::{
39 ScopeRef, ScopeWatchEvent, ScopeWatchPush, ScopeWatchResponse, ScopeWatchSelector,
40 ScopeWatchState, ScopeWatchStatus,
41};
42
43mod module_subscription;
44pub use module_subscription::ScopeSubscription;
45
46use std::{
47 collections::{BTreeMap, HashMap, VecDeque},
48 env,
49 error::Error,
50 ffi::OsString,
51 fmt,
52 future::Future,
53 io,
54 panic::{catch_unwind, AssertUnwindSafe},
55 path::{Path, PathBuf},
56 pin::Pin,
57 sync::{
58 atomic::{AtomicBool, AtomicU64, Ordering},
59 Arc, Mutex,
60 },
61 time::{Duration, SystemTime, UNIX_EPOCH},
62};
63
64pub use async_trait::async_trait;
65pub use subc_control::{CatalogEntry, ConsumerIdentity};
66mod scope_served_by;
67pub use scope_served_by::{ScopeServedByAnswer, ScopeServedByReply};
68pub mod registration;
69mod route_authorize;
70pub use route_authorize::{
71 RouteAuthorizeExpected, RouteAuthorizeOutcome, RouteAuthorizeResponse, RouteAuthorizeScope,
72};
73mod request_pools;
74pub use registration::ClaimError;
75use request_pools::{wait_for_permit, DispatchPools, PermitWaitOutcome, WaitingRequest};
76pub use request_pools::{RequestClass, RequestPool, RequestPoolSnapshot, RequestPools};
77pub use subc_os::launch_nonce;
84use subc_protocol::{
85 call_mode::derive_call_mode,
86 error_codes,
87 manifest::ModuleManifest,
88 scope::{
89 ScopeEnd, ScopeEndResult, ScopeEnded, ScopeRecord, ScopeRecordResult, ScopeStamp,
90 ScopeStatus, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
91 },
92 session::{
93 ModuleControlCommand, ModuleControlRequest, ModuleControlRequestFromModule,
94 ModuleControlResponse, ModuleControlResponseToModule, OperatorConfirmKind,
95 OperatorConfirmReply, OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK,
96 MODULE_TO_SUBC_OP_CATALOG_UPDATE, MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
97 },
98 BindIdentity, ErrorBody, Flags, Frame, FrameBuildError, FrameType, ModuleHelloAckBody,
99 ModuleHelloBody, Principal, Priority, RouteTarget, PROTOCOL_VERSION, SUBC_MODULE_ID_ENV,
100};
101pub use subc_protocol::{
102 call_mode::{CallMode, CALL_MODES_CAPABILITY},
103 manifest::{
104 build_provenance, CapabilityDeclarations, CapabilityNeed, CapabilityRequirement,
105 ExecutionMode, LaunchNonceSource, ManifestProvenance, ProvenanceFormError, ProviderRole,
106 Tool, PROVENANCE_SENTINELS,
107 },
108 session::{HealthReport, HealthStatus},
109 AdmissionClass, MachineId, SUBC_PROTOCOL_CRATE_VERSION,
110};
111pub use subc_transport::connection_file::{
112 discover, discovery_candidates, Discovered, DiscoveryError,
113};
114
115use subc_transport::{
116 authenticate_client, connection_file, read_frame, write_frame, AuthError, ConnectionFileError,
117 FrameIoError,
118};
119use tokio::{
120 io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufWriter},
121 net::TcpStream,
122 sync::{mpsc, oneshot},
123 time::{timeout, timeout_at, Instant},
124};
125use tokio_util::{
126 sync::{CancellationToken, WaitForCancellationFuture},
127 task::TaskTracker,
128};
129
130const AUTH_DEADLINE: Duration = Duration::from_secs(2);
131const CATALOG_UPDATE_TIMEOUT: Duration = Duration::from_secs(10);
132const OPERATOR_CONFIRM_TIMEOUT: Duration = Duration::from_secs(290);
133const OPERATOR_CONFIRM_OP: &str = "operator.confirm";
134const EGRESS_BUFFER: usize = 64;
135const HANDLER_TASK_CAPACITY: usize = 64;
136const DISPATCH_SATURATION_WAIT: Duration = Duration::from_secs(2);
139const HELLO_CORR: u64 = 1;
140const DEFAULT_WRITER_DRAIN_TIMEOUT: Duration = Duration::from_secs(2);
141const DRAINING_HOOK_START_LIMIT: Duration = Duration::from_secs(2);
149static NEXT_MODULE_CONNECTION_TOKEN: AtomicU64 = AtomicU64::new(1);
150const DETACHED_CONNECTION_TOKEN: u64 = 0;
154
155type RequestKey = (u16, u32, u64);
156type InFlight = Arc<Mutex<HashMap<RequestKey, CancellationToken>>>;
157
158#[derive(Clone, Copy)]
164pub struct RouteHandle {
165 pub channel: u16,
166 pub epoch: u32,
167 connection_token: u64,
168 reverse_request_registry_id: u64,
169}
170
171impl RouteHandle {
172 pub fn detached(channel: u16, epoch: u32) -> Self {
186 Self::new(channel, epoch, DETACHED_CONNECTION_TOKEN)
187 }
188
189 pub(crate) fn new(channel: u16, epoch: u32, connection_token: u64) -> Self {
190 Self {
191 channel,
192 epoch,
193 connection_token,
194 reverse_request_registry_id: 0,
195 }
196 }
197
198 pub(crate) fn new_consumer(
199 channel: u16,
200 epoch: u32,
201 connection_token: u64,
202 reverse_requests: ReverseRequestRegistry,
203 ) -> Self {
204 reverse_requests.seal();
205 Self {
206 channel,
207 epoch,
208 connection_token,
209 reverse_request_registry_id: consumer::install_reverse_request_registry(
210 reverse_requests,
211 ),
212 }
213 }
214
215 pub(crate) fn connection_token(self) -> u64 {
216 self.connection_token
217 }
218
219 pub fn on_request<F, Fut>(
220 &self,
221 method_family: impl Into<String>,
222 handler: F,
223 ) -> Result<(), ReverseRequestRegistrationError>
224 where
225 F: Fn(Vec<u8>, ReverseRequestContext) -> Fut + Send + Sync + 'static,
226 Fut: Future<Output = Vec<u8>> + Send + 'static,
227 {
228 consumer::route_reverse_request_registry(self.reverse_request_registry_id)?
229 .on_request(method_family, handler)
230 }
231
232 pub fn on_request_fallible<F, Fut>(
233 &self,
234 method_family: impl Into<String>,
235 handler: F,
236 ) -> Result<(), ReverseRequestRegistrationError>
237 where
238 F: Fn(Vec<u8>, ReverseRequestContext) -> Fut + Send + Sync + 'static,
239 Fut: Future<Output = Result<Vec<u8>, ReverseRequestError>> + Send + 'static,
240 {
241 consumer::route_reverse_request_registry(self.reverse_request_registry_id)?
242 .on_request_fallible(method_family, handler)
243 }
244
245 pub(crate) fn reverse_request_registry_id(self) -> u64 {
246 self.reverse_request_registry_id
247 }
248}
249
250impl PartialEq for RouteHandle {
251 fn eq(&self, other: &Self) -> bool {
252 self.channel == other.channel
253 && self.epoch == other.epoch
254 && self.connection_token == other.connection_token
255 }
256}
257
258impl Eq for RouteHandle {}
259
260impl std::hash::Hash for RouteHandle {
261 fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
262 std::hash::Hash::hash(&self.channel, state);
263 std::hash::Hash::hash(&self.epoch, state);
264 std::hash::Hash::hash(&self.connection_token, state);
265 }
266}
267
268impl fmt::Debug for RouteHandle {
269 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
270 f.debug_struct("RouteHandle")
271 .field("channel", &self.channel)
272 .field("epoch", &self.epoch)
273 .finish_non_exhaustive()
274 }
275}
276#[derive(Debug, Clone, PartialEq, Eq)]
278pub struct LiveRootsSnapshot {
279 pub roots: Vec<subc_protocol::session::LiveRoot>,
280 pub unknown_root_bindings: u64,
281 pub total_bindings: u64,
282}
283
284#[derive(Debug, Clone, PartialEq, Eq)]
289#[non_exhaustive]
290pub struct ScopeSyncReply {
291 pub generation: u64,
293 pub results: Vec<ScopeRecordResult>,
297 pub ended: Vec<ScopeEnded>,
300}
301
302#[derive(Debug, Clone, PartialEq, Eq)]
306#[non_exhaustive]
307pub struct ScopeApplyReply {
308 pub generation: u64,
310 pub results: Vec<ScopeRecordResult>,
312 pub end_results: Vec<ScopeEndResult>,
315 pub ended: Vec<ScopeEnded>,
318}
319
320#[derive(Debug, Clone, PartialEq, Eq)]
323#[non_exhaustive]
324pub struct ScopeDescribeReply {
325 pub status: ScopeStatus,
326 pub scope_epoch: Option<u64>,
328 pub daemon_incarnation: String,
331 pub owner_synced: bool,
333 pub owner_configured: bool,
335 pub scope: Option<ScopeStamp>,
339}
340
341#[derive(Debug, serde::Deserialize)]
344#[serde(untagged)]
345enum ModuleControlReply {
346 Control(Box<ModuleControlResponseToModule>),
347 RouteAuthorize(RouteAuthorizeResponse),
348}
349
350impl ModuleControlReply {
351 fn control(self) -> Result<ModuleControlResponseToModule, CatalogUpdateError> {
352 match self {
353 Self::Control(reply) => Ok(*reply),
354 other => Err(CatalogUpdateError::Protocol(format!(
355 "unexpected module control response: {other:?}"
356 ))),
357 }
358 }
359}
360
361type CatalogUpdateReply = oneshot::Sender<Result<ModuleControlReply, CatalogUpdateError>>;
362type CatalogUpdateWaiter = oneshot::Receiver<Result<ModuleControlReply, CatalogUpdateError>>;
363type CatalogUpdateRequest = (u64, mpsc::Sender<Frame>, CatalogUpdateWaiter);
364type OperatorConfirmRequestWaiter = (u64, mpsc::Sender<Frame>, oneshot::Receiver<Frame>);
365
366pub type ModuleServeFuture = Pin<Box<dyn Future<Output = Result<(), SubcModuleError>> + Send>>;
368
369#[derive(Clone)]
370struct RequestDispatcher {
371 in_flight: InFlight,
372 control_slots: Arc<tokio::sync::Semaphore>,
373 pools: Arc<DispatchPools>,
376 draining_hooks: Arc<Mutex<Vec<oneshot::Receiver<()>>>>,
381 undecodable_push_logged: Arc<AtomicBool>,
384 failure: Arc<Mutex<Option<SubcModuleError>>>,
385 failed: CancellationToken,
386 tasks: TaskTracker,
387 aborts: Arc<Mutex<Vec<tokio::task::AbortHandle>>>,
388 undelivered_replies: Arc<AtomicU64>,
389}
390
391impl RequestDispatcher {
392 #[cfg(test)]
395 fn new() -> Self {
396 Self::with_pools(Arc::new(DispatchPools::new(RequestPools::new())))
397 }
398
399 fn with_pools(pools: Arc<DispatchPools>) -> Self {
400 Self {
401 in_flight: Arc::new(Mutex::new(HashMap::new())),
402 control_slots: Arc::new(tokio::sync::Semaphore::new(16)),
403 pools,
404 draining_hooks: Arc::new(Mutex::new(Vec::new())),
405 undecodable_push_logged: Arc::new(AtomicBool::new(false)),
406 failure: Arc::new(Mutex::new(None)),
407 failed: CancellationToken::new(),
408 tasks: TaskTracker::new(),
409 aborts: Arc::new(Mutex::new(Vec::new())),
410 undelivered_replies: Arc::new(AtomicU64::new(0)),
411 }
412 }
413
414 fn fail(&self, error: SubcModuleError) {
415 let mut failure = self.failure.lock().unwrap_or_else(|e| e.into_inner());
416 if failure.is_none() {
417 *failure = Some(error);
418 }
419 self.failed.cancel();
420 }
421
422 fn take_failure(&self) -> Option<SubcModuleError> {
423 self.failure
424 .lock()
425 .unwrap_or_else(|e| e.into_inner())
426 .take()
427 }
428
429 fn spawn(&self, future: impl Future<Output = ()> + Send + 'static) {
430 let task = self.tasks.spawn(future);
431 let mut aborts = self.aborts.lock().unwrap_or_else(|e| e.into_inner());
432 aborts.retain(|task| !task.is_finished());
433 aborts.push(task.abort_handle());
434 }
435
436 fn abort_all(&self) {
437 for task in self
438 .aborts
439 .lock()
440 .unwrap_or_else(|e| e.into_inner())
441 .drain(..)
442 {
443 task.abort();
444 }
445 }
446
447 fn fold_health(&self, mut report: HealthReport) -> HealthReport {
448 if let Some(saturation) = self.pools.saturation() {
449 if report.status == HealthStatus::Ok {
450 report.status = HealthStatus::Degraded;
451 }
452 report.detail = Some(match report.detail {
453 Some(detail) => format!("{saturation}; {detail}"),
454 None => saturation,
455 });
456 }
457 report
458 }
459
460 async fn wait_for_draining_hooks_to_start(&self) {
464 let started = std::mem::take(
465 &mut *self
466 .draining_hooks
467 .lock()
468 .unwrap_or_else(|poisoned| poisoned.into_inner()),
469 );
470 let deadline = tokio::time::Instant::now() + DRAINING_HOOK_START_LIMIT;
471 for hook in started {
472 if tokio::time::timeout_at(deadline, hook).await.is_err() {
475 return;
476 }
477 }
478 }
479
480 fn cancel_all(&self) {
485 self.pools.close();
486 let cancelled = match self.in_flight.lock() {
487 Ok(guard) => guard.values().cloned().collect::<Vec<_>>(),
488 Err(poisoned) => poisoned.into_inner().values().cloned().collect(),
489 };
490 for cancellation in cancelled {
491 cancellation.cancel();
492 }
493 }
494}
495
496#[derive(Clone)]
498pub struct ModuleHandle {
499 shared: Arc<ModuleHandleShared>,
500}
501
502struct ModuleHandleShared {
503 registration: registration::Session,
504 negotiated_ver: u8,
505 daemon_capabilities: Vec<String>,
506 supports_catalog_update: bool,
507 supports_role_declarations_update: bool,
508 supports_live_roots: bool,
509 supports_agent_commands: bool,
510 supports_scope_sync: bool,
511 supports_scope_apply: bool,
512 supports_scope_describe: bool,
513 supports_scope_watch: bool,
514 supports_route_authorize: bool,
515 supports_operator_confirm: bool,
516 supports_activity: bool,
517 connection_runtime: tokio::runtime::Handle,
518 connection_token: u64,
519 machine_id: Option<MachineId>,
520 live_routes: Mutex<HashMap<u16, RouteHandle>>,
521 dropped_route_frames: AtomicU64,
522 close_token: CancellationToken,
523 dispatch_pools: Mutex<Arc<DispatchPools>>,
526 catalog_send: tokio::sync::Mutex<()>,
527 config: config::Session,
528 served_control_ops: std::sync::OnceLock<Vec<String>>,
529 inner: Mutex<ModuleHandleState>,
530}
531
532struct ModuleHandleState {
533 pending_control_streams: HashMap<u64, module_subscription::ControlStreamSender>,
534 pending_claim_calls: HashMap<u64, oneshot::Sender<Frame>>,
535 writer: Option<mpsc::Sender<Frame>>,
536 next_corr: Option<u64>,
537 pending_catalog_updates: HashMap<u64, CatalogUpdateReply>,
538 pending_operator_confirms: HashMap<u64, oneshot::Sender<Frame>>,
539 pending_activity_records: HashMap<u64, oneshot::Sender<Frame>>,
540 pending_scope_served_by: HashMap<u64, oneshot::Sender<Frame>>,
541 pending_config_calls: HashMap<u64, oneshot::Sender<Frame>>,
542 pending_agent_commands: HashMap<u64, oneshot::Sender<Frame>>,
543 current_provides: Option<Vec<ProviderRole>>,
544 catalog_changes: VecDeque<CatalogChange>,
545 closed: bool,
546}
547
548struct CatalogChange {
549 corr: u64,
550 provides: Vec<ProviderRole>,
551 outcome: Option<CatalogChangeOutcome>,
552}
553
554enum CatalogChangeOutcome {
555 Accepted,
556 Rejected,
557 Unknown,
558}
559
560struct CatalogUpdateGuard {
561 shared: Arc<ModuleHandleShared>,
562 corr: u64,
563}
564
565impl Drop for CatalogUpdateGuard {
566 fn drop(&mut self) {
567 self.shared.remove_pending_catalog_update(self.corr);
568 }
569}
570
571impl ModuleHandle {
572 fn new(
573 ack: &ModuleHelloAckBody,
574 writer: mpsc::Sender<Frame>,
575 connection_token: u64,
576 close_token: CancellationToken,
577 connection_runtime: tokio::runtime::Handle,
578 ) -> Self {
579 Self {
580 shared: Arc::new(ModuleHandleShared {
581 registration: registration::Session::default(),
582 negotiated_ver: ack.negotiated_ver,
583 daemon_capabilities: ack.subc_capabilities.clone(),
584 supports_catalog_update: ack
585 .subc_ops
586 .iter()
587 .any(|op| op == MODULE_TO_SUBC_OP_CATALOG_UPDATE),
588 supports_role_declarations_update: ack
589 .subc_ops
590 .iter()
591 .any(|op| op == MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS),
592 supports_live_roots: ack.subc_ops.iter().any(|op| op == "supervisor.live_roots"),
593 supports_agent_commands: ack
594 .subc_ops
595 .iter()
596 .any(|op| op == "supervisor.agent_commands"),
597 supports_scope_sync: ack.subc_ops.iter().any(|op| op == SCOPE_SYNC_OP),
598 supports_scope_apply: ack.subc_ops.iter().any(|op| op == SCOPE_APPLY_OP),
599 supports_scope_describe: ack.subc_ops.iter().any(|op| op == SCOPE_DESCRIBE_OP),
600 supports_scope_watch: ack
601 .subc_ops
602 .iter()
603 .any(|op| op == subc_control::ops::SCOPE_WATCH),
604 supports_route_authorize: ack
605 .subc_ops
606 .iter()
607 .any(|op| op == subc_protocol::scope::ROUTE_AUTHORIZE_OP),
608 supports_operator_confirm: ack.subc_ops.iter().any(|op| op == OPERATOR_CONFIRM_OP),
609 supports_activity: ack
610 .subc_capabilities
611 .iter()
612 .any(|capability| capability == ACTIVITY_LEDGER_CAPABILITY),
613 connection_runtime,
614 connection_token,
615 machine_id: ack
618 .machine_id
619 .as_deref()
620 .and_then(|id| MachineId::parse(id).ok()),
621 live_routes: Mutex::new(HashMap::new()),
622 dropped_route_frames: AtomicU64::new(0),
623 close_token,
624 dispatch_pools: Mutex::new(Arc::new(DispatchPools::new(RequestPools::new()))),
625 catalog_send: tokio::sync::Mutex::new(()),
626 config: config::Session::default(),
627 served_control_ops: std::sync::OnceLock::new(),
628 inner: Mutex::new(ModuleHandleState {
629 pending_control_streams: HashMap::new(),
630 pending_claim_calls: HashMap::new(),
631 writer: Some(writer),
632 next_corr: Some(HELLO_CORR + 1),
633 pending_catalog_updates: HashMap::new(),
634 pending_operator_confirms: HashMap::new(),
635 pending_activity_records: HashMap::new(),
636 pending_scope_served_by: HashMap::new(),
637 pending_config_calls: HashMap::new(),
638 pending_agent_commands: HashMap::new(),
639 current_provides: None,
640 catalog_changes: VecDeque::new(),
641 closed: false,
642 }),
643 }),
644 }
645 }
646
647 pub fn machine_id(&self) -> Option<&MachineId> {
654 self.shared.machine_id.as_ref()
655 }
656
657 pub fn daemon_capabilities(&self) -> &[String] {
661 &self.shared.daemon_capabilities
662 }
663
664 pub fn daemon_supports(&self, capability: &str) -> bool {
671 self.shared
672 .daemon_capabilities
673 .iter()
674 .any(|advertised| advertised == capability)
675 }
676
677 pub fn closed(&self) -> impl Future<Output = ()> + Send + 'static {
681 self.shared.close_token.clone().cancelled_owned()
682 }
683
684 pub fn is_closed(&self) -> bool {
686 self.shared.close_token.is_cancelled()
687 }
688
689 pub async fn catalog_update(
694 &self,
695 provides: Vec<ProviderRole>,
696 ) -> Result<(), CatalogUpdateError> {
697 self.catalog_update_inner(Some(provides), None, None).await
698 }
699
700 pub async fn update_role_declarations(
708 &self,
709 role_declarations: BTreeMap<String, serde_json::Map<String, serde_json::Value>>,
710 ) -> Result<(), CatalogUpdateError> {
711 if !self.shared.supports_role_declarations_update {
712 return Err(CatalogUpdateError::NotSupported);
713 }
714 let body = serde_json::to_vec(
715 &ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations {
716 role_declarations: serde_json::to_value(role_declarations)
717 .map_err(|err| CatalogUpdateError::Protocol(err.to_string()))?,
718 },
719 )
720 .map_err(|err| CatalogUpdateError::Protocol(err.to_string()))?;
721 let (corr, writer, rx) = self.shared.begin_catalog_update()?;
722 let _pending = CatalogUpdateGuard {
723 shared: self.shared.clone(),
724 corr,
725 };
726 let frame = Frame::build_with_version(
727 self.shared.negotiated_ver,
728 FrameType::Request,
729 control_flags(),
730 0,
731 0,
732 corr,
733 body,
734 )
735 .map_err(|err| CatalogUpdateError::Protocol(err.to_string()))?;
736 if writer.send(frame).await.is_err() {
737 return Err(CatalogUpdateError::ConnectionClosed);
738 }
739 match timeout(CATALOG_UPDATE_TIMEOUT, rx).await {
740 Ok(Ok(Ok(reply))) => match reply.control()? {
741 ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {} => Ok(()),
742 _ => Err(CatalogUpdateError::Protocol(
743 "unexpected catalog.update_role_declarations response".into(),
744 )),
745 },
746 Ok(Ok(Err(err))) => Err(err),
747 Ok(Err(_)) => Err(CatalogUpdateError::ConnectionClosed),
748 Err(_) => Err(CatalogUpdateError::Timeout),
749 }
750 }
751
752 pub async fn catalog_update_with_capabilities(
758 &self,
759 provides: Vec<ProviderRole>,
760 capabilities: CapabilityDeclarations,
761 ) -> Result<(), CatalogUpdateError> {
762 self.catalog_update_inner(Some(provides), Some(capabilities), None)
763 .await
764 }
765
766 pub async fn set_ready(&self, ready: bool) -> Result<(), CatalogUpdateError> {
778 self.catalog_update_inner(None, None, Some(ready)).await
779 }
780
781 async fn catalog_update_inner(
782 &self,
783 provides: Option<Vec<ProviderRole>>,
784 capabilities: Option<CapabilityDeclarations>,
785 ready: Option<bool>,
786 ) -> Result<(), CatalogUpdateError> {
787 if !self.shared.supports_catalog_update {
788 return Err(CatalogUpdateError::NotSupported);
789 }
790
791 let send_guard = self.shared.catalog_send.lock().await;
794 let provides = match provides {
795 Some(provides) => provides,
796 None => {
797 let inner = self.shared.lock_inner();
798 if inner.closed {
799 return Err(CatalogUpdateError::ConnectionClosed);
800 }
801 inner
802 .current_provides
803 .as_ref()
804 .filter(|_| inner.catalog_changes.is_empty())
805 .cloned()
806 .ok_or_else(|| {
807 CatalogUpdateError::Protocol(
808 "current provider roles are unknown or a catalog update is in flight"
809 .into(),
810 )
811 })?
812 }
813 };
814 let body = serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
815 provides: provides.clone(),
816 capabilities,
817 ready,
818 })
819 .map_err(|err| {
820 CatalogUpdateError::Protocol(format!(
821 "failed to encode catalog.update request body: {err}"
822 ))
823 })?;
824 let (corr, writer, rx) = self.shared.begin_catalog_update()?;
825 let _pending = CatalogUpdateGuard {
826 shared: self.shared.clone(),
827 corr,
828 };
829 let frame = Frame::build_with_version(
830 self.shared.negotiated_ver,
831 FrameType::Request,
832 control_flags(),
833 0,
834 0,
835 corr,
836 body,
837 )
838 .map_err(|err| {
839 self.shared.remove_pending_catalog_update(corr);
840 CatalogUpdateError::Protocol(format!(
841 "failed to build catalog.update request frame: {err}"
842 ))
843 })?;
844
845 let permit = match writer.reserve().await {
846 Ok(permit) => permit,
847 Err(_) => {
848 self.shared.remove_pending_catalog_update(corr);
849 return Err(CatalogUpdateError::ConnectionClosed);
850 }
851 };
852 self.shared
853 .lock_inner()
854 .catalog_changes
855 .push_back(CatalogChange {
856 corr,
857 provides,
858 outcome: None,
859 });
860 permit.send(frame);
861 drop(send_guard);
862
863 match timeout(CATALOG_UPDATE_TIMEOUT, async {
864 rx.await
865 .map(|reply| reply.and_then(ModuleControlReply::control))
866 })
867 .await
868 {
869 Ok(Ok(Ok(ModuleControlResponseToModule::CatalogUpdate {}))) => Ok(()),
870 Ok(Ok(Ok(_))) => Err(CatalogUpdateError::Protocol(
871 "unexpected catalog.update response".into(),
872 )),
873 Ok(Ok(Err(err))) => Err(err),
874 Ok(Err(_)) => Err(CatalogUpdateError::ConnectionClosed),
875 Err(_) => {
876 self.shared.remove_pending_catalog_update(corr);
877 Err(CatalogUpdateError::Timeout)
878 }
879 }
880 }
881
882 pub async fn live_roots(&self) -> Result<LiveRootsSnapshot, CatalogUpdateError> {
884 if !self.shared.supports_live_roots {
885 return Err(CatalogUpdateError::NotSupported);
886 }
887 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {})
888 .map_err(|err| CatalogUpdateError::Protocol(err.to_string()))?;
889 let (corr, writer, rx) = self.shared.begin_catalog_update()?;
890 let frame = Frame::build_with_version(
891 self.shared.negotiated_ver,
892 FrameType::Request,
893 control_flags(),
894 0,
895 0,
896 corr,
897 body,
898 )
899 .map_err(|err| {
900 self.shared.remove_pending_catalog_update(corr);
901 CatalogUpdateError::Protocol(err.to_string())
902 })?;
903 if writer.send(frame).await.is_err() {
904 self.shared.remove_pending_catalog_update(corr);
905 return Err(CatalogUpdateError::ConnectionClosed);
906 }
907 match timeout(CATALOG_UPDATE_TIMEOUT, async {
908 rx.await
909 .map(|reply| reply.and_then(ModuleControlReply::control))
910 })
911 .await
912 {
913 Ok(Ok(Ok(ModuleControlResponseToModule::LiveRoots {
914 roots,
915 unknown_root_bindings,
916 total_bindings,
917 }))) => Ok(LiveRootsSnapshot {
918 roots,
919 unknown_root_bindings,
920 total_bindings,
921 }),
922 Ok(Ok(Ok(_))) => Err(CatalogUpdateError::Protocol(
923 "unexpected live_roots response".into(),
924 )),
925 Ok(Ok(Err(err))) => Err(err),
926 Ok(Err(_)) => Err(CatalogUpdateError::ConnectionClosed),
927 Err(_) => {
928 self.shared.remove_pending_catalog_update(corr);
929 Err(CatalogUpdateError::Timeout)
930 }
931 }
932 }
933
934 pub async fn scope_sync(
950 &self,
951 generation: u64,
952 scopes: Vec<ScopeRecord>,
953 ) -> Result<ScopeSyncReply, ScopeCallError> {
954 if !self.shared.supports_scope_sync {
955 return Err(ScopeCallError::NotSupported { op: SCOPE_SYNC_OP });
956 }
957 let request = ModuleControlRequestFromModule::ScopeSync { generation, scopes };
958 match self
959 .scope_call(SCOPE_SYNC_OP, &request)
960 .await?
961 .control()
962 .map_err(ScopeCallError::from_control_error)?
963 {
964 ModuleControlResponseToModule::ScopeSync {
965 generation,
966 results,
967 ended,
968 } => Ok(ScopeSyncReply {
969 generation,
970 results,
971 ended,
972 }),
973 other => Err(ScopeCallError::Protocol(format!(
974 "unexpected {SCOPE_SYNC_OP} response: {other:?}"
975 ))),
976 }
977 }
978
979 pub async fn scope_apply(
990 &self,
991 generation: u64,
992 upsert: Vec<ScopeRecord>,
993 end: Vec<ScopeEnd>,
994 ) -> Result<ScopeApplyReply, ScopeCallError> {
995 if !self.shared.supports_scope_apply {
996 return Err(ScopeCallError::NotSupported { op: SCOPE_APPLY_OP });
997 }
998 let request = ModuleControlRequestFromModule::ScopeApply {
999 generation,
1000 upsert,
1001 end,
1002 };
1003 match self
1004 .scope_call(SCOPE_APPLY_OP, &request)
1005 .await?
1006 .control()
1007 .map_err(ScopeCallError::from_control_error)?
1008 {
1009 ModuleControlResponseToModule::ScopeApply {
1010 generation,
1011 results,
1012 end_results,
1013 ended,
1014 } => Ok(ScopeApplyReply {
1015 generation,
1016 results,
1017 end_results,
1018 ended,
1019 }),
1020 other => Err(ScopeCallError::Protocol(format!(
1021 "unexpected {SCOPE_APPLY_OP} response: {other:?}"
1022 ))),
1023 }
1024 }
1025
1026 pub async fn scope_describe(
1034 &self,
1035 owner: Principal,
1036 scope_ref: String,
1037 ) -> Result<ScopeDescribeReply, ScopeCallError> {
1038 if !self.shared.supports_scope_describe {
1039 return Err(ScopeCallError::NotSupported {
1040 op: SCOPE_DESCRIBE_OP,
1041 });
1042 }
1043 let request = ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref };
1044 match self
1045 .scope_call(SCOPE_DESCRIBE_OP, &request)
1046 .await?
1047 .control()
1048 .map_err(ScopeCallError::from_control_error)?
1049 {
1050 ModuleControlResponseToModule::ScopeDescribe {
1051 status,
1052 scope_epoch,
1053 daemon_incarnation,
1054 owner_synced,
1055 owner_configured,
1056 scope,
1057 } => Ok(ScopeDescribeReply {
1058 status,
1059 scope_epoch,
1060 daemon_incarnation,
1061 owner_synced,
1062 owner_configured,
1063 scope,
1064 }),
1065 other => Err(ScopeCallError::Protocol(format!(
1066 "unexpected {SCOPE_DESCRIBE_OP} response: {other:?}"
1067 ))),
1068 }
1069 }
1070
1071 pub async fn confirm_operator(
1083 &self,
1084 summary: &str,
1085 caller_route: &RouteHandle,
1086 ) -> Result<(), OperatorConfirmError> {
1087 self.confirm_operator_with_deadline(summary, caller_route, OPERATOR_CONFIRM_TIMEOUT)
1088 .await
1089 }
1090
1091 pub async fn confirm_operation(
1112 &self,
1113 summary: &str,
1114 caller_route: &RouteHandle,
1115 operation: &str,
1116 ) -> Result<(), OperatorConfirmError> {
1117 if operation.is_empty() {
1118 return Err(OperatorConfirmError::OperationInvalid);
1119 }
1120 self.confirm_operation_with_deadline(
1121 summary,
1122 caller_route,
1123 operation,
1124 OPERATOR_CONFIRM_TIMEOUT,
1125 )
1126 .await
1127 }
1128
1129 pub async fn confirm_enrollment(
1147 &self,
1148 summary: &str,
1149 caller_route: &RouteHandle,
1150 ) -> Result<(), OperatorConfirmError> {
1151 self.confirm_inner(
1152 summary,
1153 caller_route,
1154 Some(OperatorConfirmKind::Enrollment),
1155 None,
1156 OPERATOR_CONFIRM_TIMEOUT,
1157 )
1158 .await
1159 }
1160
1161 pub(crate) async fn confirm_operator_with_deadline(
1162 &self,
1163 summary: &str,
1164 caller_route: &RouteHandle,
1165 duration: Duration,
1166 ) -> Result<(), OperatorConfirmError> {
1167 self.confirm_inner(summary, caller_route, None, None, duration)
1168 .await
1169 }
1170
1171 async fn confirm_operation_with_deadline(
1172 &self,
1173 summary: &str,
1174 caller_route: &RouteHandle,
1175 operation: &str,
1176 duration: Duration,
1177 ) -> Result<(), OperatorConfirmError> {
1178 self.confirm_inner(
1179 summary,
1180 caller_route,
1181 Some(OperatorConfirmKind::Operation),
1182 Some(operation.to_owned()),
1183 duration,
1184 )
1185 .await
1186 }
1187
1188 async fn confirm_inner(
1189 &self,
1190 summary: &str,
1191 caller_route: &RouteHandle,
1192 kind: Option<OperatorConfirmKind>,
1193 operation: Option<String>,
1194 duration: Duration,
1195 ) -> Result<(), OperatorConfirmError> {
1196 if caller_route.connection_token() != self.shared.connection_token {
1197 return Err(OperatorConfirmError::NotPermitted);
1198 }
1199 if !self.shared.supports_operator_confirm {
1200 return Err(OperatorConfirmError::Unsupported);
1201 }
1202 let deadline = Instant::now() + duration;
1205 let body = serde_json::to_vec(
1206 &OperatorConfirmRequest::new(summary, caller_route.channel, caller_route.epoch)
1207 .with_kind(kind)
1208 .with_operation(operation),
1209 )
1210 .map_err(|_| OperatorConfirmError::PresenceUnavailable)?;
1211 let (corr, writer, rx) = self.shared.begin_operator_confirm()?;
1212 let mut guard = OperatorConfirmGuard {
1213 shared: Arc::clone(&self.shared),
1214 runtime: self.shared.connection_runtime.clone(),
1215 writer: writer.clone(),
1216 corr,
1217 accepted: false,
1218 armed: true,
1219 };
1220 let frame = Frame::build_with_version(
1221 self.shared.negotiated_ver,
1222 FrameType::Request,
1223 control_flags(),
1224 0,
1225 0,
1226 corr,
1227 body,
1228 )
1229 .map_err(|_| OperatorConfirmError::PresenceUnavailable)?;
1230 tokio::select! {
1231 result = timeout_at(deadline, writer.send(frame)) => {
1232 result.map_err(|_| OperatorConfirmError::PresenceUnavailable)?
1233 .map_err(|_| OperatorConfirmError::PresenceUnavailable)?;
1234 }
1235 _ = self.shared.close_token.cancelled() => {
1236 return Err(OperatorConfirmError::PresenceUnavailable);
1237 }
1238 }
1239 guard.accepted = true;
1240 let reply = timeout_at(deadline, rx)
1241 .await
1242 .map_err(|_| OperatorConfirmError::PresenceUnavailable)?
1243 .map_err(|_| OperatorConfirmError::PresenceUnavailable)?;
1244 guard.armed = false;
1245 match reply.header.ty {
1246 FrameType::Response => {
1247 match serde_json::from_slice::<OperatorConfirmReply>(&reply.body) {
1248 Ok(body) if body.outcome == "confirmed" => Ok(()),
1249 _ => Err(OperatorConfirmError::PresenceUnavailable),
1250 }
1251 }
1252 FrameType::Error => match serde_json::from_slice::<ErrorBody>(&reply.body) {
1253 Ok(body) => Err(match body.code.as_str() {
1254 error_codes::OPERATOR_DECLINED => {
1255 let detail = body.detail.as_ref();
1256 if detail
1257 .and_then(|detail| detail.get("reason"))
1258 .and_then(serde_json::Value::as_str)
1259 == Some("backoff")
1260 {
1261 OperatorConfirmError::Backoff {
1262 retry_after_ms: detail
1263 .and_then(|detail| detail.get("retry_after_ms"))
1264 .and_then(serde_json::Value::as_u64),
1265 }
1266 } else {
1267 OperatorConfirmError::Declined
1268 }
1269 }
1270 error_codes::OPERATOR_PRESENCE_UNAVAILABLE
1271 if body
1272 .detail
1273 .as_ref()
1274 .and_then(|detail| detail.get("reason"))
1275 .and_then(serde_json::Value::as_str)
1276 == Some("unsupported_platform") =>
1277 {
1278 OperatorConfirmError::PresenceUnsupportedPlatform
1279 }
1280 error_codes::OPERATOR_SUMMARY_INVALID => OperatorConfirmError::SummaryInvalid,
1281 error_codes::OPERATOR_REQUEST_NOT_PERMITTED | "not_registered" => {
1282 OperatorConfirmError::NotPermitted
1283 }
1284 "unsupported_control_frame" => OperatorConfirmError::Unsupported,
1285 _ => OperatorConfirmError::PresenceUnavailable,
1286 }),
1287 Err(_) => Err(OperatorConfirmError::PresenceUnavailable),
1288 },
1289 _ => Err(OperatorConfirmError::PresenceUnavailable),
1290 }
1291 }
1292
1293 async fn scope_call(
1297 &self,
1298 op: &'static str,
1299 request: &ModuleControlRequestFromModule,
1300 ) -> Result<ModuleControlReply, ScopeCallError> {
1301 let body = serde_json::to_vec(request).map_err(|err| {
1302 ScopeCallError::Protocol(format!("failed to encode {op} request body: {err}"))
1303 })?;
1304 let (corr, writer, rx) = self
1305 .shared
1306 .begin_catalog_update()
1307 .map_err(ScopeCallError::from_control_error)?;
1308 let frame = Frame::build_with_version(
1309 self.shared.negotiated_ver,
1310 FrameType::Request,
1311 control_flags(),
1312 0,
1313 0,
1314 corr,
1315 body,
1316 )
1317 .map_err(|err| {
1318 self.shared.remove_pending_catalog_update(corr);
1319 ScopeCallError::Protocol(format!("failed to build {op} request frame: {err}"))
1320 })?;
1321 if writer.send(frame).await.is_err() {
1322 self.shared.remove_pending_catalog_update(corr);
1323 return Err(ScopeCallError::ConnectionClosed);
1324 }
1325 match timeout(CATALOG_UPDATE_TIMEOUT, rx).await {
1326 Ok(Ok(reply)) => reply.map_err(ScopeCallError::from_control_error),
1327 Ok(Err(_)) => Err(ScopeCallError::ConnectionClosed),
1328 Err(_) => {
1329 self.shared.remove_pending_catalog_update(corr);
1330 Err(ScopeCallError::Timeout)
1331 }
1332 }
1333 }
1334
1335 pub async fn push(
1337 &self,
1338 handle: &RouteHandle,
1339 body: Vec<u8>,
1340 admission_class: Option<AdmissionClass>,
1341 ) -> Result<(), SubcModuleError> {
1342 self.validate_route(*handle)?;
1343 let writer = self
1344 .shared
1345 .lock_inner()
1346 .writer
1347 .clone()
1348 .ok_or(SubcModuleError::WriterClosed)?;
1349 let frame = Frame::build_with_version(
1350 self.shared.negotiated_ver,
1351 FrameType::Push,
1352 data_flags().with_admission_class(admission_class.unwrap_or(AdmissionClass::Normal)),
1353 handle.channel,
1354 handle.epoch,
1355 0,
1356 body,
1357 )
1358 .map_err(SubcModuleError::FrameBuild)?;
1359 send_outbound(&writer, frame).await
1360 }
1361
1362 pub fn dropped_route_frames(&self) -> u64 {
1364 self.shared.dropped_route_frames.load(Ordering::Relaxed)
1365 }
1366
1367 fn validate_route(&self, handle: RouteHandle) -> Result<(), SubcModuleError> {
1368 if handle.connection_token() != self.shared.connection_token {
1369 return Err(SubcModuleError::StaleRouteHandle(handle));
1370 }
1371 let routes = self
1372 .shared
1373 .live_routes
1374 .lock()
1375 .map_err(|_| SubcModuleError::InFlightPoisoned)?;
1376 if routes.get(&handle.channel) == Some(&handle) {
1377 Ok(())
1378 } else {
1379 Err(SubcModuleError::StaleRouteHandle(handle))
1380 }
1381 }
1382
1383 fn install_route(&self, handle: RouteHandle) -> Result<(), SubcModuleError> {
1384 self.shared
1385 .live_routes
1386 .lock()
1387 .map_err(|_| SubcModuleError::InFlightPoisoned)?
1388 .insert(handle.channel, handle);
1389 Ok(())
1390 }
1391
1392 fn installed_route(&self, channel: u16) -> Result<Option<RouteHandle>, SubcModuleError> {
1393 Ok(self
1394 .shared
1395 .live_routes
1396 .lock()
1397 .map_err(|_| SubcModuleError::InFlightPoisoned)?
1398 .get(&channel)
1399 .copied())
1400 }
1401
1402 fn remove_route(&self, handle: RouteHandle) -> Result<bool, SubcModuleError> {
1403 let mut routes = self
1404 .shared
1405 .live_routes
1406 .lock()
1407 .map_err(|_| SubcModuleError::InFlightPoisoned)?;
1408 if routes.get(&handle.channel) == Some(&handle) {
1409 routes.remove(&handle.channel);
1410 Ok(true)
1411 } else {
1412 Ok(false)
1413 }
1414 }
1415
1416 fn validate_ingress(&self, channel: u16, epoch: u32) -> Result<bool, SubcModuleError> {
1417 let handle = RouteHandle::new(channel, epoch, self.shared.connection_token);
1418 let valid = self
1419 .shared
1420 .live_routes
1421 .lock()
1422 .map_err(|_| SubcModuleError::InFlightPoisoned)?
1423 .get(&channel)
1424 == Some(&handle);
1425 if !valid {
1426 self.shared
1427 .dropped_route_frames
1428 .fetch_add(1, Ordering::Relaxed);
1429 }
1430 Ok(valid)
1431 }
1432
1433 fn route_handle(&self, channel: u16, epoch: u32) -> RouteHandle {
1434 RouteHandle::new(channel, epoch, self.shared.connection_token)
1435 }
1436
1437 fn handle_control_reply(&self, frame: Frame) -> bool {
1438 let Some(frame) = self.handle_control_stream_frame(frame) else {
1439 return true;
1440 };
1441 let claim_reply = self
1442 .shared
1443 .lock_inner()
1444 .pending_claim_calls
1445 .remove(&frame.header.corr);
1446 if let Some(reply) = claim_reply {
1447 let _ = reply.send(frame);
1448 return true;
1449 }
1450 let config_reply = self
1451 .shared
1452 .lock_inner()
1453 .pending_config_calls
1454 .remove(&frame.header.corr);
1455 if let Some(reply) = config_reply {
1456 let _ = reply.send(frame);
1457 return true;
1458 }
1459 if let Some(reply) = self
1460 .shared
1461 .lock_inner()
1462 .pending_agent_commands
1463 .remove(&frame.header.corr)
1464 {
1465 let _ = reply.send(frame);
1466 return true;
1467 }
1468 let served_by_reply = self
1469 .shared
1470 .lock_inner()
1471 .pending_scope_served_by
1472 .remove(&frame.header.corr);
1473 if let Some(reply) = served_by_reply {
1474 let _ = reply.send(frame);
1475 return true;
1476 }
1477 let activity_reply = self
1478 .shared
1479 .lock_inner()
1480 .pending_activity_records
1481 .remove(&frame.header.corr);
1482 if let Some(reply) = activity_reply {
1483 let _ = reply.send(frame);
1484 return true;
1485 }
1486 let operator_reply = self
1487 .shared
1488 .lock_inner()
1489 .pending_operator_confirms
1490 .remove(&frame.header.corr);
1491 if let Some(reply) = operator_reply {
1492 let _ = reply.send(frame);
1493 return true;
1494 }
1495 let Some(reply) = self.shared.take_pending_catalog_update(frame.header.corr) else {
1496 return false;
1497 };
1498 let result = match frame.header.ty {
1499 FrameType::Response => serde_json::from_slice::<ModuleControlReply>(&frame.body)
1500 .map_err(|err| {
1501 CatalogUpdateError::Protocol(format!(
1502 "invalid module control response body: {err}"
1503 ))
1504 }),
1505 FrameType::Error => match serde_json::from_slice::<ErrorBody>(&frame.body) {
1506 Ok(body) => Err(match body.code.as_str() {
1507 "catalog_update_frozen_field" => CatalogUpdateError::FrozenField(body),
1508 "not_registered" => CatalogUpdateError::NotRegistered(body),
1509 _ => CatalogUpdateError::Rejected(body),
1510 }),
1511 Err(err) => Err(CatalogUpdateError::Protocol(format!(
1512 "invalid catalog.update error body: {err}"
1513 ))),
1514 },
1515 ty => Err(CatalogUpdateError::Protocol(format!(
1516 "unexpected catalog.update terminal frame: {ty:?}"
1517 ))),
1518 };
1519 let outcome = match &result {
1520 Ok(ModuleControlReply::Control(reply))
1521 if matches!(
1522 reply.as_ref(),
1523 ModuleControlResponseToModule::CatalogUpdate {}
1524 ) =>
1525 {
1526 CatalogChangeOutcome::Accepted
1527 }
1528 Err(CatalogUpdateError::FrozenField(_))
1529 | Err(CatalogUpdateError::NotRegistered(_))
1530 | Err(CatalogUpdateError::Rejected(_)) => CatalogChangeOutcome::Rejected,
1531 _ => CatalogChangeOutcome::Unknown,
1532 };
1533 self.shared
1534 .finish_catalog_change(frame.header.corr, outcome);
1535 let _ = reply.send(result);
1536 true
1537 }
1538
1539 fn close_connection(&self) {
1540 self.shared.close_connection();
1541 }
1542}
1543
1544impl ModuleHandleShared {
1545 fn begin_catalog_update(&self) -> Result<CatalogUpdateRequest, CatalogUpdateError> {
1546 let mut inner = self.lock_inner();
1547 if inner.closed {
1548 return Err(CatalogUpdateError::ConnectionClosed);
1549 }
1550 let Some(writer) = inner.writer.clone() else {
1551 drop(inner);
1552 self.close_connection();
1553 return Err(CatalogUpdateError::ConnectionClosed);
1554 };
1555 let Some(corr) = next_module_control_corr(&mut inner) else {
1556 drop(inner);
1557 self.close_connection();
1558 return Err(CatalogUpdateError::ConnectionClosed);
1559 };
1560 let (tx, rx) = oneshot::channel();
1561 inner.pending_catalog_updates.insert(corr, tx);
1562 Ok((corr, writer, rx))
1563 }
1564
1565 fn begin_operator_confirm(&self) -> Result<OperatorConfirmRequestWaiter, OperatorConfirmError> {
1566 let mut inner = self.lock_inner();
1567 if inner.closed {
1568 return Err(OperatorConfirmError::PresenceUnavailable);
1569 }
1570 let Some(writer) = inner.writer.clone() else {
1571 drop(inner);
1572 self.close_connection();
1573 return Err(OperatorConfirmError::PresenceUnavailable);
1574 };
1575 let Some(corr) = next_module_control_corr(&mut inner) else {
1576 drop(inner);
1577 self.close_connection();
1578 return Err(OperatorConfirmError::PresenceUnavailable);
1579 };
1580 let (tx, rx) = oneshot::channel();
1581 inner.pending_operator_confirms.insert(corr, tx);
1582 Ok((corr, writer, rx))
1583 }
1584
1585 fn take_pending_catalog_update(&self, corr: u64) -> Option<CatalogUpdateReply> {
1586 self.lock_inner().pending_catalog_updates.remove(&corr)
1587 }
1588
1589 fn remove_pending_catalog_update(&self, corr: u64) {
1590 let removed = self
1591 .lock_inner()
1592 .pending_catalog_updates
1593 .remove(&corr)
1594 .is_some();
1595 if removed {
1596 self.finish_catalog_change(corr, CatalogChangeOutcome::Unknown);
1597 }
1598 }
1599
1600 fn finish_catalog_change(&self, corr: u64, outcome: CatalogChangeOutcome) {
1601 let mut inner = self.lock_inner();
1602 if let Some(change) = inner
1603 .catalog_changes
1604 .iter_mut()
1605 .find(|change| change.corr == corr)
1606 {
1607 change.outcome = Some(outcome);
1608 }
1609 while inner
1612 .catalog_changes
1613 .front()
1614 .is_some_and(|change| change.outcome.is_some())
1615 {
1616 let change = inner.catalog_changes.pop_front().expect("completed change");
1617 match change.outcome.expect("completed outcome") {
1618 CatalogChangeOutcome::Accepted => inner.current_provides = Some(change.provides),
1619 CatalogChangeOutcome::Rejected => {}
1620 CatalogChangeOutcome::Unknown => inner.current_provides = None,
1621 }
1622 }
1623 }
1624
1625 fn close_connection(&self) {
1626 let pending = {
1627 let mut inner = self.lock_inner();
1628 if inner.closed {
1629 return;
1630 }
1631 inner.closed = true;
1632 inner.writer = None;
1633 inner.current_provides = None;
1634 inner.catalog_changes.clear();
1635 self.close_token.cancel();
1636 inner.pending_operator_confirms.clear();
1638 inner.pending_activity_records.clear();
1639 inner.pending_scope_served_by.clear();
1640 inner.pending_config_calls.clear();
1641 inner.pending_agent_commands.clear();
1642 inner.pending_claim_calls.clear();
1643 inner.pending_control_streams.clear();
1644 inner
1645 .pending_catalog_updates
1646 .drain()
1647 .map(|(_, reply)| reply)
1648 .collect::<Vec<_>>()
1649 };
1650 self.clear_live_routes();
1651 for reply in pending {
1652 let _ = reply.send(Err(CatalogUpdateError::ConnectionClosed));
1653 }
1654 }
1655
1656 fn clear_live_routes(&self) {
1657 if let Ok(mut routes) = self.live_routes.lock() {
1658 routes.clear();
1659 }
1660 }
1661
1662 fn lock_inner(&self) -> std::sync::MutexGuard<'_, ModuleHandleState> {
1663 self.inner
1664 .lock()
1665 .unwrap_or_else(|poisoned| poisoned.into_inner())
1666 }
1667}
1668
1669fn next_module_control_corr(inner: &mut ModuleHandleState) -> Option<u64> {
1670 let corr = inner.next_corr?;
1671 inner.next_corr = corr.checked_add(1);
1672 Some(corr)
1673}
1674
1675struct OperatorConfirmGuard {
1676 shared: Arc<ModuleHandleShared>,
1677 runtime: tokio::runtime::Handle,
1678 writer: mpsc::Sender<Frame>,
1679 corr: u64,
1680 accepted: bool,
1681 armed: bool,
1682}
1683
1684impl Drop for OperatorConfirmGuard {
1685 fn drop(&mut self) {
1686 if !self.armed {
1687 return;
1688 }
1689 self.shared
1690 .lock_inner()
1691 .pending_operator_confirms
1692 .remove(&self.corr);
1693 if !self.accepted {
1694 return;
1695 }
1696 let Ok(cancel) = Frame::build_with_version(
1697 self.shared.negotiated_ver,
1698 FrameType::Cancel,
1699 control_flags(),
1700 0,
1701 0,
1702 self.corr,
1703 Vec::new(),
1704 ) else {
1705 return;
1706 };
1707 match self.writer.try_send(cancel) {
1708 Ok(()) | Err(mpsc::error::TrySendError::Closed(_)) => {}
1709 Err(mpsc::error::TrySendError::Full(cancel)) => {
1710 let writer = self.writer.clone();
1714 self.runtime.spawn(async move {
1715 let _ = writer.send(cancel).await;
1716 });
1717 }
1718 }
1719 }
1720}
1721
1722#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1727#[non_exhaustive]
1728pub enum OperatorConfirmError {
1729 Declined,
1730 Backoff {
1733 retry_after_ms: Option<u64>,
1734 },
1735 PresenceUnavailable,
1740 PresenceUnsupportedPlatform,
1752 SummaryInvalid,
1753 OperationInvalid,
1759 NotPermitted,
1760 Unsupported,
1761}
1762
1763impl fmt::Display for OperatorConfirmError {
1764 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1765 f.write_str(match self {
1766 Self::Declined => "operator declined the write",
1767 Self::Backoff { retry_after_ms: None } => {
1768 "operator confirmation refused without asking: a recent decline holds new prompts for a short time"
1769 }
1770 Self::PresenceUnavailable => "operator confirmation is unavailable",
1771 Self::PresenceUnsupportedPlatform => {
1772 "operator confirmation has no prompt on this platform"
1773 }
1774 Self::SummaryInvalid => "operator confirmation summary is invalid",
1775 Self::OperationInvalid => "operator confirmation operation is invalid",
1776 Self::NotPermitted => "operator confirmation is not permitted for this route",
1777 Self::Unsupported => "daemon does not support operator confirmation",
1778 Self::Backoff {
1779 retry_after_ms: Some(_),
1780 } => return write!(
1781 f,
1782 "operator confirmation refused without asking: a recent decline holds new prompts for {} s",
1783 self.backoff_seconds().expect("backoff retry duration is present")
1784 ),
1785 })
1786 }
1787}
1788
1789impl OperatorConfirmError {
1790 fn backoff_seconds(&self) -> Option<u64> {
1791 match self {
1792 Self::Backoff {
1793 retry_after_ms: Some(milliseconds),
1794 } => Some(milliseconds / 1_000 + u64::from(!milliseconds.is_multiple_of(1_000))),
1795 _ => None,
1796 }
1797 }
1798}
1799
1800impl Error for OperatorConfirmError {}
1801
1802#[derive(Debug)]
1805pub enum CatalogUpdateError {
1806 NotSupported,
1807 FrozenField(ErrorBody),
1808 NotRegistered(ErrorBody),
1809 Rejected(ErrorBody),
1810 Timeout,
1811 ConnectionClosed,
1812 Protocol(String),
1813}
1814
1815impl fmt::Display for CatalogUpdateError {
1816 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1817 match self {
1818 Self::NotSupported => write!(
1819 f,
1820 "daemon HELLO_ACK did not advertise catalog.update support"
1821 ),
1822 Self::FrozenField(body) => {
1823 write!(f, "catalog.update rejected frozen field: {}", body.message)
1824 }
1825 Self::NotRegistered(body) => write!(
1826 f,
1827 "catalog.update requires a registered module: {}",
1828 body.message
1829 ),
1830 Self::Rejected(body) => write!(
1831 f,
1832 "catalog.update rejected by subc: {} ({})",
1833 body.code, body.message
1834 ),
1835 Self::Timeout => write!(f, "catalog.update timed out waiting for an ACK"),
1836 Self::ConnectionClosed => {
1837 write!(f, "subc connection closed before catalog.update completed")
1838 }
1839 Self::Protocol(message) => write!(f, "catalog.update protocol error: {message}"),
1840 }
1841 }
1842}
1843
1844impl Error for CatalogUpdateError {}
1845
1846#[derive(Debug, Clone, PartialEq, Eq)]
1855#[non_exhaustive]
1856pub enum ScopeCallError {
1857 NotSupported { op: &'static str },
1860 Refused { code: String, message: String },
1865 Timeout,
1867 ConnectionClosed,
1869 Protocol(String),
1871}
1872
1873impl ScopeCallError {
1874 pub fn code(&self) -> Option<&str> {
1876 match self {
1877 Self::Refused { code, .. } => Some(code),
1878 _ => None,
1879 }
1880 }
1881
1882 fn from_control_error(err: CatalogUpdateError) -> Self {
1887 match err {
1888 CatalogUpdateError::FrozenField(body)
1889 | CatalogUpdateError::NotRegistered(body)
1890 | CatalogUpdateError::Rejected(body) => Self::Refused {
1891 code: body.code,
1892 message: body.message,
1893 },
1894 CatalogUpdateError::Timeout => Self::Timeout,
1895 CatalogUpdateError::ConnectionClosed => Self::ConnectionClosed,
1896 CatalogUpdateError::Protocol(message) => Self::Protocol(message),
1897 CatalogUpdateError::NotSupported => {
1900 Self::Protocol("unexpected NotSupported from the control reply table".into())
1901 }
1902 }
1903 }
1904}
1905
1906impl fmt::Display for ScopeCallError {
1907 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1908 match self {
1909 Self::NotSupported { op } => {
1910 write!(f, "daemon HELLO_ACK did not advertise {op} support")
1911 }
1912 Self::Refused { code, message } => {
1913 write!(f, "subc refused the request: {code} ({message})")
1914 }
1915 Self::Timeout => write!(f, "timed out waiting for the daemon's reply"),
1916 Self::ConnectionClosed => {
1917 write!(f, "subc connection closed before the daemon replied")
1918 }
1919 Self::Protocol(message) => write!(f, "scope call protocol error: {message}"),
1920 }
1921 }
1922}
1923
1924impl Error for ScopeCallError {}
1925
1926#[async_trait]
1974pub trait ModuleHandler: Send + Sync + 'static {
1975 async fn handle(&self, ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome;
1980
1981 async fn on_hello_ack(&self, _ack: &ModuleHelloAckBody) {}
1984
1985 fn control_ops(&self) -> Vec<String> {
1991 Vec::new()
1992 }
1993
1994 async fn on_control_request(
2007 &self,
2008 _op: &str,
2009 _body: serde_json::Value,
2010 _cancel: CancellationToken,
2011 ) -> Result<serde_json::Value, ErrorBody> {
2012 Err(ErrorBody::new(
2013 "unsupported_op",
2014 "no control handler implemented",
2015 ))
2016 }
2017
2018 fn config_declaration(&self) -> Option<config::ConfigDeclareRequest> {
2022 None
2023 }
2024
2025 async fn config_review(
2028 &self,
2029 _request: config::ConfigReviewRequest,
2030 ) -> config::ConfigReviewReply {
2031 config::ConfigReviewReply::new(
2032 vec![],
2033 vec![config::ConfigReviewRefusal::new(
2034 "value_invalid",
2035 vec![],
2036 vec!["module has no config review implementation".into()],
2037 )],
2038 )
2039 }
2040
2041 async fn config_apply(&self, _view: config::ConfigView) -> Result<(), Vec<String>> {
2045 Err(vec!["module has no config apply implementation".into()])
2046 }
2047
2048 async fn config_files(&self) {}
2051
2052 async fn on_config_state(&self, _state: config::ConfigState) {}
2054
2055 async fn on_config_error(&self, error: config::ConfigError) {
2058 tracing::warn!(%error, "module configuration request failed");
2059 }
2060
2061 async fn on_bind(&self, _req: &RouteBindRequest) -> BindDecision {
2063 BindDecision::accept()
2064 }
2065
2066 async fn on_bound(&self, _handle: &RouteHandle) {}
2068
2069 async fn health(&self) -> HealthReport {
2099 HealthReport {
2112 detail: Some("no health implementation; inherited default".to_string()),
2113 ..HealthReport::ok()
2114 }
2115 }
2116
2117 async fn on_route_gone(&self, _handle: &RouteHandle) {}
2119
2120 fn request_pools(&self) -> RequestPools {
2152 RequestPools::new()
2153 }
2154
2155 async fn on_draining(&self, _reason: RouteCloseReason, _deadline: SystemTime) {}
2173
2174 async fn on_connection_end(&self, _end: ConnectionEnd) {}
2181}
2182
2183#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2188#[non_exhaustive]
2189pub enum ConnectionEnd {
2190 Goodbye,
2192 Eof,
2194 Reset,
2197 Closed,
2199}
2200
2201impl ConnectionEnd {
2202 pub fn as_str(self) -> &'static str {
2204 match self {
2205 Self::Goodbye => "goodbye",
2206 Self::Eof => "eof",
2207 Self::Reset => "reset",
2208 Self::Closed => "closed",
2209 }
2210 }
2211}
2212
2213impl std::fmt::Display for ConnectionEnd {
2214 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2215 f.write_str(self.as_str())
2216 }
2217}
2218
2219#[derive(Debug, Clone, PartialEq)]
2221pub enum HandlerOutcome {
2222 Response(Vec<u8>),
2224 Error { code: String, message: String },
2226 ErrorWithDetail {
2228 code: String,
2229 message: String,
2230 detail: serde_json::Value,
2231 },
2232 Streamed,
2235}
2236
2237#[derive(Clone)]
2240pub struct RequestCtx {
2241 handle: RouteHandle,
2242 corr: u64,
2243 ver: u8,
2244 egress: mpsc::Sender<Frame>,
2245 module_handle: ModuleHandle,
2246 cancelled: CancellationToken,
2247 binary_reply: Arc<AtomicBool>,
2248 request_flags: Flags,
2249 request_body: Arc<[u8]>,
2250}
2251
2252impl RequestCtx {
2253 pub fn module_handle(&self) -> ModuleHandle {
2255 self.module_handle.clone()
2256 }
2257
2258 pub fn request_payload(
2263 &self,
2264 ) -> Result<Option<(serde_json::Value, &[u8])>, subc_protocol::framed_body::FramedBodyError>
2265 {
2266 if !self.request_flags.is_binary() {
2267 return Ok(None);
2268 }
2269 subc_protocol::framed_body::decode(&self.request_body).map(Some)
2270 }
2271
2272 pub fn route_handle(&self) -> RouteHandle {
2274 self.handle
2275 }
2276
2277 pub fn corr(&self) -> u64 {
2279 self.corr
2280 }
2281
2282 pub fn mark_binary_reply(&self) {
2294 self.binary_reply.store(true, Ordering::Relaxed);
2295 }
2296
2297 pub async fn emit(&self, body: Vec<u8>) -> Result<(), SubcModuleError> {
2300 self.emit_with_admission(body, None).await
2301 }
2302
2303 pub async fn emit_with_admission(
2305 &self,
2306 body: Vec<u8>,
2307 admission_class: Option<AdmissionClass>,
2308 ) -> Result<(), SubcModuleError> {
2309 self.module_handle.validate_route(self.handle)?;
2310 if self.cancelled.is_cancelled() {
2311 return Ok(());
2312 }
2313 self.send_frame(
2314 FrameType::StreamData,
2315 data_flags().with_admission_class(admission_class.unwrap_or(AdmissionClass::Normal)),
2316 body,
2317 )
2318 .await
2319 }
2320
2321 pub fn cancelled(&self) -> WaitForCancellationFuture<'_> {
2324 self.cancelled.cancelled()
2325 }
2326
2327 pub fn cancellation_token(&self) -> CancellationToken {
2329 self.cancelled.clone()
2330 }
2331
2332 async fn send_frame(
2333 &self,
2334 frame_type: FrameType,
2335 flags: Flags,
2336 body: Vec<u8>,
2337 ) -> Result<(), SubcModuleError> {
2338 self.module_handle.validate_route(self.handle)?;
2339 let (frame_type, flags, body) = bounded_terminal(frame_type, flags, body)?;
2340 let frame = Frame::build_with_version(
2341 self.ver,
2342 frame_type,
2343 flags,
2344 self.handle.channel,
2345 self.handle.epoch,
2346 self.corr,
2347 body,
2348 )
2349 .map_err(SubcModuleError::FrameBuild)?;
2350 send_outbound(&self.egress, frame).await
2351 }
2352}
2353
2354fn checked_increment(counter: &AtomicU64) -> Option<u64> {
2359 let mut current = counter.load(Ordering::Relaxed);
2360 loop {
2361 let next = current.checked_add(1)?;
2362 match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
2363 Ok(previous) => return Some(previous),
2364 Err(actual) => current = actual,
2365 }
2366 }
2367}
2368
2369#[derive(Debug, Clone)]
2376#[non_exhaustive]
2377pub struct RouteBindRequest {
2378 pub handle: RouteHandle,
2379 pub target: RouteTarget,
2380 pub identity: BindIdentity,
2381 pub principal: Option<Principal>,
2382 pub consumer_capabilities: Option<Vec<String>>,
2389 pub role_versions: Option<std::collections::BTreeMap<String, String>>,
2396 pub admission_facts: Option<serde_json::Value>,
2398 pub scope: Option<ScopeStamp>,
2407 pub call_mode: CallMode,
2410 pub peer: subc_protocol::peer::RouteBindPeer,
2438}
2439
2440impl RouteBindRequest {
2441 pub fn new(handle: RouteHandle, target: RouteTarget, identity: BindIdentity) -> Self {
2446 Self {
2447 handle,
2448 target,
2449 identity,
2450 principal: None,
2451 consumer_capabilities: None,
2452 role_versions: None,
2453 admission_facts: None,
2454 scope: None,
2455 call_mode: derive_call_mode(None),
2456 peer: subc_protocol::peer::RouteBindPeer::new(),
2457 }
2458 }
2459
2460 pub fn with_peer(mut self, peer: subc_protocol::peer::RouteBindPeer) -> Self {
2463 self.peer = peer;
2464 self
2465 }
2466
2467 pub fn peer_build(&self) -> Option<&str> {
2470 self.peer.peer_build.as_deref()
2471 }
2472
2473 pub fn with_principal(mut self, principal: Principal) -> Self {
2475 self.principal = Some(principal);
2476 self
2477 }
2478
2479 pub fn with_consumer_capabilities(mut self, consumer_capabilities: Vec<String>) -> Self {
2481 self.consumer_capabilities = Some(consumer_capabilities);
2482 self
2483 }
2484
2485 pub fn with_role_versions(
2487 mut self,
2488 role_versions: std::collections::BTreeMap<String, String>,
2489 ) -> Self {
2490 self.role_versions = Some(role_versions);
2491 self
2492 }
2493
2494 pub fn with_admission_facts(mut self, admission_facts: serde_json::Value) -> Self {
2496 self.admission_facts = Some(admission_facts);
2497 self
2498 }
2499
2500 pub fn with_scope(mut self, scope: ScopeStamp) -> Self {
2503 self.call_mode = derive_call_mode(Some(&scope));
2504 self.scope = Some(scope);
2505 self
2506 }
2507
2508 pub fn with_call_mode(mut self, call_mode: CallMode) -> Self {
2510 self.call_mode = call_mode;
2511 self
2512 }
2513}
2514
2515#[derive(Debug, Clone)]
2517pub struct BindDecision {
2518 kind: BindDecisionKind,
2519}
2520
2521impl BindDecision {
2522 pub fn accept() -> Self {
2524 Self {
2525 kind: BindDecisionKind::Accept,
2526 }
2527 }
2528
2529 pub fn reject(code: impl Into<String>, message: impl Into<String>) -> Self {
2531 Self {
2532 kind: BindDecisionKind::Reject {
2533 code: code.into(),
2534 message: message.into(),
2535 detail: None,
2536 },
2537 }
2538 }
2539
2540 pub fn reject_with_detail(
2547 code: impl Into<String>,
2548 message: impl Into<String>,
2549 detail: serde_json::Value,
2550 ) -> Self {
2551 Self {
2552 kind: BindDecisionKind::Reject {
2553 code: code.into(),
2554 message: message.into(),
2555 detail: Some(detail),
2556 },
2557 }
2558 }
2559}
2560
2561#[derive(Debug, Clone)]
2562enum BindDecisionKind {
2563 Accept,
2564 Reject {
2565 code: String,
2566 message: String,
2567 detail: Option<serde_json::Value>,
2568 },
2569}
2570
2571pub async fn serve<H>(manifest: ModuleManifest, handler: H) -> Result<(), SubcModuleError>
2580where
2581 H: ModuleHandler,
2582{
2583 ServeOptions::default().serve(manifest, handler).await
2584}
2585
2586#[derive(Clone, Debug, Default)]
2590pub struct ModuleStopHandle {
2591 token: CancellationToken,
2592}
2593
2594impl ModuleStopHandle {
2595 pub fn new() -> Self {
2596 Self::default()
2597 }
2598
2599 pub fn stop(&self) {
2602 self.token.cancel();
2603 }
2604}
2605
2606#[derive(Clone, Debug)]
2609#[non_exhaustive]
2610pub struct ServeOptions {
2611 registration_channel: bool,
2612 prefix_child: bool,
2613 writer_drain_timeout: Duration,
2614 stop_handle: ModuleStopHandle,
2615}
2616
2617impl Default for ServeOptions {
2618 fn default() -> Self {
2619 Self {
2620 writer_drain_timeout: DEFAULT_WRITER_DRAIN_TIMEOUT,
2621 registration_channel: false,
2622 prefix_child: false,
2623 stop_handle: ModuleStopHandle::new(),
2624 }
2625 }
2626}
2627
2628impl ServeOptions {
2629 pub fn with_writer_drain_timeout(mut self, timeout: Duration) -> Self {
2639 self.writer_drain_timeout = timeout;
2640 self
2641 }
2642
2643 pub fn with_stop_handle(mut self, handle: ModuleStopHandle) -> Self {
2646 self.stop_handle = handle;
2647 self
2648 }
2649
2650 pub async fn serve<H: ModuleHandler>(
2652 self,
2653 mut manifest: ModuleManifest,
2654 handler: H,
2655 ) -> Result<(), SubcModuleError> {
2656 let connection_file = parse_subc_arg(env::args_os().skip(1))?;
2657 if let Some(module_id) = module_id_from_env()? {
2658 manifest.module_id = module_id;
2659 }
2660 self.serve_with(&connection_file, manifest, handler).await
2661 }
2662
2663 pub async fn serve_with<H: ModuleHandler>(
2665 self,
2666 connection_file: &Path,
2667 manifest: ModuleManifest,
2668 handler: H,
2669 ) -> Result<(), SubcModuleError> {
2670 if let Some((_handle, future)) = self
2671 .serve_with_handle(connection_file, manifest, handler)
2672 .await?
2673 {
2674 future.await?;
2675 }
2676 Ok(())
2677 }
2678
2679 pub async fn serve_from_env_with_handle<H: ModuleHandler>(
2693 self,
2694 mut manifest: ModuleManifest,
2695 handler: H,
2696 ) -> Result<Option<(ModuleHandle, ModuleServeFuture)>, SubcModuleError> {
2697 let connection_file = parse_subc_arg(env::args_os().skip(1))?;
2698 if let Some(module_id) = module_id_from_env()? {
2699 manifest.module_id = module_id;
2700 }
2701 self.serve_with_handle(&connection_file, manifest, handler)
2702 .await
2703 }
2704
2705 pub async fn serve_with_handle<H: ModuleHandler>(
2717 self,
2718 connection_file: &Path,
2719 manifest: ModuleManifest,
2720 handler: H,
2721 ) -> Result<Option<(ModuleHandle, ModuleServeFuture)>, SubcModuleError> {
2722 prepare_connection(connection_file, manifest, handler, self).await
2723 }
2724}
2725
2726pub async fn serve_from_env_with_handle<H>(
2741 manifest: ModuleManifest,
2742 handler: H,
2743) -> Result<(ModuleHandle, ModuleServeFuture), SubcModuleError>
2744where
2745 H: ModuleHandler,
2746{
2747 ServeOptions::default()
2748 .serve_from_env_with_handle(manifest, handler)
2749 .await?
2750 .ok_or(SubcModuleError::ConnectionClosedBeforeHelloAck)
2751}
2752
2753pub async fn serve_with<H>(
2756 connection_file: &Path,
2757 manifest: ModuleManifest,
2758 handler: H,
2759) -> Result<(), SubcModuleError>
2760where
2761 H: ModuleHandler,
2762{
2763 ServeOptions::default()
2764 .serve_with(connection_file, manifest, handler)
2765 .await
2766}
2767
2768pub async fn serve_with_handle<H>(
2771 connection_file: &Path,
2772 manifest: ModuleManifest,
2773 handler: H,
2774) -> Result<(ModuleHandle, ModuleServeFuture), SubcModuleError>
2775where
2776 H: ModuleHandler,
2777{
2778 ServeOptions::default()
2779 .serve_with_handle(connection_file, manifest, handler)
2780 .await?
2781 .ok_or(SubcModuleError::ConnectionClosedBeforeHelloAck)
2782}
2783
2784async fn prepare_connection<H: ModuleHandler>(
2785 connection_file: &Path,
2786 manifest: ModuleManifest,
2787 handler: H,
2788 mut options: ServeOptions,
2789) -> Result<Option<(ModuleHandle, ModuleServeFuture)>, SubcModuleError> {
2790 options.registration_channel = registration::adopt(&manifest.module_id)?;
2791 let stream = tokio::select! {
2792 biased;
2793 () = options.stop_handle.token.cancelled() => return Ok(None),
2794 stream = async {
2795 let claim = if registration::supervised() && !options.registration_channel {
2796 registration::prefix_claim(&manifest.module_id).await
2797 .map_err(|error| registration::serve_claim_error(&manifest.module_id, error))?
2798 } else {
2799 None
2800 };
2801 let mut stream = connect_to_subc(connection_file, options.registration_channel).await?;
2802 if let Some(claim) = claim {
2803 registration::present_claim(&mut stream, claim, AUTH_DEADLINE).await
2804 .map_err(|error| registration::serve_claim_error(&manifest.module_id, error))?;
2805 options.prefix_child = true;
2806 }
2807 Ok::<_, SubcModuleError>(stream)
2808 } => stream?,
2809 };
2810 register_connection(stream, manifest, handler, options).await
2811}
2812
2813async fn register_connection<S, H>(
2814 stream: S,
2815 manifest: ModuleManifest,
2816 handler: H,
2817 options: ServeOptions,
2818) -> Result<Option<(ModuleHandle, ModuleServeFuture)>, SubcModuleError>
2819where
2820 S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
2821 H: ModuleHandler,
2822{
2823 let control_ops = control_ops::validate(handler.control_ops())?;
2824 let (mut read_half, write_half) = tokio::io::split(stream);
2825 let (tx, rx) = mpsc::channel::<Frame>(EGRESS_BUFFER);
2826 let connection_runtime = tokio::runtime::Handle::current();
2827 let mut writer = tokio::spawn(drain_writer(write_half, rx));
2828 let handler = Arc::new(handler);
2829 let declaration = if options.prefix_child {
2830 None
2831 } else {
2832 handler.config_declaration()
2833 };
2834
2835 let mut hello_ack = None;
2836 let provides = manifest.provides.clone();
2837 let handshake = async {
2838 send_hello(&tx, manifest, declaration.is_some(), &control_ops).await?;
2839 hello_ack = Some(expect_hello_ack(&mut read_half).await?);
2842 handler
2843 .on_hello_ack(hello_ack.as_ref().expect("validated HELLO_ACK"))
2844 .await;
2845 Ok::<_, SubcModuleError>(())
2846 };
2847 let mut writer_result = None;
2848 let handshake_result = tokio::select! {
2849 biased;
2850 () = options.stop_handle.token.cancelled() => None,
2851 result = handshake => Some(result),
2852 result = &mut writer => {
2853 writer_result = Some(result);
2854 Some(Ok(()))
2855 }
2856 };
2857 match handshake_result {
2858 Some(Ok(())) => {}
2859 other => {
2860 let result = match other {
2861 None => {
2862 let deadline = Instant::now() + options.writer_drain_timeout;
2863 let result = timeout_at(deadline, send_goodbye(&tx)).await;
2864 drop(tx);
2865 let flush = finish_writer_and_drain(writer, &mut read_half, deadline).await;
2866 handler.on_connection_end(ConnectionEnd::Closed).await;
2867 match result.unwrap_or(Ok(())) {
2868 Ok(()) | Err(SubcModuleError::WriterClosed) => {}
2869 Err(error) => return Err(error),
2870 }
2871 match flush.0 {
2872 Err(SubcModuleError::FrameIo(FrameIoError::Io(error)))
2873 if is_disconnect(&error) => {}
2874 result => result?,
2875 }
2876 return Ok(None);
2877 }
2878 Some(Err(error)) => error,
2879 Some(Ok(_)) => unreachable!(),
2880 };
2881 drop(tx);
2882 let deadline = Instant::now() + options.writer_drain_timeout;
2883 let _ = finish_writer_and_drain(writer, &mut read_half, deadline).await;
2884 return Err(result);
2885 }
2886 }
2887 let mut ack = match hello_ack {
2888 Some(ack) => ack,
2889 None => {
2890 return Err(writer_ended(
2891 writer_result.expect("writer ended before HELLO_ACK"),
2892 ))
2893 }
2894 };
2895 if options.prefix_child {
2896 ack.subc_ops.retain(|op| {
2899 matches!(
2900 op.as_str(),
2901 "catalog.update" | "scope.describe" | "supervisor.live_roots" | "activity.record"
2902 )
2903 });
2904 }
2905 let writer_end = writer_result.map(writer_connection_end);
2908
2909 let Some(connection_token) = checked_increment(&NEXT_MODULE_CONNECTION_TOKEN) else {
2910 drop(tx);
2911 let deadline = Instant::now() + options.writer_drain_timeout;
2912 if writer_end.is_none() {
2913 let _ = finish_writer_and_drain(writer, &mut read_half, deadline).await;
2914 }
2915 return Err(SubcModuleError::ConnectionTokenExhausted);
2916 };
2917 let close_token = CancellationToken::new();
2918 let handle = ModuleHandle::new(
2919 &ack,
2920 tx.clone(),
2921 connection_token,
2922 close_token,
2923 connection_runtime,
2924 );
2925 handle.shared.lock_inner().current_provides = Some(provides);
2926 handle
2927 .shared
2928 .registration
2929 .prefix_child
2930 .store(options.prefix_child, Ordering::Release);
2931 let _ = handle.shared.served_control_ops.set(control_ops);
2932 if options.registration_channel {
2933 registration::install_owner(&handle)?;
2934 }
2935 if let Some(declaration) = declaration {
2936 let _ = handle.shared.config.declaration.set(declaration);
2937 }
2938 let serve_future = connection_serve_future_with_options(
2939 read_half,
2940 tx,
2941 handler,
2942 handle.clone(),
2943 writer,
2944 options,
2945 writer_end,
2946 );
2947 Ok(Some((handle, serve_future)))
2948}
2949
2950#[cfg(any(test, feature = "test-support"))]
2953fn connection_serve_future<R, H>(
2954 read_half: R,
2955 tx: mpsc::Sender<Frame>,
2956 handler: Arc<H>,
2957 handle: ModuleHandle,
2958 writer: tokio::task::JoinHandle<Result<(), WriterFailure>>,
2959) -> ModuleServeFuture
2960where
2961 R: AsyncRead + Unpin + Send + 'static,
2962 H: ModuleHandler,
2963{
2964 connection_serve_future_with_options(
2965 read_half,
2966 tx,
2967 handler,
2968 handle,
2969 writer,
2970 ServeOptions::default(),
2971 None,
2972 )
2973}
2974
2975fn registration_channel_exit_code(result: &Result<ConnectionEnd, SubcModuleError>) -> i32 {
2976 if matches!(result, Ok(ConnectionEnd::Goodbye)) {
2977 0
2978 } else {
2979 1
2980 }
2981}
2982
2983fn connection_serve_future_with_options<R, H>(
2984 mut read_half: R,
2985 tx: mpsc::Sender<Frame>,
2986 handler: Arc<H>,
2987 handle: ModuleHandle,
2988 mut writer: tokio::task::JoinHandle<Result<(), WriterFailure>>,
2989 options: ServeOptions,
2990 writer_end: Option<Result<ConnectionEnd, SubcModuleError>>,
2991) -> ModuleServeFuture
2992where
2993 R: AsyncRead + Unpin + Send + 'static,
2994 H: ModuleHandler,
2995{
2996 let serve_handle = handle.clone();
2997 handle.install_dispatch_pools(handler.request_pools());
3001 Box::pin(async move {
3002 let dispatcher = RequestDispatcher::with_pools(serve_handle.dispatch_pools());
3006 if !options.prefix_child {
3007 config::start(serve_handle.clone(), Arc::clone(&handler), &dispatcher);
3008 }
3009 let mut writer_finished = writer_end.is_some();
3010 let mut loop_result = if let Some(end) = writer_end {
3011 end
3012 } else {
3013 tokio::select! {
3014 biased;
3015 result = module_loop(&mut read_half, &tx, Arc::clone(&handler), serve_handle.clone(), &dispatcher, &options.stop_handle) => result,
3016 result = &mut writer => {
3017 writer_finished = true;
3018 writer_connection_end(result)
3019 }
3020 }
3021 };
3022 if matches!(loop_result, Err(SubcModuleError::WriterClosed)) && !writer_finished {
3026 writer_finished = true;
3027 loop_result = writer_connection_end((&mut writer).await);
3028 }
3029 let deadline = Instant::now() + options.writer_drain_timeout;
3030 serve_handle.shared.config.stop.cancel();
3031 dispatcher.wait_for_draining_hooks_to_start().await;
3032 dispatcher.tasks.close();
3033 if matches!(loop_result, Ok(ConnectionEnd::Closed))
3034 && options.stop_handle.token.is_cancelled()
3035 {
3036 let _ = timeout_at(deadline, dispatcher.tasks.wait()).await;
3038 }
3039 let orderly = loop_result.is_ok();
3040 if let Some(error) = dispatcher.take_failure() {
3041 if !orderly || !matches!(error, SubcModuleError::WriterClosed) {
3042 loop_result = Err(error);
3043 }
3044 }
3045 dispatcher.cancel_all();
3046 let _ = timeout_at(deadline, dispatcher.tasks.wait()).await;
3047 let abandoned_replies = dispatcher
3048 .in_flight
3049 .lock()
3050 .unwrap_or_else(|e| e.into_inner())
3051 .len();
3052 if let Some(error) = dispatcher.take_failure() {
3053 if !orderly || !matches!(error, SubcModuleError::WriterClosed) {
3054 loop_result = Err(error);
3055 }
3056 }
3057 dispatcher.abort_all();
3058 dispatcher.tasks.wait().await;
3059 let daemon_eof = matches!(loop_result, Ok(ConnectionEnd::Eof));
3060 let locally_closed = matches!(loop_result, Ok(ConnectionEnd::Closed));
3061 let module_ended =
3066 !writer_finished && matches!(loop_result, Ok(ConnectionEnd::Closed) | Err(_));
3067 let deadline = Instant::now() + options.writer_drain_timeout;
3068 if matches!(loop_result, Ok(ConnectionEnd::Closed))
3069 && options.stop_handle.token.is_cancelled()
3070 {
3071 match timeout_at(deadline, send_goodbye(&tx)).await {
3072 Ok(Ok(())) | Err(_) => {}
3073 Ok(Err(SubcModuleError::WriterClosed)) => {}
3074 Ok(Err(error)) => loop_result = Err(error),
3075 }
3076 }
3077 serve_handle.close_connection();
3078 drop(tx);
3079 let loop_result = match loop_result {
3080 Ok(end) => {
3081 handler.on_connection_end(end).await;
3082 Ok(end)
3083 }
3084 Err(error) => Err(error),
3085 };
3086
3087 let (writer_result, undelivered_replies) = if writer_finished {
3088 (Ok(()), 0)
3089 } else if module_ended {
3090 finish_writer_and_drain(writer, &mut read_half, deadline).await
3091 } else {
3092 finish_writer(writer, deadline).await
3093 };
3094 let undelivered_replies = undelivered_replies as u64
3095 + dispatcher.undelivered_replies.load(Ordering::Relaxed)
3096 + abandoned_replies as u64;
3097 if daemon_eof && undelivered_replies > 0 {
3098 tracing::warn!(
3099 undelivered_replies,
3100 "daemon EOF: replies could not be delivered during drain"
3101 );
3102 }
3103 if options.registration_channel && !locally_closed {
3104 std::process::exit(registration_channel_exit_code(&loop_result));
3109 }
3110 match (loop_result.map(|_| ()), writer_result) {
3111 (Err(SubcModuleError::WriterClosed), Err(writer_error)) => Err(writer_error),
3112 (Err(loop_err), _) => Err(loop_err),
3113 (Ok(()), Ok(())) => Ok(()),
3114 (Ok(()), Err(SubcModuleError::FrameIo(FrameIoError::Io(err))))
3119 if is_disconnect(&err) =>
3120 {
3121 Ok(())
3122 }
3123 (Ok(()), Err(error)) => Err(error),
3124 }
3125 })
3126}
3127
3128async fn module_loop<R, H>(
3129 mut reader: R,
3130 egress: &mpsc::Sender<Frame>,
3131 handler: Arc<H>,
3132 module_handle: ModuleHandle,
3133 dispatcher: &RequestDispatcher,
3134 stop: &ModuleStopHandle,
3135) -> Result<ConnectionEnd, SubcModuleError>
3136where
3137 R: AsyncRead + Unpin,
3138 H: ModuleHandler,
3139{
3140 serve_frames(
3141 &mut reader,
3142 egress,
3143 &handler,
3144 &module_handle,
3145 dispatcher,
3146 stop,
3147 )
3148 .await
3149}
3150
3151async fn serve_frames<R, H>(
3152 reader: &mut R,
3153 egress: &mpsc::Sender<Frame>,
3154 handler: &Arc<H>,
3155 module_handle: &ModuleHandle,
3156 dispatcher: &RequestDispatcher,
3157 stop: &ModuleStopHandle,
3158) -> Result<ConnectionEnd, SubcModuleError>
3159where
3160 R: AsyncRead + Unpin,
3161 H: ModuleHandler,
3162{
3163 loop {
3164 let read = tokio::select! {
3165 biased;
3166 () = dispatcher.failed.cancelled() => return Err(dispatcher.take_failure().expect("failure signalled")),
3167 () = stop.token.cancelled() => return Ok(ConnectionEnd::Closed),
3168 () = module_handle.shared.close_token.cancelled() => return Ok(ConnectionEnd::Closed),
3169 read = read_frame(reader) => read,
3170 };
3171 let frame = match read {
3172 Ok(Some(frame)) => frame,
3173 Ok(None) => return Ok(ConnectionEnd::Eof),
3175 Err(FrameIoError::Io(error)) if is_disconnect(&error) => {
3178 return Ok(ConnectionEnd::Reset);
3179 }
3180 Err(err) => return Err(SubcModuleError::FrameIo(err)),
3181 };
3182 if !handle_frame(
3183 frame,
3184 egress,
3185 Arc::clone(handler),
3186 dispatcher.clone(),
3187 module_handle.clone(),
3188 )
3189 .await?
3190 {
3191 return Ok(ConnectionEnd::Goodbye);
3193 }
3194 }
3195}
3196
3197async fn handle_frame<H>(
3198 frame: Frame,
3199 egress: &mpsc::Sender<Frame>,
3200 handler: Arc<H>,
3201 dispatcher: RequestDispatcher,
3202 module_handle: ModuleHandle,
3203) -> Result<bool, SubcModuleError>
3204where
3205 H: ModuleHandler,
3206{
3207 if frame.header.channel != 0
3208 && !module_handle.validate_ingress(frame.header.channel, frame.header.epoch)?
3209 {
3210 return Ok(true);
3211 }
3212 match frame.header.ty {
3213 FrameType::Ping if frame.header.channel == 0 => {
3214 let pong = Frame::build_with_version(
3215 frame.header.ver,
3216 FrameType::Pong,
3217 frame.header.flags,
3218 0,
3219 0,
3220 frame.header.corr,
3221 Vec::new(),
3222 )
3223 .map_err(SubcModuleError::FrameBuild)?;
3224 send_outbound(egress, pong).await?;
3225 Ok(true)
3226 }
3227 FrameType::Goodbye if frame.header.channel == 0 => Ok(false),
3228 FrameType::Goodbye => {
3229 let handle = module_handle.route_handle(frame.header.channel, frame.header.epoch);
3230 if module_handle.remove_route(handle)? {
3231 cancel_handle(&dispatcher.in_flight, handle)?;
3232 handler.on_route_gone(&handle).await;
3233 }
3234 Ok(true)
3235 }
3236 FrameType::Response if frame.header.channel == 0 => {
3237 let _ = module_handle.handle_control_reply(frame);
3238 Ok(true)
3239 }
3240 FrameType::Error | FrameType::StreamData | FrameType::StreamEnd
3241 if frame.header.channel == 0 =>
3242 {
3243 let _ = module_handle.handle_control_reply(frame);
3244 Ok(true)
3245 }
3246 FrameType::Cancel => {
3247 handle_cancel(frame, &dispatcher.in_flight)?;
3248 Ok(true)
3249 }
3250 FrameType::Request if frame.header.channel == 0 => {
3251 handle_control_request(frame, egress, handler, dispatcher, module_handle.clone())
3252 .await?;
3253 Ok(true)
3254 }
3255 FrameType::Request => {
3256 spawn_data_request(frame, egress.clone(), handler, dispatcher, module_handle)?;
3257 Ok(true)
3258 }
3259 FrameType::Push if frame.header.channel == 0 => {
3260 if config::changed(&frame, &module_handle) {
3261 return Ok(true);
3262 }
3263 handle_control_push(&frame, handler, &dispatcher);
3264 Ok(true)
3265 }
3266 _ => Ok(true),
3267 }
3268}
3269
3270fn handle_control_push<H>(frame: &Frame, handler: Arc<H>, dispatcher: &RequestDispatcher)
3275where
3276 H: ModuleHandler,
3277{
3278 match serde_json::from_slice::<ModuleControlCommand>(&frame.body) {
3279 Ok(ModuleControlCommand::Draining {
3280 reason,
3281 deadline_ms,
3282 }) => {
3283 let started = spawn_draining_hook(
3284 handler,
3285 sdk_route_close_reason(reason),
3286 drain_deadline(deadline_ms),
3287 dispatcher,
3288 );
3289 dispatcher
3290 .draining_hooks
3291 .lock()
3292 .unwrap_or_else(|poisoned| poisoned.into_inner())
3293 .push(started);
3294 }
3295 Err(error) => {
3296 if !dispatcher
3297 .undecodable_push_logged
3298 .swap(true, Ordering::Relaxed)
3299 {
3300 let op = serde_json::from_slice::<serde_json::Value>(&frame.body)
3301 .ok()
3302 .and_then(|body| body.get("op")?.as_str().map(str::to_string));
3303 tracing::warn!(
3304 op = op.as_deref().unwrap_or("<none>"),
3305 error = %error,
3306 "ignoring a channel-0 push from subc that this SDK cannot decode; \
3307 further undecodable pushes on this connection are ignored without logging"
3308 );
3309 }
3310 }
3311 }
3312}
3313
3314fn spawn_draining_hook<H>(
3318 handler: Arc<H>,
3319 reason: RouteCloseReason,
3320 deadline: SystemTime,
3321 dispatcher: &RequestDispatcher,
3322) -> oneshot::Receiver<()>
3323where
3324 H: ModuleHandler,
3325{
3326 let (started_tx, started_rx) = oneshot::channel();
3327 dispatcher.spawn(async move {
3328 let mut hook = handler.on_draining(reason, deadline);
3329 let mut started_tx = Some(started_tx);
3330 std::future::poll_fn(|cx| {
3331 let poll = hook.as_mut().poll(cx);
3332 if let Some(started) = started_tx.take() {
3333 let _ = started.send(());
3334 }
3335 poll
3336 })
3337 .await;
3338 });
3339 started_rx
3340}
3341
3342fn sdk_route_close_reason(reason: subc_protocol::RouteCloseReason) -> RouteCloseReason {
3346 match serde_json::to_value(reason) {
3347 Ok(serde_json::Value::String(wire)) => RouteCloseReason::from_wire(&wire),
3348 _ => RouteCloseReason::Unknown(format!("{reason:?}")),
3349 }
3350}
3351
3352fn drain_deadline(deadline_ms: u64) -> SystemTime {
3356 UNIX_EPOCH
3357 .checked_add(Duration::from_millis(deadline_ms))
3358 .unwrap_or_else(SystemTime::now)
3359}
3360
3361fn spawn_data_request<H>(
3362 frame: Frame,
3363 egress: mpsc::Sender<Frame>,
3364 handler: Arc<H>,
3365 dispatcher: RequestDispatcher,
3366 module_handle: ModuleHandle,
3367) -> Result<(), SubcModuleError>
3368where
3369 H: ModuleHandler,
3370{
3371 let handle = module_handle.route_handle(frame.header.channel, frame.header.epoch);
3372 let corr = frame.header.corr;
3373 let cancellation = CancellationToken::new();
3374 {
3375 let mut guard = lock_in_flight(&dispatcher.in_flight)?;
3376 guard.insert((handle.channel, handle.epoch, corr), cancellation.clone());
3377 }
3378
3379 let ctx = RequestCtx {
3380 handle,
3381 corr,
3382 ver: frame.header.ver,
3383 egress,
3384 module_handle,
3385 cancelled: cancellation,
3386 binary_reply: Arc::new(AtomicBool::new(false)),
3387 request_flags: frame.header.flags,
3388 request_body: Arc::from(frame.body.as_slice()),
3389 };
3390 let body = frame.body;
3391 let in_flight = Arc::clone(&dispatcher.in_flight);
3392 let (pool, op) = dispatcher.pools.classify(&body);
3395 let pool = pool.clone();
3396 let wait = pool.start_waiting((handle.channel, handle.epoch, corr));
3397 dispatcher.clone().spawn(async move {
3398 let waiting = WaitingRequest {
3399 op,
3400 channel: handle.channel,
3401 corr,
3402 };
3403 let permit = match wait_for_permit(&pool, wait, waiting, &ctx.cancelled).await {
3404 PermitWaitOutcome::Acquired(permit) => Some(permit),
3405 PermitWaitOutcome::Cancelled => None,
3408 PermitWaitOutcome::Closed => {
3409 dispatcher
3411 .undelivered_replies
3412 .fetch_add(1, Ordering::Relaxed);
3413 if let Ok(mut guard) = in_flight.lock() {
3414 guard.remove(&(handle.channel, handle.epoch, corr));
3415 }
3416 return;
3417 }
3418 };
3419 let outcome = if permit.is_none() || ctx.cancelled.is_cancelled() {
3422 Ok(HandlerOutcome::Error {
3423 code: "cancelled".to_string(),
3424 message: "request cancelled".to_string(),
3425 })
3426 } else {
3427 catch_handler_panic(async { handler.handle(ctx.clone(), body).await }).await
3428 };
3429 let panicked = outcome.is_err();
3430 let outcome = outcome.unwrap_or_else(|()| HandlerOutcome::Error {
3431 code: "internal".to_string(),
3432 message: "module handler panicked".to_string(),
3433 });
3434 match send_handler_outcome(&ctx, outcome).await {
3435 Err(SubcModuleError::StaleRouteHandle(_)) if !panicked => {}
3439 Err(error) => {
3440 if matches!(error, SubcModuleError::WriterClosed) {
3441 dispatcher
3442 .undelivered_replies
3443 .fetch_add(1, Ordering::Relaxed);
3444 }
3445 dispatcher.fail(error);
3446 }
3447 Ok(()) if panicked => dispatcher.fail(SubcModuleError::HandlerPanic),
3448 Ok(()) => {}
3449 }
3450 if let Ok(mut guard) = in_flight.lock() {
3451 guard.remove(&(handle.channel, handle.epoch, corr));
3452 }
3453 });
3454 Ok(())
3455}
3456
3457fn spawn_health_request<H>(
3458 frame: Frame,
3459 egress: mpsc::Sender<Frame>,
3460 handler: Arc<H>,
3461 dispatcher: RequestDispatcher,
3462) -> Result<(), SubcModuleError>
3463where
3464 H: ModuleHandler,
3465{
3466 let channel = frame.header.channel;
3467 let epoch = frame.header.epoch;
3468 let corr = frame.header.corr;
3469 let ver = frame.header.ver;
3470 let cancellation = CancellationToken::new();
3471 {
3472 let mut guard = lock_in_flight(&dispatcher.in_flight)?;
3473 guard.insert((channel, epoch, corr), cancellation.clone());
3474 }
3475
3476 let in_flight = Arc::clone(&dispatcher.in_flight);
3477 dispatcher.clone().spawn(async move {
3478 if !cancellation.is_cancelled() {
3481 let result = async {
3482 let report = catch_handler_panic(async { handler.health().await }).await;
3483 let panicked = report.is_err();
3484 let (ty, body) = match report {
3485 Ok(report) => (
3486 FrameType::Response,
3487 serde_json::to_vec(&ModuleControlResponse::from(
3488 dispatcher.fold_health(report),
3489 ))?,
3490 ),
3491 Err(()) => (
3492 FrameType::Error,
3493 serde_json::to_vec(&ErrorBody::new("internal", "module handler panicked"))?,
3494 ),
3495 };
3496 let (ty, flags, body) = bounded_terminal(ty, control_flags(), body)?;
3497 let frame = Frame::build_with_version(ver, ty, flags, channel, epoch, corr, body)
3498 .map_err(SubcModuleError::FrameBuild)?;
3499 send_outbound(&egress, frame).await?;
3500 if panicked {
3501 return Err(SubcModuleError::HandlerPanic);
3502 }
3503 Ok(())
3504 }
3505 .await;
3506 if let Err(error) = result {
3507 if matches!(error, SubcModuleError::WriterClosed) {
3508 dispatcher
3509 .undelivered_replies
3510 .fetch_add(1, Ordering::Relaxed);
3511 }
3512 dispatcher.fail(error);
3513 }
3514 }
3515 if let Ok(mut guard) = in_flight.lock() {
3516 guard.remove(&(channel, epoch, corr));
3517 }
3518 });
3519 Ok(())
3520}
3521
3522async fn send_handler_outcome(
3523 ctx: &RequestCtx,
3524 outcome: HandlerOutcome,
3525) -> Result<(), SubcModuleError> {
3526 match outcome {
3527 HandlerOutcome::Response(body) => {
3528 let flags = Flags::new(
3529 ctx.binary_reply.load(Ordering::Relaxed),
3530 Priority::Interactive,
3531 false,
3532 );
3533 ctx.send_frame(FrameType::Response, flags, body).await
3534 }
3535 HandlerOutcome::Error { code, message } => {
3536 let body = serde_json::to_vec(&ErrorBody::new(code, message))
3537 .map_err(SubcModuleError::Json)?;
3538 ctx.send_frame(FrameType::Error, data_flags(), body).await
3539 }
3540 HandlerOutcome::ErrorWithDetail {
3541 code,
3542 message,
3543 detail,
3544 } => {
3545 let body = serde_json::to_vec(&ErrorBody::new(code, message).with_detail(detail))
3546 .map_err(SubcModuleError::Json)?;
3547 ctx.send_frame(FrameType::Error, data_flags(), body).await
3548 }
3549 HandlerOutcome::Streamed => {
3550 ctx.send_frame(FrameType::StreamEnd, data_flags(), Vec::new())
3551 .await
3552 }
3553 }
3554}
3555
3556async fn catch_handler_panic<F: Future>(future: F) -> Result<F::Output, ()> {
3559 let mut future = std::pin::pin!(future);
3560 std::future::poll_fn(
3561 |cx| match catch_unwind(AssertUnwindSafe(|| future.as_mut().poll(cx))) {
3562 Ok(poll) => poll.map(Ok),
3563 Err(_) => std::task::Poll::Ready(Err(())),
3564 },
3565 )
3566 .await
3567}
3568
3569fn bounded_terminal(
3570 ty: FrameType,
3571 flags: Flags,
3572 body: Vec<u8>,
3573) -> Result<(FrameType, Flags, Vec<u8>), SubcModuleError> {
3574 let size = subc_protocol::HEADER_LEN + body.len();
3575 let limit = subc_protocol::MAX_DELIVERED_FRAME_LEN;
3576 if matches!(ty, FrameType::Response | FrameType::Error) && size > limit {
3577 let body = serde_json::to_vec(
3578 &ErrorBody::new("frame_too_large", "module reply exceeds frame limit")
3579 .with_detail(serde_json::json!({ "size": size, "limit": limit })),
3580 )?;
3581 return Ok((FrameType::Error, flags, body));
3583 }
3584 Ok((ty, flags, body))
3585}
3586
3587fn handle_cancel(frame: Frame, in_flight: &InFlight) -> Result<(), SubcModuleError> {
3588 let guard = lock_in_flight(in_flight)?;
3589 if let Some(cancellation) =
3590 guard.get(&(frame.header.channel, frame.header.epoch, frame.header.corr))
3591 {
3592 cancellation.cancel();
3593 }
3594 Ok(())
3595}
3596
3597fn cancel_handle(in_flight: &InFlight, handle: RouteHandle) -> Result<(), SubcModuleError> {
3598 let cancelled = {
3599 let mut guard = lock_in_flight(in_flight)?;
3600 let keys = guard
3601 .keys()
3602 .copied()
3603 .filter(|(channel, epoch, _)| *channel == handle.channel && *epoch == handle.epoch)
3604 .collect::<Vec<_>>();
3605 keys.into_iter()
3606 .filter_map(|key| guard.remove(&key))
3607 .collect::<Vec<_>>()
3608 };
3609 for cancellation in cancelled {
3610 cancellation.cancel();
3611 }
3612 Ok(())
3613}
3614
3615async fn handle_control_request<H>(
3616 frame: Frame,
3617 egress: &mpsc::Sender<Frame>,
3618 handler: Arc<H>,
3619 dispatcher: RequestDispatcher,
3620 module_handle: ModuleHandle,
3621) -> Result<(), SubcModuleError>
3622where
3623 H: ModuleHandler,
3624{
3625 if control_ops::request(
3626 &frame,
3627 egress,
3628 Arc::clone(&handler),
3629 &dispatcher,
3630 &module_handle,
3631 )
3632 .await?
3633 {
3634 return Ok(());
3635 }
3636 if config::review(
3637 &frame,
3638 egress,
3639 Arc::clone(&handler),
3640 &dispatcher,
3641 &module_handle,
3642 ) {
3643 return Ok(());
3644 }
3645 let request = match serde_json::from_slice::<ModuleControlRequest>(&frame.body) {
3651 Ok(request) => request,
3652 Err(error) => {
3653 let body = serde_json::to_vec(&ErrorBody::new(
3654 "invalid_request",
3655 format!("control request could not be decoded by this module: {error}"),
3656 ))
3657 .map_err(SubcModuleError::Json)?;
3658 let refusal = Frame::build_with_version(
3659 frame.header.ver,
3660 FrameType::Error,
3661 control_flags(),
3662 0,
3663 0,
3664 frame.header.corr,
3665 body,
3666 )
3667 .map_err(SubcModuleError::FrameBuild)?;
3668 send_outbound(egress, refusal).await?;
3669 return Ok(());
3670 }
3671 };
3672 match request {
3673 ModuleControlRequest::RouteBind {
3674 route_channel,
3675 epoch,
3676 target,
3677 identity,
3678 principal,
3679 consumer_capabilities,
3680 role_versions,
3681 admission_facts,
3682 scope,
3683 call_mode,
3684 } => {
3685 if let Some(stale) = module_handle.installed_route(route_channel)? {
3692 if epoch <= stale.epoch {
3693 let body = serde_json::to_vec(&ErrorBody::new(
3694 "route_rejected",
3695 format!(
3696 "route.bind epoch {epoch} does not supersede installed epoch {} on channel {route_channel}",
3697 stale.epoch
3698 ),
3699 ))
3700 .map_err(SubcModuleError::Json)?;
3701 let reject = Frame::build_with_version(
3702 frame.header.ver,
3703 FrameType::Error,
3704 control_flags(),
3705 0,
3706 0,
3707 frame.header.corr,
3708 body,
3709 )
3710 .map_err(SubcModuleError::FrameBuild)?;
3711 send_outbound(egress, reject).await?;
3712 return Ok(());
3713 }
3714 if module_handle.remove_route(stale)? {
3715 cancel_handle(&dispatcher.in_flight, stale)?;
3716 handler.on_route_gone(&stale).await;
3717 }
3718 }
3719 let handle = module_handle.route_handle(route_channel, epoch);
3720 let peer =
3724 match serde_json::from_slice::<subc_protocol::peer::RouteBindPeer>(&frame.body) {
3725 Ok(peer) => peer,
3726 Err(error) => {
3727 tracing::warn!(
3728 channel = route_channel,
3729 epoch,
3730 %error,
3731 "route.bind peer keys did not decode; treating them as absent"
3732 );
3733 subc_protocol::peer::RouteBindPeer::new()
3734 }
3735 };
3736 let req = RouteBindRequest {
3737 handle,
3738 target,
3739 identity,
3740 principal,
3741 consumer_capabilities,
3742 role_versions,
3743 admission_facts,
3744 call_mode: call_mode.unwrap_or_else(|| derive_call_mode(scope.as_ref())),
3745 scope,
3746 peer,
3747 };
3748 let decision = handler.on_bind(&req).await;
3749 match decision.kind {
3750 BindDecisionKind::Accept => {
3751 let response = match serde_json::to_vec(&ModuleControlResponse::RouteBindAck {})
3752 .map_err(SubcModuleError::Json)
3753 .and_then(|body| {
3754 Frame::build_with_version(
3755 frame.header.ver,
3756 FrameType::Response,
3757 control_flags(),
3758 0,
3759 0,
3760 frame.header.corr,
3761 body,
3762 )
3763 .map_err(SubcModuleError::FrameBuild)
3764 }) {
3765 Ok(response) => response,
3766 Err(err) => {
3767 handler.on_route_gone(&handle).await;
3768 return Err(err);
3769 }
3770 };
3771 if let Err(err) = send_outbound(egress, response).await {
3772 handler.on_route_gone(&handle).await;
3773 return Err(err);
3774 }
3775 if let Err(err) = module_handle.install_route(handle) {
3776 handler.on_route_gone(&handle).await;
3777 return Err(err);
3778 }
3779 handler.on_bound(&handle).await;
3780 }
3781 BindDecisionKind::Reject {
3782 code,
3783 message,
3784 detail,
3785 } => {
3786 let body = match detail {
3787 Some(detail) => ErrorBody::new(code, message).with_detail(detail),
3788 None => ErrorBody::new(code, message),
3789 };
3790 let result = serde_json::to_vec(&body)
3791 .map_err(SubcModuleError::Json)
3792 .and_then(|body| {
3793 Frame::build_with_version(
3794 frame.header.ver,
3795 FrameType::Error,
3796 control_flags(),
3797 0,
3798 0,
3799 frame.header.corr,
3800 body,
3801 )
3802 .map_err(SubcModuleError::FrameBuild)
3803 });
3804 let result = match result {
3805 Ok(response) => send_outbound(egress, response).await,
3806 Err(err) => Err(err),
3807 };
3808 handler.on_route_gone(&handle).await;
3809 result?;
3810 }
3811 }
3812 }
3813 ModuleControlRequest::HealthCheck {} => {
3814 spawn_health_request(frame, egress.clone(), handler, dispatcher)?;
3815 }
3816 }
3817 Ok(())
3818}
3819
3820async fn send_hello(
3821 egress: &mpsc::Sender<Frame>,
3822 mut manifest: ModuleManifest,
3823 config_gate: bool,
3824 extra_control_ops: &[String],
3825) -> Result<(), SubcModuleError> {
3826 let launch_nonce = if registration::supervised() {
3827 None
3828 } else {
3829 match retained_launch_nonce() {
3830 Some(retained) => Some(retained),
3831 None => launch_nonce()
3832 .map_err(SubcModuleError::LaunchNonce)?
3833 .map(|nonce| nonce.value().to_string()),
3834 }
3835 };
3836 stamp_launch_nonce_source(&mut manifest, launch_nonce.as_deref());
3837 let mut control_ops = vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()];
3838 if config_gate {
3839 control_ops.push(config::CONFIG_REVIEW_OP.to_owned());
3840 }
3841 control_ops.extend_from_slice(extra_control_ops);
3842 let body = serde_json::to_vec(&ModuleHelloBody {
3843 manifest,
3844 protocol_ver: PROTOCOL_VERSION,
3845 control_ops: Some(control_ops),
3846 launch_nonce,
3847 })
3848 .map_err(SubcModuleError::Json)?;
3849 let frame = Frame::build(FrameType::Hello, control_flags(), 0, 0, HELLO_CORR, body)
3850 .map_err(SubcModuleError::FrameBuild)?;
3851 send_outbound(egress, frame).await
3852}
3853
3854async fn expect_hello_ack<R>(reader: &mut R) -> Result<ModuleHelloAckBody, SubcModuleError>
3855where
3856 R: AsyncRead + Unpin,
3857{
3858 let Some(frame) = read_frame(reader).await.map_err(SubcModuleError::FrameIo)? else {
3859 return Err(SubcModuleError::ConnectionClosedBeforeHelloAck);
3860 };
3861 match frame.header.ty {
3862 FrameType::HelloAck => serde_json::from_slice(&frame.body).map_err(SubcModuleError::Json),
3863 FrameType::Error => {
3864 let body =
3865 serde_json::from_slice::<ErrorBody>(&frame.body).map_err(SubcModuleError::Json)?;
3866 Err(SubcModuleError::HelloRejected { body })
3867 }
3868 ty => Err(SubcModuleError::UnexpectedHelloAck { ty }),
3869 }
3870}
3871
3872pub trait ModuleStream: AsyncRead + AsyncWrite + Unpin + Send {}
3874impl<T: AsyncRead + AsyncWrite + Unpin + Send> ModuleStream for T {}
3875
3876async fn connect_to_subc(
3877 connection_file_path: &Path,
3878 registration_channel: bool,
3879) -> Result<Box<dyn ModuleStream>, SubcModuleError> {
3880 let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
3881 SubcModuleError::ConnectionFile {
3882 path: connection_file_path.to_path_buf(),
3883 source,
3884 }
3885 })?;
3886 if let Some(mut stream) = if registration_channel {
3887 registration::take_stream()?
3888 } else {
3889 None
3890 } {
3891 authenticate_client(&mut stream, &conn, AUTH_DEADLINE)
3892 .await
3893 .map_err(|source| SubcModuleError::Auth {
3894 path: connection_file_path.to_path_buf(),
3895 endpoint: "registration channel".into(),
3896 source,
3897 })?;
3898 return Ok(stream);
3899 }
3900 let endpoint = conn
3901 .endpoints
3902 .first()
3903 .ok_or_else(|| SubcModuleError::NoEndpoint {
3904 path: connection_file_path.to_path_buf(),
3905 })?;
3906 let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
3907 let mut stream = connect_endpoint(&endpoint.host, endpoint.port)
3908 .await
3909 .map_err(|source| SubcModuleError::Connect {
3910 path: connection_file_path.to_path_buf(),
3911 endpoint: endpoint_label.clone(),
3912 source,
3913 })?;
3914 let _ = stream.set_nodelay(true);
3924 authenticate_client(&mut stream, &conn, AUTH_DEADLINE)
3925 .await
3926 .map_err(|source| SubcModuleError::Auth {
3927 path: connection_file_path.to_path_buf(),
3928 endpoint: endpoint_label,
3929 source,
3930 })?;
3931 Ok(Box::new(stream))
3932}
3933
3934async fn connect_endpoint(host: &str, port: u16) -> io::Result<TcpStream> {
3935 let host = host
3936 .strip_prefix('[')
3937 .and_then(|h| h.strip_suffix(']'))
3938 .unwrap_or(host);
3939 TcpStream::connect((host, port)).await
3940}
3941
3942async fn send_goodbye(tx: &mpsc::Sender<Frame>) -> Result<(), SubcModuleError> {
3943 let frame = Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 0, Vec::new())
3944 .map_err(SubcModuleError::FrameBuild)?;
3945 send_outbound(tx, frame).await
3946}
3947
3948fn is_disconnect(error: &io::Error) -> bool {
3949 matches!(
3950 error.kind(),
3951 io::ErrorKind::ConnectionReset
3952 | io::ErrorKind::ConnectionAborted
3953 | io::ErrorKind::BrokenPipe
3954 )
3955}
3956
3957fn writer_ended(
3958 result: Result<Result<(), WriterFailure>, tokio::task::JoinError>,
3959) -> SubcModuleError {
3960 match result {
3961 Ok(Ok(())) => SubcModuleError::WriterClosed,
3962 Ok(Err(error)) => SubcModuleError::FrameIo(error.error),
3963 Err(error) => SubcModuleError::WriterTask(error),
3964 }
3965}
3966
3967fn writer_connection_end(
3968 result: Result<Result<(), WriterFailure>, tokio::task::JoinError>,
3969) -> Result<ConnectionEnd, SubcModuleError> {
3970 match writer_ended(result) {
3971 SubcModuleError::FrameIo(FrameIoError::Io(error)) if is_disconnect(&error) => {
3972 Ok(ConnectionEnd::Reset)
3973 }
3974 error => Err(error),
3975 }
3976}
3977
3978async fn finish_writer(
3979 mut writer: tokio::task::JoinHandle<Result<(), WriterFailure>>,
3980 deadline: Instant,
3981) -> (Result<(), SubcModuleError>, usize) {
3982 match timeout_at(deadline, &mut writer).await {
3983 Ok(Ok(Ok(()))) => (Ok(()), 0),
3984 Ok(Ok(Err(failure))) => (
3985 Err(SubcModuleError::FrameIo(failure.error)),
3986 failure.undelivered_replies,
3987 ),
3988 Ok(Err(error)) => (Err(SubcModuleError::WriterTask(error)), 0),
3989 Err(_) => {
3990 writer.abort();
3991 let _ = writer.await;
3992 (Ok(()), 0)
3993 }
3994 }
3995}
3996
3997struct WriterFailure {
3998 error: FrameIoError,
3999 undelivered_replies: usize,
4000}
4001
4002async fn finish_writer_and_drain<R>(
4007 writer: tokio::task::JoinHandle<Result<(), WriterFailure>>,
4008 reader: &mut R,
4009 deadline: Instant,
4010) -> (Result<(), SubcModuleError>, usize)
4011where
4012 R: AsyncRead + Unpin,
4013{
4014 let flushed = finish_writer(writer, deadline).await;
4015 if flushed.0.is_ok() {
4016 drain_until_peer_closes(reader, deadline).await;
4017 }
4018 flushed
4019}
4020
4021async fn drain_until_peer_closes<R>(reader: &mut R, deadline: Instant)
4039where
4040 R: AsyncRead + Unpin,
4041{
4042 if Instant::now() >= deadline {
4043 return;
4044 }
4045 let mut discard = [0u8; 4096];
4046 let _ = timeout_at(deadline, async {
4047 while let Ok(read) = reader.read(&mut discard).await {
4048 if read == 0 {
4049 break;
4050 }
4051 }
4052 })
4053 .await;
4054}
4055
4056fn is_reply(frame: &Frame) -> bool {
4057 matches!(
4058 frame.header.ty,
4059 FrameType::Response | FrameType::Error | FrameType::StreamEnd
4060 )
4061}
4062
4063async fn drain_writer<W>(write_half: W, mut rx: mpsc::Receiver<Frame>) -> Result<(), WriterFailure>
4064where
4065 W: AsyncWrite + Unpin,
4066{
4067 let mut writer = BufWriter::new(write_half);
4068 let mut pending_replies = 0;
4069 let result = async {
4070 while let Some(frame) = rx.recv().await {
4071 pending_replies += usize::from(is_reply(&frame));
4072 write_frame(&mut writer, &frame).await?;
4073 while let Ok(frame) = rx.try_recv() {
4074 pending_replies += usize::from(is_reply(&frame));
4075 write_frame(&mut writer, &frame).await?;
4076 }
4077 writer.flush().await.map_err(FrameIoError::Io)?;
4078 pending_replies = 0;
4079 }
4080 writer.flush().await.map_err(FrameIoError::Io)?;
4081 let _ = writer.shutdown().await;
4090 Ok(())
4091 }
4092 .await;
4093 result.map_err(|error| {
4094 rx.close();
4095 while let Ok(frame) = rx.try_recv() {
4096 pending_replies += usize::from(is_reply(&frame));
4097 }
4098 WriterFailure {
4099 error,
4100 undelivered_replies: pending_replies,
4101 }
4102 })
4103}
4104
4105async fn send_outbound(egress: &mpsc::Sender<Frame>, frame: Frame) -> Result<(), SubcModuleError> {
4106 egress
4107 .send(frame)
4108 .await
4109 .map_err(|_| SubcModuleError::WriterClosed)
4110}
4111
4112fn parse_subc_arg(args: impl IntoIterator<Item = OsString>) -> Result<PathBuf, SubcModuleError> {
4113 let mut args = args.into_iter();
4114 while let Some(arg) = args.next() {
4115 if arg == "--subc" {
4116 let value = args.next().ok_or(SubcModuleError::MissingSubcValue)?;
4117 return Ok(PathBuf::from(value));
4118 }
4119 if let Some(raw) = arg.to_str().and_then(|arg| arg.strip_prefix("--subc=")) {
4120 if raw.is_empty() {
4121 return Err(SubcModuleError::MissingSubcValue);
4122 }
4123 return Ok(PathBuf::from(raw));
4124 }
4125 }
4126 Err(SubcModuleError::MissingSubcArg)
4127}
4128
4129fn module_id_from_env() -> Result<Option<String>, SubcModuleError> {
4130 match env::var(SUBC_MODULE_ID_ENV) {
4131 Ok(value) if !value.trim().is_empty() => Ok(Some(value)),
4132 Ok(_) => Err(SubcModuleError::EmptyModuleIdEnv),
4133 Err(env::VarError::NotPresent) => Ok(None),
4134 Err(env::VarError::NotUnicode(value)) => {
4135 Err(SubcModuleError::NonUnicodeModuleIdEnv { value })
4136 }
4137 }
4138}
4139
4140fn lock_in_flight(
4141 in_flight: &InFlight,
4142) -> Result<std::sync::MutexGuard<'_, HashMap<RequestKey, CancellationToken>>, SubcModuleError> {
4143 in_flight
4144 .lock()
4145 .map_err(|_| SubcModuleError::InFlightPoisoned)
4146}
4147
4148fn control_flags() -> Flags {
4149 Flags::new(false, Priority::Passive, false)
4150}
4151
4152fn data_flags() -> Flags {
4153 Flags::new(false, Priority::Interactive, false)
4154}
4155
4156#[derive(Debug)]
4157#[non_exhaustive]
4158pub enum SubcModuleError {
4159 InvalidControlOp(String),
4161 DuplicateControlCorrelation(u64),
4163 RegistrationChannel(io::Error),
4164 MissingSubcArg,
4165 MissingSubcValue,
4166 EmptyModuleIdEnv,
4167 NonUnicodeModuleIdEnv {
4168 value: OsString,
4169 },
4170 ConnectionFile {
4171 path: PathBuf,
4172 source: ConnectionFileError,
4173 },
4174 NoEndpoint {
4175 path: PathBuf,
4176 },
4177 Connect {
4178 path: PathBuf,
4179 endpoint: String,
4180 source: io::Error,
4181 },
4182 Auth {
4183 path: PathBuf,
4184 endpoint: String,
4185 source: AuthError,
4186 },
4187 FrameIo(FrameIoError),
4188 FrameBuild(FrameBuildError),
4189 Json(serde_json::Error),
4190 WriterClosed,
4191 HandlerPanic,
4193 StaleRouteHandle(RouteHandle),
4194 ConnectionTokenExhausted,
4195 WriterTask(tokio::task::JoinError),
4196 InFlightPoisoned,
4197 ConnectionClosedBeforeHelloAck,
4198 UnexpectedHelloAck {
4199 ty: FrameType,
4200 },
4201 HelloRejected {
4202 body: ErrorBody,
4203 },
4204 LaunchNonce(launch_nonce::LaunchNonceError),
4207}
4208
4209impl fmt::Display for SubcModuleError {
4210 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4211 match self {
4212 Self::InvalidControlOp(op) => {
4213 write!(f, "invalid or reserved control operation: {op:?}")
4214 }
4215 Self::DuplicateControlCorrelation(corr) => {
4216 write!(f, "duplicate in-flight control correlation: {corr}")
4217 }
4218 Self::RegistrationChannel(error) => write!(f, "registration channel: {error}"),
4219 Self::MissingSubcArg => write!(f, "missing required --subc <connection-file> argument"),
4220 Self::MissingSubcValue => write!(f, "--subc requires a connection-file path value"),
4221 Self::EmptyModuleIdEnv => write!(f, "{SUBC_MODULE_ID_ENV} must not be empty when set"),
4222 Self::NonUnicodeModuleIdEnv { value } => write!(
4223 f,
4224 "{SUBC_MODULE_ID_ENV} must be valid UTF-8, got '{}'",
4225 value.to_string_lossy()
4226 ),
4227 Self::ConnectionFile { path, source } => write!(
4228 f,
4229 "failed to read subc connection file '{}': {source}",
4230 path.display()
4231 ),
4232 Self::NoEndpoint { path } => write!(
4233 f,
4234 "subc connection file '{}' has no endpoints",
4235 path.display()
4236 ),
4237 Self::Connect {
4238 path,
4239 endpoint,
4240 source,
4241 } => write!(
4242 f,
4243 "failed to connect to subc endpoint {endpoint} from '{}': {source}",
4244 path.display()
4245 ),
4246 Self::Auth {
4247 path,
4248 endpoint,
4249 source,
4250 } => write!(
4251 f,
4252 "failed to authenticate to subc endpoint {endpoint} from '{}': {source}",
4253 path.display()
4254 ),
4255 Self::FrameIo(err) => fmt::Display::fmt(err, f),
4256 Self::FrameBuild(err) => write!(f, "frame build error: {err}"),
4257 Self::Json(err) => write!(f, "JSON error: {err}"),
4258 Self::WriterClosed => write!(f, "module writer task closed"),
4259 Self::HandlerPanic => write!(f, "module handler panicked"),
4260 Self::StaleRouteHandle(handle) => write!(f, "stale route handle: {handle:?}"),
4261 Self::ConnectionTokenExhausted => write!(f, "module connection token exhausted"),
4262 Self::WriterTask(err) => write!(f, "module writer task failed: {err}"),
4263 Self::InFlightPoisoned => write!(f, "in-flight registry lock poisoned"),
4264 Self::ConnectionClosedBeforeHelloAck => write!(f, "connection closed before HELLO_ACK"),
4265 Self::UnexpectedHelloAck { ty } => write!(f, "expected HELLO_ACK, got {ty:?}"),
4266 Self::HelloRejected { body } => write!(
4267 f,
4268 "HELLO rejected by subc: {} ({})",
4269 body.code, body.message
4270 ),
4271 Self::LaunchNonce(err) => write!(f, "launch nonce unavailable: {err}"),
4272 }
4273 }
4274}
4275
4276impl Error for SubcModuleError {
4277 fn source(&self) -> Option<&(dyn Error + 'static)> {
4278 match self {
4279 Self::RegistrationChannel(error) => Some(error),
4280 Self::ConnectionFile { source, .. } => Some(source),
4281 Self::Connect { source, .. } => Some(source),
4282 Self::Auth { source, .. } => Some(source),
4283 Self::FrameIo(err) => Some(err),
4284 Self::FrameBuild(err) => Some(err),
4285 Self::Json(err) => Some(err),
4286 Self::LaunchNonce(err) => Some(err),
4287 Self::WriterTask(err) => Some(err),
4288 Self::MissingSubcArg
4289 | Self::InvalidControlOp(_)
4290 | Self::DuplicateControlCorrelation(_)
4291 | Self::MissingSubcValue
4292 | Self::EmptyModuleIdEnv
4293 | Self::NonUnicodeModuleIdEnv { .. }
4294 | Self::NoEndpoint { .. }
4295 | Self::WriterClosed
4296 | Self::HandlerPanic
4297 | Self::StaleRouteHandle(_)
4298 | Self::ConnectionTokenExhausted
4299 | Self::InFlightPoisoned
4300 | Self::ConnectionClosedBeforeHelloAck
4301 | Self::UnexpectedHelloAck { .. }
4302 | Self::HelloRejected { .. } => None,
4303 }
4304 }
4305}
4306
4307impl From<serde_json::Error> for SubcModuleError {
4308 fn from(err: serde_json::Error) -> Self {
4309 Self::Json(err)
4310 }
4311}
4312
4313#[cfg(test)]
4314mod call_mode_tests;
4315
4316#[cfg(test)]
4317mod tests {
4318 use std::sync::atomic::{AtomicUsize, Ordering};
4319
4320 use serde_json::json;
4321
4322 use subc_protocol::manifest::{Concurrency, ExecutionMode, IdentityScope, Tool};
4323 use tokio::sync::Semaphore;
4324 use tokio::time::{advance, timeout};
4325
4326 use super::*;
4327
4328 struct EchoHandler;
4329
4330 #[async_trait]
4331 impl ModuleHandler for EchoHandler {
4332 async fn handle(&self, _ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome {
4333 HandlerOutcome::Response(body)
4334 }
4335 }
4336
4337 struct BlockingHandler {
4338 entered: Arc<AtomicUsize>,
4339 release: Semaphore,
4340 report: HealthReport,
4341 }
4342
4343 #[async_trait]
4344 impl ModuleHandler for BlockingHandler {
4345 async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
4346 self.entered.fetch_add(1, Ordering::SeqCst);
4347 self.release.acquire().await.unwrap().forget();
4348 HandlerOutcome::Streamed
4349 }
4350
4351 async fn health(&self) -> HealthReport {
4352 self.report.clone()
4353 }
4354 }
4355
4356 fn health_request(corr: u64) -> Frame {
4357 Frame::build(
4358 FrameType::Request,
4359 control_flags(),
4360 0,
4361 0,
4362 corr,
4363 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).unwrap(),
4364 )
4365 .unwrap()
4366 }
4367
4368 fn data_request(channel: u16, corr: u64) -> Frame {
4369 Frame::build(
4370 FrameType::Request,
4371 data_flags(),
4372 channel,
4373 1,
4374 corr,
4375 b"opaque".to_vec(),
4376 )
4377 .unwrap()
4378 }
4379
4380 fn catalog_update_response(corr: u64) -> Frame {
4381 Frame::build(
4382 FrameType::Response,
4383 control_flags(),
4384 0,
4385 0,
4386 corr,
4387 serde_json::to_vec(&ModuleControlResponseToModule::CatalogUpdate {}).unwrap(),
4388 )
4389 .unwrap()
4390 }
4391
4392 fn test_module_handle(subc_ops: &[&str]) -> (ModuleHandle, mpsc::Receiver<Frame>) {
4393 let (tx, rx) = mpsc::channel(4);
4394 let ack = ModuleHelloAckBody {
4395 negotiated_ver: PROTOCOL_VERSION,
4396 subc_ops: subc_ops.iter().map(|op| (*op).to_string()).collect(),
4397 subc_capabilities: Vec::new(),
4398 storage: None,
4399 machine_id: None,
4400 };
4401 (
4402 ModuleHandle::new(
4403 &ack,
4404 tx,
4405 1,
4406 CancellationToken::new(),
4407 tokio::runtime::Handle::current(),
4408 ),
4409 rx,
4410 )
4411 }
4412
4413 #[test]
4414 fn module_handle_exposes_the_acked_machine_id_and_none_otherwise() {
4415 let runtime = tokio::runtime::Runtime::new().unwrap();
4416 let handle_for = |machine_id: Option<&str>| {
4417 let (tx, _rx) = mpsc::channel(1);
4418 let ack = ModuleHelloAckBody {
4419 negotiated_ver: PROTOCOL_VERSION,
4420 subc_ops: Vec::new(),
4421 subc_capabilities: Vec::new(),
4422 storage: None,
4423 machine_id: machine_id.map(str::to_owned),
4424 };
4425 ModuleHandle::new(
4426 &ack,
4427 tx,
4428 1,
4429 CancellationToken::new(),
4430 runtime.handle().clone(),
4431 )
4432 };
4433 assert_eq!(
4434 handle_for(Some("0123456789abcdef0123456789abcdef"))
4435 .machine_id()
4436 .map(MachineId::as_str),
4437 Some("0123456789abcdef0123456789abcdef")
4438 );
4439 assert_eq!(handle_for(None).machine_id(), None);
4441 assert_eq!(handle_for(Some("not-a-machine-id")).machine_id(), None);
4442 }
4443
4444 fn test_provider_role(tool_names: &[&str]) -> ProviderRole {
4445 ProviderRole::ToolProvider {
4446 tools: tool_names
4447 .iter()
4448 .map(|name| Tool {
4449 name: (*name).to_string(),
4450 description: None,
4451 execution_mode: ExecutionMode::Pure,
4452 schema: json!({"type": "object"}),
4453 })
4454 .collect(),
4455 identity_scope: vec![IdentityScope::Project],
4456 concurrency: Concurrency::ModuleManaged,
4457 emits_push: false,
4458 sub_supervises: false,
4459 }
4460 }
4461
4462 #[tokio::test]
4463 async fn catalog_update_fails_fast_when_hello_ack_does_not_advertise_support() {
4464 let (handle, mut rx) = test_module_handle(&[]);
4465
4466 let error = handle
4467 .catalog_update(vec![test_provider_role(&["a"])])
4468 .await
4469 .unwrap_err();
4470 assert!(matches!(error, CatalogUpdateError::NotSupported));
4471 assert!(matches!(
4472 handle.set_ready(true).await,
4473 Err(CatalogUpdateError::NotSupported)
4474 ));
4475 assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
4476 }
4477
4478 #[tokio::test]
4479 async fn catalog_update_demuxes_multiple_in_flight_requests() {
4480 let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
4481 let first_handle = handle.clone();
4482 let second_handle = handle.clone();
4483 let first = tokio::spawn(async move {
4484 first_handle
4485 .catalog_update(vec![test_provider_role(&["a"])])
4486 .await
4487 });
4488 let second = tokio::spawn(async move {
4489 second_handle
4490 .catalog_update(vec![test_provider_role(&["b"])])
4491 .await
4492 });
4493
4494 let first_frame = timeout(Duration::from_secs(1), rx.recv())
4495 .await
4496 .unwrap()
4497 .unwrap();
4498 let second_frame = timeout(Duration::from_secs(1), rx.recv())
4499 .await
4500 .unwrap()
4501 .unwrap();
4502 assert_eq!(first_frame.header.ty, FrameType::Request);
4503 assert_eq!(first_frame.header.channel, 0);
4504 assert_eq!(second_frame.header.ty, FrameType::Request);
4505 assert_eq!(second_frame.header.channel, 0);
4506 assert_ne!(first_frame.header.corr, second_frame.header.corr);
4507
4508 let (earlier, later) =
4509 if json_body(&first_frame.body)["provides"] == json!([test_provider_role(&["a"])]) {
4510 (first, second)
4511 } else {
4512 (second, first)
4513 };
4514 assert!(handle.handle_control_reply(catalog_update_response(second_frame.header.corr)));
4515 assert!(later.await.unwrap().is_ok());
4516 assert!(handle.handle_control_reply(catalog_update_response(first_frame.header.corr)));
4517 assert!(earlier.await.unwrap().is_ok());
4518
4519 let expected = json_body(&second_frame.body)["provides"].clone();
4520 let (result, ()) = tokio::join!(handle.set_ready(true), async {
4521 let frame = rx.recv().await.unwrap();
4522 assert_eq!(json_body(&frame.body)["provides"], expected);
4523 assert_eq!(json_body(&frame.body)["ready"], true);
4524 handle.handle_control_reply(catalog_update_response(frame.header.corr));
4525 });
4526 result.unwrap();
4527 }
4528
4529 fn json_body(body: &[u8]) -> serde_json::Value {
4530 serde_json::from_slice(body).unwrap()
4531 }
4532
4533 #[tokio::test]
4534 async fn role_declarations_update_requires_its_own_advertised_op() {
4535 for ops in [vec![], vec![MODULE_TO_SUBC_OP_CATALOG_UPDATE]] {
4536 let (handle, mut rx) = test_module_handle(&ops);
4537 assert!(matches!(
4538 handle.update_role_declarations(BTreeMap::new()).await,
4539 Err(CatalogUpdateError::NotSupported)
4540 ));
4541 assert!(matches!(
4542 rx.try_recv(),
4543 Err(mpsc::error::TryRecvError::Empty)
4544 ));
4545 }
4546 }
4547
4548 #[tokio::test]
4549 async fn role_declarations_update_sends_map_and_preserves_refusal() {
4550 let (handle, mut rx) =
4551 test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS]);
4552 let initial_roles = vec![test_provider_role(&["original"])];
4553 handle.shared.lock_inner().current_provides = Some(initial_roles.clone());
4554 for declarations in [
4555 json!({"media-provider/v1": {"engines": ["image"]}}),
4556 json!({}),
4557 ] {
4558 let map = serde_json::from_value(declarations.clone()).unwrap();
4559 let (result, ()) = tokio::join!(handle.update_role_declarations(map), async {
4560 let frame = rx.recv().await.unwrap();
4561 assert_eq!(frame.header.ty, FrameType::Request);
4562 assert_eq!(frame.header.channel, 0);
4563 assert_eq!(
4564 json_body(&frame.body),
4565 json!({"op": "catalog.update_role_declarations", "role_declarations": declarations})
4566 );
4567 handle.handle_control_reply(control_reply(
4568 FrameType::Response,
4569 frame.header.corr,
4570 serde_json::to_vec(&json!({"op": "catalog.update_role_declarations"})).unwrap(),
4571 ));
4572 });
4573 result.unwrap();
4574 }
4575 let (result, ()) = tokio::join!(handle.update_role_declarations(BTreeMap::new()), async {
4576 let frame = rx.recv().await.unwrap();
4577 handle.handle_control_reply(control_reply(FrameType::Error, frame.header.corr,
4578 serde_json::to_vec(&json!({"code": "invalid_role_declarations", "message": "role_declarations key must use capability-name grammar"})).unwrap()));
4579 });
4580 let Err(CatalogUpdateError::Rejected(body)) = result else {
4581 panic!("expected typed refusal")
4582 };
4583 assert_eq!(body.code, "invalid_role_declarations");
4584 assert_eq!(
4585 body.message,
4586 "role_declarations key must use capability-name grammar"
4587 );
4588 assert_eq!(
4589 handle.shared.lock_inner().current_provides,
4590 Some(initial_roles)
4591 );
4592 }
4593
4594 async fn ack_catalog_update(
4595 handle: &ModuleHandle,
4596 rx: &mut mpsc::Receiver<Frame>,
4597 provides: &[ProviderRole],
4598 ready: Option<bool>,
4599 capabilities: Option<&CapabilityDeclarations>,
4600 ) {
4601 let frame = rx.recv().await.unwrap();
4602 let mut expected = json!({"op": "catalog.update", "provides": provides});
4603 if let Some(ready) = ready {
4604 expected["ready"] = json!(ready);
4605 }
4606 if let Some(capabilities) = capabilities {
4607 expected["capabilities"] = json!(capabilities);
4608 }
4609 assert_eq!(json_body(&frame.body), expected);
4610 let mut wire = format!(
4611 r#"{{"op":"catalog.update","provides":{}"#,
4612 serde_json::to_string(provides).unwrap()
4613 );
4614 if let Some(capabilities) = capabilities {
4615 wire.push_str(&format!(
4616 r#","capabilities":{}"#,
4617 serde_json::to_string(capabilities).unwrap()
4618 ));
4619 }
4620 if let Some(ready) = ready {
4621 wire.push_str(&format!(r#","ready":{ready}"#));
4622 }
4623 wire.push('}');
4624 assert_eq!(frame.body, wire.as_bytes());
4625 assert!(handle.handle_control_reply(catalog_update_response(frame.header.corr)));
4626 }
4627
4628 #[tokio::test]
4629 async fn catalog_update_set_ready_preserves_registered_and_replaced_roles() {
4630 let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
4631 let initial = vec![test_provider_role(&["initial", "second"])];
4632 handle.shared.lock_inner().current_provides = Some(initial.clone());
4633 let (result, ()) = tokio::join!(
4634 handle.set_ready(true),
4635 ack_catalog_update(&handle, &mut rx, &initial, Some(true), None)
4636 );
4637 result.unwrap();
4638
4639 let replaced = vec![test_provider_role(&["replacement"])];
4640 let (result, ()) = tokio::join!(
4641 handle.catalog_update(replaced.clone()),
4642 ack_catalog_update(&handle, &mut rx, &replaced, None, None)
4643 );
4644 result.unwrap();
4645 let (result, ()) = tokio::join!(
4646 handle.set_ready(false),
4647 ack_catalog_update(&handle, &mut rx, &replaced, Some(false), None)
4648 );
4649 result.unwrap();
4650
4651 let attested = vec![test_provider_role(&["attested"])];
4652 let capabilities = CapabilityDeclarations {
4653 provides: vec!["test.capability".into()],
4654 requires: Vec::new(),
4655 must_never_reach: Vec::new(),
4656 };
4657 let (result, ()) = tokio::join!(
4658 handle.catalog_update_with_capabilities(attested.clone(), capabilities.clone()),
4659 ack_catalog_update(&handle, &mut rx, &attested, None, Some(&capabilities))
4660 );
4661 result.unwrap();
4662 let (result, ()) = tokio::join!(
4663 handle.set_ready(true),
4664 ack_catalog_update(&handle, &mut rx, &attested, Some(true), None)
4665 );
4666 result.unwrap();
4667 }
4668
4669 #[tokio::test]
4670 async fn catalog_update_set_ready_refuses_unknown_or_in_flight_roles() {
4671 let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
4672 assert!(matches!(
4673 handle.set_ready(true).await,
4674 Err(CatalogUpdateError::Protocol(_))
4675 ));
4676 assert!(rx.try_recv().is_err());
4677
4678 let updating = handle.clone();
4679 let update = tokio::spawn(async move {
4680 updating
4681 .catalog_update(vec![test_provider_role(&["new"])])
4682 .await
4683 });
4684 let frame = rx.recv().await.unwrap();
4685 assert!(matches!(
4686 handle.set_ready(true).await,
4687 Err(CatalogUpdateError::Protocol(_))
4688 ));
4689 assert!(rx.try_recv().is_err());
4690 update.abort();
4691 assert!(update.await.unwrap_err().is_cancelled());
4692 assert!(!handle.handle_control_reply(catalog_update_response(frame.header.corr)));
4693 assert!(matches!(
4694 handle.set_ready(true).await,
4695 Err(CatalogUpdateError::Protocol(_))
4696 ));
4697
4698 let roles = vec![test_provider_role(&["recovered"])];
4699 let (result, ()) = tokio::join!(
4700 handle.catalog_update(roles.clone()),
4701 ack_catalog_update(&handle, &mut rx, &roles, None, None)
4702 );
4703 result.unwrap();
4704 let (result, ()) = tokio::join!(
4705 handle.set_ready(true),
4706 ack_catalog_update(&handle, &mut rx, &roles, Some(true), None)
4707 );
4708 result.unwrap();
4709 }
4710
4711 #[tokio::test(start_paused = true)]
4712 async fn catalog_update_set_ready_timeout_does_not_replay_stale_roles() {
4713 let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
4714 handle.shared.lock_inner().current_provides = Some(vec![test_provider_role(&["old"])]);
4715 assert!(matches!(
4716 handle
4717 .catalog_update(vec![test_provider_role(&["new"])])
4718 .await,
4719 Err(CatalogUpdateError::Timeout)
4720 ));
4721 let frame = rx.recv().await.unwrap();
4722 assert!(!handle.handle_control_reply(catalog_update_response(frame.header.corr)));
4723 assert!(matches!(
4724 handle.set_ready(true).await,
4725 Err(CatalogUpdateError::Protocol(_))
4726 ));
4727 assert!(rx.try_recv().is_err());
4728 }
4729
4730 #[tokio::test]
4731 async fn catalog_update_set_ready_rejection_retains_last_accepted_roles() {
4732 let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
4733 let initial = vec![test_provider_role(&["kept"])];
4734 handle.shared.lock_inner().current_provides = Some(initial.clone());
4735 let (result, ()) = tokio::join!(
4736 handle.catalog_update(vec![test_provider_role(&["rejected"])]),
4737 async {
4738 let frame = rx.recv().await.unwrap();
4739 handle.handle_control_reply(control_reply(
4740 FrameType::Error,
4741 frame.header.corr,
4742 br#"{"code":"catalog_update_frozen_field","message":"frozen"}"#.to_vec(),
4743 ));
4744 }
4745 );
4746 assert!(matches!(result, Err(CatalogUpdateError::FrozenField(_))));
4747 let (result, ()) = tokio::join!(
4748 handle.set_ready(true),
4749 ack_catalog_update(&handle, &mut rx, &initial, Some(true), None)
4750 );
4751 result.unwrap();
4752 }
4753
4754 fn control_reply(ty: FrameType, corr: u64, body: Vec<u8>) -> Frame {
4755 Frame::build(ty, control_flags(), 0, 0, corr, body).unwrap()
4756 }
4757
4758 async fn authorize_for_test(
4759 handle: &ModuleHandle,
4760 ) -> Result<RouteAuthorizeResponse, ScopeCallError> {
4761 handle
4762 .route_authorize(
4763 Principal::Direct,
4764 RouteAuthorizeScope::new(
4765 Principal::Reserved {
4766 module_id: "owner".into(),
4767 },
4768 "stored-scope",
4769 7,
4770 ),
4771 "target".into(),
4772 RouteAuthorizeExpected::new(
4773 subc_protocol::scope::ScopeKind::Head,
4774 &subc_protocol::scope::ScopeAttributes::default(),
4775 ),
4776 )
4777 .await
4778 }
4779
4780 #[tokio::test]
4781 async fn route_authorize_requires_advertised_operation() {
4782 let (tx, mut rx) = mpsc::channel(4);
4783 let ack = ModuleHelloAckBody {
4784 negotiated_ver: PROTOCOL_VERSION,
4785 subc_ops: vec![],
4786 subc_capabilities: vec![subc_protocol::scope::ROUTE_AUTHORIZE_CAPABILITY.into()],
4788 storage: None,
4789 machine_id: None,
4790 };
4791 let handle = ModuleHandle::new(
4792 &ack,
4793 tx,
4794 1,
4795 CancellationToken::new(),
4796 tokio::runtime::Handle::current(),
4797 );
4798 let result = timeout(Duration::from_millis(100), authorize_for_test(&handle))
4799 .await
4800 .expect("an unsupported operation must return without sending or awaiting a reply");
4801 assert_eq!(
4802 result,
4803 Err(ScopeCallError::NotSupported {
4804 op: subc_protocol::scope::ROUTE_AUTHORIZE_OP,
4805 })
4806 );
4807 assert!(
4808 rx.try_recv().is_err(),
4809 "unsupported calls must send nothing"
4810 );
4811 }
4812
4813 #[tokio::test]
4814 async fn route_authorize_decodes_every_answer_and_permission_refusal() {
4815 use RouteAuthorizeOutcome::*;
4816 let (handle, mut rx) = test_module_handle(&[subc_protocol::scope::ROUTE_AUTHORIZE_OP]);
4817 for (wire, outcome, code) in [
4818 ("authorized", Authorized, None),
4819 ("scope_ended", ScopeEnded, Some("scope_not_live")),
4820 ("indeterminate", Indeterminate, Some("scope_not_synced")),
4821 ("epoch_changed", EpochChanged, Some("scope_ended")),
4822 ("claim_changed", ClaimChanged, None),
4823 (
4824 "proof_required",
4825 ProofRequired,
4826 Some("scope_bind_proof_required"),
4827 ),
4828 ("not_authorized", NotAuthorized, Some("scope_not_carrier")),
4829 ("target_absent", TargetAbsent, None),
4830 ] {
4831 let caller = handle.clone();
4832 let pending = tokio::spawn(async move { authorize_for_test(&caller).await });
4833 let request = rx.recv().await.unwrap();
4834 assert_eq!(request.header.ty, FrameType::Request);
4835 assert_eq!(request.header.channel, 0);
4836 assert_eq!(request.header.ver, PROTOCOL_VERSION);
4837 assert_eq!(
4838 serde_json::from_slice::<serde_json::Value>(&request.body).unwrap(),
4839 serde_json::json!({
4840 "op": "route.authorize", "principal": {"kind": "direct"},
4841 "scope": {"owner": {"kind": "reserved", "module_id": "owner"},
4842 "ref": "stored-scope", "epoch": 7},
4843 "target": "target", "expected": {"kind": "head"}
4844 })
4845 );
4846 let content_version = (outcome == Authorized).then_some("sha256:content");
4847 let mut body = serde_json::json!({
4848 "op": "route.authorize", "outcome": wire, "message": "judgment"
4849 });
4850 if let Some(code) = code {
4851 body["route_open_code"] = code.into();
4852 }
4853 if let Some(version) = content_version {
4854 body["content_version"] = version.into();
4855 }
4856 assert!(handle.handle_control_reply(control_reply(
4857 FrameType::Response,
4858 request.header.corr,
4859 serde_json::to_vec(&body).unwrap(),
4860 )));
4861 let reply = pending.await.unwrap().unwrap();
4862 assert_eq!(reply.outcome, outcome);
4863 assert_eq!(reply.message, "judgment");
4864 assert_eq!(reply.route_open_code.as_deref(), code);
4865 assert_eq!(reply.content_version.as_deref(), content_version);
4866 }
4867
4868 for (code, message) in [
4869 ("route_authorize_not_permitted", "not the configured owner"),
4870 ("not_registered", "connection is not registered"),
4871 ("unexpected", "a future daemon call refusal"),
4872 ] {
4873 let caller = handle.clone();
4874 let pending = tokio::spawn(async move { authorize_for_test(&caller).await });
4875 let request = rx.recv().await.unwrap();
4876 assert!(handle.handle_control_reply(control_reply(
4877 FrameType::Error,
4878 request.header.corr,
4879 serde_json::to_vec(&serde_json::json!({"code": code, "message": message})).unwrap(),
4880 )));
4881 assert_eq!(
4882 pending.await.unwrap(),
4883 Err(ScopeCallError::Refused {
4884 code: code.into(),
4885 message: message.into(),
4886 })
4887 );
4888 }
4889 }
4890
4891 #[tokio::test]
4892 async fn route_authorize_rejects_malformed_and_unexpected_replies() {
4893 let (handle, mut rx) = test_module_handle(&[subc_protocol::scope::ROUTE_AUTHORIZE_OP]);
4894 for (ty, body) in [
4895 (FrameType::Response, r#"{"op":"catalog.update"}"#),
4896 (
4897 FrameType::Response,
4898 r#"{"op":"route.authorize","outcome":"authorized"}"#,
4899 ),
4900 (
4901 FrameType::Response,
4902 r#"{"op":"route.authorize","outcome":"future","message":"new"}"#,
4903 ),
4904 (FrameType::Error, "not json"),
4905 (FrameType::Request, "{}"),
4906 ] {
4907 let caller = handle.clone();
4908 let pending = tokio::spawn(async move { authorize_for_test(&caller).await });
4909 let request = rx.recv().await.unwrap();
4910 assert!(handle.handle_control_reply(control_reply(
4911 ty,
4912 request.header.corr,
4913 body.as_bytes().to_vec(),
4914 )));
4915 assert!(
4916 matches!(pending.await.unwrap(), Err(ScopeCallError::Protocol(_))),
4917 "{body}"
4918 );
4919 }
4920 }
4921
4922 #[tokio::test]
4923 async fn scope_served_by_requires_capability_and_decodes_open_replies() {
4924 let (unsupported, mut rx) = test_module_handle(&[subc_control::SCOPE_SERVED_BY_OP]);
4925 assert_eq!(
4926 unsupported
4927 .scope_served_by("s".into(), 1, "tool".into())
4928 .await,
4929 Err(ScopeCallError::NotSupported {
4930 op: subc_control::SCOPE_SERVED_BY_OP
4931 })
4932 );
4933 assert!(rx.try_recv().is_err());
4934 let (tx, mut rx) = mpsc::channel(4);
4935 let ack = ModuleHelloAckBody {
4936 negotiated_ver: PROTOCOL_VERSION,
4937 subc_ops: vec![],
4938 subc_capabilities: vec![subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.into()],
4939 storage: None,
4940 machine_id: None,
4941 };
4942 let handle = ModuleHandle::new(
4943 &ack,
4944 tx,
4945 1,
4946 CancellationToken::new(),
4947 tokio::runtime::Handle::current(),
4948 );
4949 for answer in ["served", "not_served", "unknown", "future"] {
4950 let caller = handle.clone();
4951 let pending =
4952 tokio::spawn(
4953 async move { caller.scope_served_by("s".into(), 7, "tool".into()).await },
4954 );
4955 let request = rx.recv().await.unwrap();
4956 let decoded: subc_control::ScopeServedByRequest =
4957 serde_json::from_slice(&request.body).unwrap();
4958 assert_eq!(
4959 (
4960 decoded.scope_ref.as_str(),
4961 decoded.scope_epoch,
4962 decoded.module_id.as_str()
4963 ),
4964 ("s", 7, "tool")
4965 );
4966 assert!(handle.handle_control_reply(control_reply(FrameType::Response, request.header.corr,
4967 serde_json::to_vec(&serde_json::json!({"op":"scope.served_by", "answer":answer, "daemon_incarnation":"d"})).unwrap())));
4968 let reply = pending.await.unwrap().unwrap();
4969 assert_eq!(reply.daemon_incarnation, "d");
4970 assert_eq!(serde_json::to_value(reply.answer).unwrap(), answer);
4971 }
4972 let caller = handle.clone();
4973 let pending =
4974 tokio::spawn(async move { caller.scope_served_by("s".into(), 7, "tool".into()).await });
4975 let request = rx.recv().await.unwrap();
4976 assert!(handle.handle_control_reply(control_reply(
4977 FrameType::Error,
4978 request.header.corr,
4979 serde_json::to_vec(&serde_json::json!({"code":"not_registered", "message":"gone"}))
4980 .unwrap()
4981 )));
4982 assert_eq!(
4983 pending.await.unwrap().unwrap_err().code(),
4984 Some("not_registered")
4985 );
4986 }
4987
4988 fn owner() -> Principal {
4989 Principal::Reserved {
4990 module_id: "owner".to_string(),
4991 }
4992 }
4993
4994 mod scope_watch {
4995 use super::*;
4996 include!("module_subscription_tests.rs");
4997 }
4998
4999 #[tokio::test]
5003 async fn scope_ops_fail_fast_when_hello_ack_does_not_advertise_them() {
5004 let (handle, mut rx) = test_module_handle(&[]);
5005 assert_eq!(
5006 handle.scope_sync(1, Vec::new()).await.unwrap_err(),
5007 ScopeCallError::NotSupported { op: SCOPE_SYNC_OP }
5008 );
5009 assert_eq!(
5010 handle
5011 .scope_describe(owner(), "s".to_string())
5012 .await
5013 .unwrap_err(),
5014 ScopeCallError::NotSupported {
5015 op: SCOPE_DESCRIBE_OP
5016 }
5017 );
5018 assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
5019
5020 let (describe_only, mut rx) = test_module_handle(&[SCOPE_DESCRIBE_OP]);
5021 assert_eq!(
5022 describe_only.scope_sync(1, Vec::new()).await.unwrap_err(),
5023 ScopeCallError::NotSupported { op: SCOPE_SYNC_OP }
5024 );
5025 assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
5026
5027 let (sync_only, mut rx) = test_module_handle(&[SCOPE_SYNC_OP]);
5028 assert_eq!(
5029 sync_only
5030 .scope_describe(owner(), "s".to_string())
5031 .await
5032 .unwrap_err(),
5033 ScopeCallError::NotSupported {
5034 op: SCOPE_DESCRIBE_OP
5035 }
5036 );
5037 assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
5038 }
5039
5040 #[tokio::test]
5041 async fn scope_apply_sends_nothing_without_its_own_advertised_op() {
5042 for ops in [&[][..], &[SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP][..]] {
5043 let (handle, mut rx) = test_module_handle(ops);
5044 assert_eq!(
5045 handle.scope_apply(2, Vec::new(), Vec::new()).await,
5046 Err(ScopeCallError::NotSupported { op: SCOPE_APPLY_OP })
5047 );
5048 assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
5049 }
5050 }
5051
5052 #[tokio::test]
5053 async fn scope_apply_sends_the_delta_and_preserves_per_entry_outcomes() {
5054 let (handle, mut rx) = test_module_handle(&[SCOPE_APPLY_OP]);
5055 let apply = tokio::spawn({
5056 let handle = handle.clone();
5057 async move {
5058 handle
5059 .scope_apply(
5060 2,
5061 vec![ScopeRecord::new(
5062 "run",
5063 1,
5064 subc_protocol::scope::ScopeKind::Ephemeral,
5065 )
5066 .with_expires_at_ms(Some(12345))
5067 .with_workdir(Some("/session/folder".into()))
5068 .with_project_id(Some("project:1".into()))],
5069 vec![ScopeEnd::new("old", 3), ScopeEnd::new("missing", 1)],
5070 )
5071 .await
5072 }
5073 });
5074 let request = timeout(Duration::from_secs(1), rx.recv())
5075 .await
5076 .unwrap()
5077 .unwrap();
5078 assert_eq!(request.header.ty, FrameType::Request);
5079 assert_eq!(request.header.channel, 0);
5080 assert_eq!(
5081 serde_json::from_slice::<serde_json::Value>(&request.body).unwrap(),
5082 serde_json::json!({
5083 "op": "scope.apply", "generation": 2,
5084 "upsert": [{"ref": "run", "scope_epoch": 1, "kind": "ephemeral", "expires_at_ms": 12345,
5085 "workdir": "/session/folder", "project_id": "project:1"}],
5086 "end": [{"ref": "old", "scope_epoch": 3}, {"ref": "missing", "scope_epoch": 1}]
5087 })
5088 );
5089 let body = serde_json::to_vec(&serde_json::json!({
5090 "op": "scope.apply", "generation": 2,
5091 "results": [{"ref": "run", "scope_epoch": 1, "outcome": "refused", "code": "scope_expired"}],
5092 "end_results": [
5093 {"ref": "old", "scope_epoch": 3, "outcome": "ended"},
5094 {"ref": "missing", "scope_epoch": 1, "outcome": "not_live"}
5095 ],
5096 "ended": [{"ref": "old", "scope_epoch": 3}]
5097 }))
5098 .unwrap();
5099 assert!(handle.handle_control_reply(control_reply(
5100 FrameType::Response,
5101 request.header.corr,
5102 body
5103 )));
5104 let reply = apply.await.unwrap().unwrap();
5105 assert_eq!(reply.generation, 2);
5106 assert_eq!(reply.results.len(), 1);
5107 assert_eq!(reply.results[0].scope_ref, "run");
5108 assert_eq!(
5109 reply.results[0].outcome,
5110 subc_protocol::scope::ScopeRecordOutcome::Refused
5111 );
5112 assert_eq!(reply.results[0].code.as_deref(), Some("scope_expired"));
5113 assert_eq!(reply.end_results.len(), 2);
5114 assert_eq!(reply.end_results[0].scope_ref, "old");
5115 assert_eq!(reply.end_results[0].scope_epoch, 3);
5116 assert_eq!(
5117 reply.end_results[0].outcome,
5118 subc_protocol::scope::ScopeEndOutcome::Ended
5119 );
5120 assert_eq!(reply.end_results[1].scope_ref, "missing");
5121 assert_eq!(
5122 reply.end_results[1].outcome,
5123 subc_protocol::scope::ScopeEndOutcome::NotLive
5124 );
5125 assert_eq!(reply.ended.len(), 1);
5126 assert_eq!(reply.ended[0].scope_ref, "old");
5127 assert_eq!(reply.ended[0].scope_epoch, 3);
5128 }
5129
5130 #[tokio::test]
5131 async fn scope_apply_rejects_a_reply_for_another_op() {
5132 let (handle, mut rx) = test_module_handle(&[SCOPE_APPLY_OP]);
5133 let apply = tokio::spawn({
5134 let handle = handle.clone();
5135 async move { handle.scope_apply(2, Vec::new(), Vec::new()).await }
5136 });
5137 let request = timeout(Duration::from_secs(1), rx.recv())
5138 .await
5139 .unwrap()
5140 .unwrap();
5141 let body = serde_json::to_vec(&ModuleControlResponseToModule::ScopeSync {
5142 generation: 2,
5143 results: Vec::new(),
5144 ended: Vec::new(),
5145 })
5146 .unwrap();
5147 assert!(handle.handle_control_reply(control_reply(
5148 FrameType::Response,
5149 request.header.corr,
5150 body
5151 )));
5152 assert!(matches!(
5153 apply.await.unwrap(),
5154 Err(ScopeCallError::Protocol(_))
5155 ));
5156 }
5157
5158 #[tokio::test]
5159 async fn scope_apply_preserves_whole_call_refusal_codes() {
5160 let (handle, mut rx) = test_module_handle(&[SCOPE_APPLY_OP]);
5161 for code in [
5162 subc_protocol::error_codes::SCOPE_SYNC_REQUIRED,
5163 subc_protocol::error_codes::SCOPE_SYNC_STALE,
5164 subc_protocol::error_codes::SCOPE_SYNC_NOT_AUTHORITY,
5165 "not_registered",
5166 ] {
5167 let apply = tokio::spawn({
5168 let handle = handle.clone();
5169 async move { handle.scope_apply(2, Vec::new(), Vec::new()).await }
5170 });
5171 let request = timeout(Duration::from_secs(1), rx.recv())
5172 .await
5173 .unwrap()
5174 .unwrap();
5175 let body = serde_json::to_vec(&ErrorBody::new(code, "why")).unwrap();
5176 assert!(handle.handle_control_reply(control_reply(
5177 FrameType::Error,
5178 request.header.corr,
5179 body
5180 )));
5181 let error = apply.await.unwrap().unwrap_err();
5182 assert_eq!(
5183 error,
5184 ScopeCallError::Refused {
5185 code: code.to_string(),
5186 message: "why".to_string()
5187 }
5188 );
5189 assert_eq!(error.code(), Some(code));
5190 }
5191 }
5192
5193 #[tokio::test]
5195 async fn a_scope_reply_of_another_op_is_a_protocol_error() {
5196 let (handle, mut rx) = test_module_handle(&[SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP]);
5197 let sync = tokio::spawn({
5198 let handle = handle.clone();
5199 async move { handle.scope_sync(1, Vec::new()).await }
5200 });
5201 let request = timeout(Duration::from_secs(1), rx.recv())
5202 .await
5203 .unwrap()
5204 .unwrap();
5205 let describe_body = serde_json::to_vec(&ModuleControlResponseToModule::ScopeDescribe {
5206 status: ScopeStatus::NotLive,
5207 scope_epoch: None,
5208 daemon_incarnation: "i".to_string(),
5209 owner_synced: false,
5210 owner_configured: false,
5211 scope: None,
5212 })
5213 .unwrap();
5214 assert!(handle.handle_control_reply(control_reply(
5215 FrameType::Response,
5216 request.header.corr,
5217 describe_body
5218 )));
5219 assert!(matches!(
5220 sync.await.unwrap(),
5221 Err(ScopeCallError::Protocol(_))
5222 ));
5223
5224 let describe = tokio::spawn({
5225 let handle = handle.clone();
5226 async move { handle.scope_describe(owner(), "s".to_string()).await }
5227 });
5228 let request = timeout(Duration::from_secs(1), rx.recv())
5229 .await
5230 .unwrap()
5231 .unwrap();
5232 let sync_body = serde_json::to_vec(&ModuleControlResponseToModule::ScopeSync {
5233 generation: 1,
5234 results: Vec::new(),
5235 ended: Vec::new(),
5236 })
5237 .unwrap();
5238 assert!(handle.handle_control_reply(control_reply(
5239 FrameType::Response,
5240 request.header.corr,
5241 sync_body
5242 )));
5243 assert!(matches!(
5244 describe.await.unwrap(),
5245 Err(ScopeCallError::Protocol(_))
5246 ));
5247 }
5248
5249 #[tokio::test]
5253 async fn a_scope_error_frame_keeps_the_daemons_code_and_message() {
5254 let (handle, mut rx) = test_module_handle(&[SCOPE_SYNC_OP]);
5255 for code in [
5256 subc_protocol::error_codes::SCOPE_SYNC_STALE,
5257 subc_protocol::error_codes::SCOPE_SYNC_NOT_AUTHORITY,
5258 "not_registered",
5259 ] {
5260 let sync = tokio::spawn({
5261 let handle = handle.clone();
5262 async move { handle.scope_sync(1, Vec::new()).await }
5263 });
5264 let request = timeout(Duration::from_secs(1), rx.recv())
5265 .await
5266 .unwrap()
5267 .unwrap();
5268 let body = serde_json::to_vec(&ErrorBody::new(code, "why")).unwrap();
5269 assert!(handle.handle_control_reply(control_reply(
5270 FrameType::Error,
5271 request.header.corr,
5272 body
5273 )));
5274 let error = sync.await.unwrap().unwrap_err();
5275 assert_eq!(
5276 error,
5277 ScopeCallError::Refused {
5278 code: code.to_string(),
5279 message: "why".to_string()
5280 }
5281 );
5282 assert_eq!(error.code(), Some(code));
5283 }
5284 }
5285
5286 #[tokio::test]
5290 async fn an_undecodable_control_request_is_refused_and_the_module_keeps_serving() {
5291 let (tx, mut rx) = mpsc::channel(4);
5292 let handler = Arc::new(EchoHandler);
5293 let dispatcher = RequestDispatcher::new();
5294 let (module_handle, _unused_rx) = test_module_handle(&[]);
5295 let mut body: serde_json::Value =
5298 serde_json::from_slice(&route_bind_frame(9, 1, 41).body).unwrap();
5299 body["scope"] = serde_json::json!({
5300 "owner": { "kind": "reserved", "module_id": "prefrontal-core" },
5301 "ref": "s",
5302 "scope_epoch": 1,
5303 "kind": "head",
5304 "attributes": { "a_field_from_a_newer_daemon": "x" },
5305 "owner_authorized": true
5306 });
5307 let body = serde_json::to_vec(&body).unwrap();
5308 let bind = Frame::build(FrameType::Request, control_flags(), 0, 0, 41, body).unwrap();
5309
5310 let kept_serving = handle_frame(
5311 bind,
5312 &tx,
5313 Arc::clone(&handler),
5314 dispatcher.clone(),
5315 module_handle.clone(),
5316 )
5317 .await
5318 .expect("an undecodable control request must not end the module");
5319 assert!(kept_serving);
5320 let refusal = timeout(Duration::from_secs(1), rx.recv())
5321 .await
5322 .unwrap()
5323 .unwrap();
5324 assert_eq!(refusal.header.ty, FrameType::Error);
5325 assert_eq!(refusal.header.corr, 41);
5326 let error: ErrorBody = serde_json::from_slice(&refusal.body).unwrap();
5327 assert_eq!(error.code, "invalid_request");
5328 assert!(
5331 error.message.contains("a_field_from_a_newer_daemon"),
5332 "{}",
5333 error.message
5334 );
5335
5336 assert!(
5337 handle_frame(health_request(42), &tx, handler, dispatcher, module_handle)
5338 .await
5339 .unwrap()
5340 );
5341 let health = timeout(Duration::from_secs(1), rx.recv())
5342 .await
5343 .unwrap()
5344 .unwrap();
5345 assert_eq!(health.header.ty, FrameType::Response);
5346 assert_eq!(health.header.corr, 42);
5347 }
5348
5349 #[tokio::test]
5350 async fn default_health_check_answers_ok() {
5351 let (tx, mut rx) = mpsc::channel(4);
5352 let handler = Arc::new(EchoHandler);
5353 let dispatcher = RequestDispatcher::new();
5354 let (module_handle, _unused_rx) = test_module_handle(&[]);
5355
5356 assert!(
5357 handle_frame(health_request(77), &tx, handler, dispatcher, module_handle)
5358 .await
5359 .unwrap()
5360 );
5361
5362 let response = timeout(Duration::from_secs(1), rx.recv())
5363 .await
5364 .unwrap()
5365 .unwrap();
5366 assert_eq!(response.header.ty, FrameType::Response);
5367 assert_eq!(response.header.channel, 0);
5368 assert_eq!(response.header.corr, 77);
5369
5370 let parsed = serde_json::from_slice::<ModuleControlResponse>(&response.body).unwrap();
5375 let ModuleControlResponse::HealthCheck { status, detail, .. } = parsed else {
5376 panic!("expected a health.check response");
5377 };
5378 assert_eq!(status, HealthStatus::Ok);
5381 assert!(
5385 detail
5386 .as_deref()
5387 .is_some_and(|d| d.contains("no health implementation")),
5388 "the inherited default must identify itself, got {detail:?}"
5389 );
5390 }
5391
5392 struct DispatchHarness {
5393 tx: mpsc::Sender<Frame>,
5394 rx: mpsc::Receiver<Frame>,
5395 handler: Arc<BlockingHandler>,
5396 dispatcher: RequestDispatcher,
5397 module_handle: ModuleHandle,
5398 }
5399
5400 impl DispatchHarness {
5401 fn new(report: HealthReport) -> Self {
5402 let (tx, rx) = mpsc::channel(HANDLER_TASK_CAPACITY + 4);
5403 let (module_handle, _unused_rx) = test_module_handle(&[]);
5404 module_handle
5405 .install_route(RouteHandle::new(7, 1, 1))
5406 .unwrap();
5407 Self {
5408 tx,
5409 rx,
5410 handler: Arc::new(BlockingHandler {
5411 entered: Arc::new(AtomicUsize::new(0)),
5412 release: Semaphore::new(0),
5413 report,
5414 }),
5415 dispatcher: RequestDispatcher::new(),
5416 module_handle,
5417 }
5418 }
5419
5420 async fn request(&self, corr: u64) {
5421 handle_frame(
5422 data_request(7, corr),
5423 &self.tx,
5424 Arc::clone(&self.handler),
5425 self.dispatcher.clone(),
5426 self.module_handle.clone(),
5427 )
5428 .await
5429 .unwrap();
5430 }
5431
5432 async fn wait_for_entered(&self, count: usize) {
5433 timeout(Duration::from_secs(1), async {
5434 while self.handler.entered.load(Ordering::SeqCst) != count {
5435 tokio::task::yield_now().await;
5436 }
5437 })
5438 .await
5439 .expect("data handlers should acquire the available slots");
5440 }
5441
5442 async fn saturate(&self) {
5443 for corr in 1..=HANDLER_TASK_CAPACITY as u64 {
5444 self.request(corr).await;
5445 }
5446 self.wait_for_entered(HANDLER_TASK_CAPACITY).await;
5447 self.request(HANDLER_TASK_CAPACITY as u64 + 1).await;
5448 tokio::task::yield_now().await;
5449 assert_eq!(self.dispatcher.pools.default.permits.available_permits(), 0);
5450 assert_eq!(
5451 self.dispatcher.pools.default.waiting.lock().unwrap().len(),
5452 1
5453 );
5454 assert_eq!(
5455 self.handler.entered.load(Ordering::SeqCst),
5456 HANDLER_TASK_CAPACITY
5457 );
5458 }
5459
5460 async fn health(&mut self, corr: u64) -> HealthReport {
5461 handle_frame(
5462 health_request(corr),
5463 &self.tx,
5464 Arc::clone(&self.handler),
5465 self.dispatcher.clone(),
5466 self.module_handle.clone(),
5467 )
5468 .await
5469 .unwrap();
5470 let frame = timeout(Duration::from_millis(100), self.rx.recv())
5471 .await
5472 .expect("health.check must answer within 100 ms while data slots are busy")
5473 .unwrap();
5474 assert_eq!(frame.header.ty, FrameType::Response);
5475 assert_eq!(frame.header.channel, 0);
5476 assert_eq!(frame.header.corr, corr);
5477 serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5478 .unwrap()
5479 .health_report()
5480 .unwrap()
5481 }
5482
5483 async fn release(&mut self, count: usize) {
5484 self.handler.release.add_permits(count);
5485 for _ in 0..count {
5486 let frame = timeout(Duration::from_secs(1), self.rx.recv())
5487 .await
5488 .unwrap()
5489 .unwrap();
5490 assert_eq!(frame.header.ty, FrameType::StreamEnd);
5491 assert_eq!(frame.header.channel, 7);
5492 }
5493 assert!(self
5494 .dispatcher
5495 .pools
5496 .default
5497 .waiting
5498 .lock()
5499 .unwrap()
5500 .is_empty());
5501 assert!(self.dispatcher.in_flight.lock().unwrap().is_empty());
5502 assert_eq!(
5503 self.dispatcher.pools.default.permits.available_permits(),
5504 HANDLER_TASK_CAPACITY
5505 );
5506 }
5507 }
5508
5509 #[tokio::test(start_paused = true)]
5510 async fn health_check_answers_degraded_under_saturation_and_recovers() {
5511 let own_report = HealthReport {
5512 status: HealthStatus::Ok,
5513 detail: Some("module gauges healthy".to_string()),
5514 metrics: Some(json!({"transforms": 64})),
5515 };
5516 let mut harness = DispatchHarness::new(own_report.clone());
5517 harness.saturate().await;
5518 assert_eq!(harness.health(899).await, own_report);
5520 advance(Duration::from_millis(7300)).await;
5523 let report = harness.health(900).await;
5524 assert_eq!(report.status, HealthStatus::Degraded);
5525 assert_eq!(report.metrics, own_report.metrics);
5526 assert_eq!(
5527 report.detail.as_deref(),
5528 Some("request dispatch saturated: 64/64 in use, oldest waiting 7.3 s; module gauges healthy")
5529 );
5530
5531 harness.release(HANDLER_TASK_CAPACITY + 1).await;
5532 assert_eq!(harness.health(901).await, own_report);
5533 }
5534
5535 #[tokio::test(start_paused = true)]
5536 async fn health_check_preserves_failing_under_saturation() {
5537 let mut harness = DispatchHarness::new(HealthReport {
5538 status: HealthStatus::Failing,
5539 detail: Some("module dispatch heartbeat stale".to_string()),
5540 metrics: Some(json!({"heartbeat_age_s": 12})),
5541 });
5542 harness.saturate().await;
5543 advance(Duration::from_millis(7300)).await;
5544 let report = harness.health(900).await;
5545 assert_eq!(report.status, HealthStatus::Failing);
5546 assert_eq!(report.metrics, Some(json!({"heartbeat_age_s": 12})));
5547 assert_eq!(
5548 report.detail.as_deref(),
5549 Some("request dispatch saturated: 64/64 in use, oldest waiting 7.3 s; module dispatch heartbeat stale")
5550 );
5551 harness.release(HANDLER_TASK_CAPACITY + 1).await;
5552 }
5553
5554 #[tokio::test(start_paused = true)]
5555 async fn health_check_passes_module_report_through_with_free_permits() {
5556 for status in [
5557 HealthStatus::Ok,
5558 HealthStatus::Degraded,
5559 HealthStatus::Failing,
5560 ] {
5561 let own_report = HealthReport {
5562 status,
5563 detail: Some("module's own assessment".to_string()),
5564 metrics: Some(json!({"gauge": 42})),
5565 };
5566 let mut harness = DispatchHarness::new(own_report.clone());
5567 harness.request(1).await;
5568 harness.wait_for_entered(1).await;
5569 advance(Duration::from_millis(7300)).await;
5570 assert_eq!(harness.health(900).await, own_report);
5571 harness.release(1).await;
5572 }
5573 }
5574
5575 struct CorrBlockingHandler {
5576 entered: Arc<Mutex<Vec<u64>>>,
5577 release_first: Arc<Semaphore>,
5578 }
5579
5580 #[async_trait]
5581 impl ModuleHandler for CorrBlockingHandler {
5582 async fn handle(&self, ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
5583 let corr = ctx.corr();
5584 self.entered.lock().unwrap().push(corr);
5585 if corr == 1 {
5586 self.release_first.acquire().await.unwrap().forget();
5587 }
5588 HandlerOutcome::Response(Vec::new())
5589 }
5590 }
5591
5592 #[tokio::test]
5596 async fn cancelled_capacity_queued_data_request_is_answered_without_waiting_for_a_slot() {
5597 let (tx, mut rx) = mpsc::channel(4);
5598 let entered = Arc::new(Mutex::new(Vec::new()));
5599 let release_first = Arc::new(Semaphore::new(0));
5600 let handler = Arc::new(CorrBlockingHandler {
5601 entered: Arc::clone(&entered),
5602 release_first: Arc::clone(&release_first),
5603 });
5604 let dispatcher = RequestDispatcher::with_pools(Arc::new(
5605 DispatchPools::with_default_limit(1, RequestPools::new()),
5606 ));
5607 let (module_handle, _unused_rx) = test_module_handle(&[]);
5608 module_handle
5609 .install_route(RouteHandle::new(7, 1, 1))
5610 .unwrap();
5611
5612 handle_frame(
5613 data_request(7, 1),
5614 &tx,
5615 Arc::clone(&handler),
5616 dispatcher.clone(),
5617 module_handle.clone(),
5618 )
5619 .await
5620 .unwrap();
5621 timeout(Duration::from_secs(1), async {
5622 while entered.lock().unwrap().as_slice() != [1] {
5623 tokio::task::yield_now().await;
5624 }
5625 })
5626 .await
5627 .unwrap();
5628
5629 handle_frame(
5630 data_request(7, 2),
5631 &tx,
5632 Arc::clone(&handler),
5633 dispatcher.clone(),
5634 module_handle.clone(),
5635 )
5636 .await
5637 .unwrap();
5638 timeout(Duration::from_secs(1), async {
5639 while !dispatcher
5640 .in_flight
5641 .lock()
5642 .unwrap()
5643 .contains_key(&(7, 1, 2))
5644 {
5645 tokio::task::yield_now().await;
5646 }
5647 })
5648 .await
5649 .unwrap();
5650 handle_frame(
5651 Frame::build(FrameType::Cancel, data_flags(), 7, 1, 2, Vec::new()).unwrap(),
5652 &tx,
5653 Arc::clone(&handler),
5654 dispatcher.clone(),
5655 module_handle,
5656 )
5657 .await
5658 .unwrap();
5659 assert!(
5660 dispatcher
5661 .in_flight
5662 .lock()
5663 .unwrap()
5664 .get(&(7, 1, 2))
5665 .unwrap()
5666 .is_cancelled(),
5667 "cancel must land while the second request waits for handler capacity"
5668 );
5669
5670 let cancelled = timeout(Duration::from_secs(1), rx.recv())
5673 .await
5674 .expect("a request cancelled while waiting must be answered without waiting for a slot")
5675 .unwrap();
5676 assert_eq!(cancelled.header.ty, FrameType::Error);
5677 assert_eq!(cancelled.header.channel, 7);
5678 assert_eq!(cancelled.header.epoch, 1);
5679 assert_eq!(cancelled.header.corr, 2);
5680 assert_eq!(
5681 serde_json::from_slice::<ErrorBody>(&cancelled.body).unwrap(),
5682 ErrorBody {
5683 code: "cancelled".to_string(),
5684 message: "request cancelled".to_string(),
5685 detail: None,
5686 }
5687 );
5688 timeout(Duration::from_secs(1), async {
5689 while dispatcher
5690 .in_flight
5691 .lock()
5692 .unwrap()
5693 .contains_key(&(7, 1, 2))
5694 {
5695 tokio::task::yield_now().await;
5696 }
5697 })
5698 .await
5699 .unwrap();
5700 assert_eq!(dispatcher.pools.default.permits.available_permits(), 0);
5701 assert!(dispatcher.pools.default.waiting.lock().unwrap().is_empty());
5702 assert_eq!(*entered.lock().unwrap(), vec![1]);
5703
5704 release_first.add_permits(1);
5705 let response = timeout(Duration::from_secs(1), rx.recv())
5706 .await
5707 .unwrap()
5708 .unwrap();
5709 assert_eq!(response.header.ty, FrameType::Response);
5710 assert_eq!(response.header.corr, 1);
5711 timeout(Duration::from_secs(1), async {
5712 while !dispatcher.in_flight.lock().unwrap().is_empty() {
5713 tokio::task::yield_now().await;
5714 }
5715 })
5716 .await
5717 .unwrap();
5718 assert_eq!(*entered.lock().unwrap(), vec![1]);
5719 assert_eq!(dispatcher.pools.default.permits.available_permits(), 1);
5720 assert!(timeout(Duration::from_millis(50), rx.recv()).await.is_err());
5721 }
5722
5723 #[tokio::test]
5724 async fn cancelled_terminal_is_not_sent_after_route_teardown() {
5725 let (tx, mut rx) = mpsc::channel(1);
5726 let (module_handle, _unused_rx) = test_module_handle(&[]);
5727 let handle = RouteHandle::new(7, 1, 1);
5728 module_handle.install_route(handle).unwrap();
5729 let ctx = RequestCtx {
5730 handle,
5731 corr: 2,
5732 ver: PROTOCOL_VERSION,
5733 egress: tx,
5734 module_handle: module_handle.clone(),
5735 cancelled: CancellationToken::new(),
5736 binary_reply: Arc::new(AtomicBool::new(false)),
5737 request_flags: Flags::new(false, Priority::Interactive, false),
5738 request_body: Arc::from([]),
5739 };
5740 assert!(module_handle.remove_route(handle).unwrap());
5741
5742 let result = send_handler_outcome(
5743 &ctx,
5744 HandlerOutcome::Error {
5745 code: "cancelled".to_string(),
5746 message: "request cancelled".to_string(),
5747 },
5748 )
5749 .await;
5750
5751 assert!(matches!(
5752 result,
5753 Err(SubcModuleError::StaleRouteHandle(stale)) if stale == handle
5754 ));
5755 assert!(rx.try_recv().is_err());
5756 }
5757
5758 struct CountingHandler {
5759 calls: Arc<AtomicUsize>,
5760 }
5761
5762 #[async_trait]
5763 impl ModuleHandler for CountingHandler {
5764 async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
5765 self.calls.fetch_add(1, Ordering::SeqCst);
5766 HandlerOutcome::Response(Vec::new())
5767 }
5768 }
5769
5770 #[tokio::test]
5771 async fn endpoint_validation_drops_stale_request_before_handler_dispatch() {
5772 let (tx, mut rx) = mpsc::channel(4);
5773 let calls = Arc::new(AtomicUsize::new(0));
5774 let handler = Arc::new(CountingHandler {
5775 calls: Arc::clone(&calls),
5776 });
5777 let dispatcher = RequestDispatcher::new();
5778 let (module_handle, _unused_rx) = test_module_handle(&[]);
5779 module_handle
5780 .install_route(RouteHandle::new(7, 2, 1))
5781 .unwrap();
5782 let stale = Frame::build(FrameType::Request, data_flags(), 7, 1, 55, Vec::new()).unwrap();
5783
5784 assert!(
5785 handle_frame(stale, &tx, handler, dispatcher, module_handle.clone())
5786 .await
5787 .unwrap()
5788 );
5789 tokio::task::yield_now().await;
5790 assert_eq!(calls.load(Ordering::SeqCst), 0);
5791 assert_eq!(module_handle.dropped_route_frames(), 1);
5792 assert!(rx.try_recv().is_err());
5793 }
5794
5795 struct BindOrderingHandler {
5796 module_handle: ModuleHandle,
5797 bind_emit_rejected: Arc<AtomicUsize>,
5798 bound: Arc<AtomicUsize>,
5799 cleanup: Arc<AtomicUsize>,
5800 }
5801
5802 #[async_trait]
5803 impl ModuleHandler for BindOrderingHandler {
5804 async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
5805 HandlerOutcome::Response(Vec::new())
5806 }
5807
5808 async fn on_bind(&self, req: &RouteBindRequest) -> BindDecision {
5809 if matches!(
5810 self.module_handle
5811 .push(&req.handle, b"too-early".to_vec(), None)
5812 .await,
5813 Err(SubcModuleError::StaleRouteHandle(_))
5814 ) {
5815 self.bind_emit_rejected.fetch_add(1, Ordering::SeqCst);
5816 }
5817 BindDecision::accept()
5818 }
5819
5820 async fn on_bound(&self, handle: &RouteHandle) {
5821 self.bound.fetch_add(1, Ordering::SeqCst);
5822 self.module_handle
5823 .push(handle, b"bound".to_vec(), Some(AdmissionClass::Expedite))
5824 .await
5825 .unwrap();
5826 }
5827
5828 async fn on_route_gone(&self, _handle: &RouteHandle) {
5829 self.cleanup.fetch_add(1, Ordering::SeqCst);
5830 }
5831 }
5832
5833 fn route_bind_frame(channel: u16, epoch: u32, corr: u64) -> Frame {
5834 let body = serde_json::to_vec(&ModuleControlRequest::RouteBind {
5835 route_channel: channel,
5836 epoch,
5837 target: RouteTarget::ToolProvider {
5838 module_id: "provider".to_string(),
5839 },
5840 identity: BindIdentity::new("test".to_string(), "bind".to_string()),
5841 principal: None,
5842 consumer_capabilities: None,
5843 role_versions: None,
5844 admission_facts: None,
5845 scope: None,
5846 call_mode: None,
5847 })
5848 .unwrap();
5849 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5850 }
5851
5852 #[tokio::test]
5853 async fn on_bound_runs_only_after_ack_queue_and_handle_install() {
5854 let (module_handle, mut rx) = test_module_handle(&[]);
5855 let tx = module_handle.shared.lock_inner().writer.clone().unwrap();
5856 let bind_emit_rejected = Arc::new(AtomicUsize::new(0));
5857 let bound = Arc::new(AtomicUsize::new(0));
5858 let cleanup = Arc::new(AtomicUsize::new(0));
5859 let handler = Arc::new(BindOrderingHandler {
5860 module_handle: module_handle.clone(),
5861 bind_emit_rejected: Arc::clone(&bind_emit_rejected),
5862 bound: Arc::clone(&bound),
5863 cleanup: Arc::clone(&cleanup),
5864 });
5865
5866 assert!(handle_frame(
5867 route_bind_frame(8, 4, 90),
5868 &tx,
5869 handler,
5870 RequestDispatcher::new(),
5871 module_handle.clone(),
5872 )
5873 .await
5874 .unwrap());
5875 assert_eq!(bind_emit_rejected.load(Ordering::SeqCst), 1);
5876 assert_eq!(bound.load(Ordering::SeqCst), 1);
5877 assert_eq!(cleanup.load(Ordering::SeqCst), 0);
5878
5879 let ack = rx.recv().await.unwrap();
5880 let push = rx.recv().await.unwrap();
5881 assert_eq!(ack.header.ty, FrameType::Response);
5882 assert_eq!(ack.header.channel, 0);
5883 assert_eq!(push.header.ty, FrameType::Push);
5884 assert_eq!((push.header.channel, push.header.epoch), (8, 4));
5885 assert_eq!(
5886 push.header.flags.admission_class(),
5887 Some(AdmissionClass::Expedite)
5888 );
5889
5890 let captured = RouteHandle::new(8, 4, 1);
5891 assert!(module_handle.remove_route(captured).unwrap());
5892 let stale = module_handle
5893 .push(&captured, Vec::new(), None)
5894 .await
5895 .unwrap_err();
5896 assert!(matches!(stale, SubcModuleError::StaleRouteHandle(_)));
5897 assert!(rx.try_recv().is_err());
5898 }
5899
5900 struct RejectingHandler {
5901 bound: Arc<AtomicUsize>,
5902 cleanup: Arc<AtomicUsize>,
5903 }
5904
5905 #[async_trait]
5906 impl ModuleHandler for RejectingHandler {
5907 async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
5908 HandlerOutcome::Response(Vec::new())
5909 }
5910
5911 async fn on_bind(&self, _req: &RouteBindRequest) -> BindDecision {
5912 BindDecision::reject("no", "rejected")
5913 }
5914
5915 async fn on_bound(&self, _handle: &RouteHandle) {
5916 self.bound.fetch_add(1, Ordering::SeqCst);
5917 }
5918
5919 async fn on_route_gone(&self, _handle: &RouteHandle) {
5920 self.cleanup.fetch_add(1, Ordering::SeqCst);
5921 }
5922 }
5923
5924 #[tokio::test]
5925 async fn rejected_bind_cleans_up_without_installing_or_calling_on_bound() {
5926 let (module_handle, mut rx) = test_module_handle(&[]);
5927 let tx = module_handle.shared.lock_inner().writer.clone().unwrap();
5928 let bound = Arc::new(AtomicUsize::new(0));
5929 let cleanup = Arc::new(AtomicUsize::new(0));
5930 let handler = Arc::new(RejectingHandler {
5931 bound: Arc::clone(&bound),
5932 cleanup: Arc::clone(&cleanup),
5933 });
5934 handle_frame(
5935 route_bind_frame(6, 3, 91),
5936 &tx,
5937 handler,
5938 RequestDispatcher::new(),
5939 module_handle.clone(),
5940 )
5941 .await
5942 .unwrap();
5943 let refusal = rx.recv().await.unwrap();
5944 assert_eq!(refusal.header.ty, FrameType::Error);
5945 assert_eq!(refusal.body, br#"{"code":"no","message":"rejected"}"#);
5947 assert_eq!(bound.load(Ordering::SeqCst), 0);
5948 assert_eq!(cleanup.load(Ordering::SeqCst), 1);
5949 assert!(matches!(
5950 module_handle.validate_route(RouteHandle::new(6, 3, 1)),
5951 Err(SubcModuleError::StaleRouteHandle(_))
5952 ));
5953 }
5954
5955 struct BusyHandler;
5956
5957 #[async_trait]
5958 impl ModuleHandler for BusyHandler {
5959 async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
5960 HandlerOutcome::Response(Vec::new())
5961 }
5962
5963 async fn on_bind(&self, _req: &RouteBindRequest) -> BindDecision {
5964 BindDecision::reject_with_detail(
5965 "bind_busy",
5966 "a read is still running on this root",
5967 serde_json::json!({"reason": "reader", "retry_after_ms": 2000}),
5968 )
5969 }
5970 }
5971
5972 #[tokio::test]
5973 async fn rejected_bind_carries_its_detail_to_the_daemon() {
5974 let (module_handle, mut rx) = test_module_handle(&[]);
5975 let tx = module_handle.shared.lock_inner().writer.clone().unwrap();
5976 handle_frame(
5977 route_bind_frame(6, 3, 92),
5978 &tx,
5979 Arc::new(BusyHandler),
5980 RequestDispatcher::new(),
5981 module_handle.clone(),
5982 )
5983 .await
5984 .unwrap();
5985 let refusal = rx.recv().await.unwrap();
5986 assert_eq!(refusal.header.ty, FrameType::Error);
5987 let body: ErrorBody = serde_json::from_slice(&refusal.body).unwrap();
5988 assert_eq!(body.code, "bind_busy");
5989 assert_eq!(
5990 body.detail,
5991 Some(serde_json::json!({"reason": "reader", "retry_after_ms": 2000}))
5992 );
5993 }
5994
5995 struct RebindCountingHandler {
5996 bound: Arc<AtomicUsize>,
5997 cleanup: Arc<AtomicUsize>,
5998 }
5999
6000 #[async_trait]
6001 impl ModuleHandler for RebindCountingHandler {
6002 async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
6003 HandlerOutcome::Response(Vec::new())
6004 }
6005
6006 async fn on_bound(&self, _handle: &RouteHandle) {
6007 self.bound.fetch_add(1, Ordering::SeqCst);
6008 }
6009
6010 async fn on_route_gone(&self, _handle: &RouteHandle) {
6011 self.cleanup.fetch_add(1, Ordering::SeqCst);
6012 }
6013 }
6014
6015 #[tokio::test]
6021 async fn rebind_on_installed_channel_replaces_on_higher_epoch_only() {
6022 let (module_handle, mut rx) = test_module_handle(&[]);
6023 let tx = module_handle.shared.lock_inner().writer.clone().unwrap();
6024 let bound = Arc::new(AtomicUsize::new(0));
6025 let cleanup = Arc::new(AtomicUsize::new(0));
6026 let handler = Arc::new(RebindCountingHandler {
6027 bound: Arc::clone(&bound),
6028 cleanup: Arc::clone(&cleanup),
6029 });
6030
6031 handle_frame(
6033 route_bind_frame(8, 4, 90),
6034 &tx,
6035 Arc::clone(&handler),
6036 RequestDispatcher::new(),
6037 module_handle.clone(),
6038 )
6039 .await
6040 .unwrap();
6041 assert_eq!(rx.recv().await.unwrap().header.ty, FrameType::Response);
6042 assert_eq!(
6043 (bound.load(Ordering::SeqCst), cleanup.load(Ordering::SeqCst)),
6044 (1, 0)
6045 );
6046
6047 handle_frame(
6049 route_bind_frame(8, 4, 91),
6050 &tx,
6051 Arc::clone(&handler),
6052 RequestDispatcher::new(),
6053 module_handle.clone(),
6054 )
6055 .await
6056 .unwrap();
6057 let reject = rx.recv().await.unwrap();
6058 assert_eq!(reject.header.ty, FrameType::Error);
6059 assert_eq!(
6060 (bound.load(Ordering::SeqCst), cleanup.load(Ordering::SeqCst)),
6061 (1, 0)
6062 );
6063 module_handle
6064 .validate_route(RouteHandle::new(8, 4, 1))
6065 .expect("epoch-4 install must survive the rejected rebind");
6066
6067 handle_frame(
6069 route_bind_frame(8, 3, 92),
6070 &tx,
6071 Arc::clone(&handler),
6072 RequestDispatcher::new(),
6073 module_handle.clone(),
6074 )
6075 .await
6076 .unwrap();
6077 assert_eq!(rx.recv().await.unwrap().header.ty, FrameType::Error);
6078 assert_eq!(
6079 (bound.load(Ordering::SeqCst), cleanup.load(Ordering::SeqCst)),
6080 (1, 0)
6081 );
6082
6083 handle_frame(
6086 route_bind_frame(8, 5, 93),
6087 &tx,
6088 Arc::clone(&handler),
6089 RequestDispatcher::new(),
6090 module_handle.clone(),
6091 )
6092 .await
6093 .unwrap();
6094 assert_eq!(rx.recv().await.unwrap().header.ty, FrameType::Response);
6095 assert_eq!(
6096 (bound.load(Ordering::SeqCst), cleanup.load(Ordering::SeqCst)),
6097 (2, 1)
6098 );
6099 assert!(matches!(
6100 module_handle.validate_route(RouteHandle::new(8, 4, 1)),
6101 Err(SubcModuleError::StaleRouteHandle(_))
6102 ));
6103 module_handle
6104 .validate_route(RouteHandle::new(8, 5, 1))
6105 .expect("epoch-5 install must be live after implicit replace");
6106 }
6107
6108 #[tokio::test]
6109 async fn module_control_corr_is_monotonic_and_exhausts_without_wrap() {
6110 let (module_handle, _rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
6111 let mut inner = module_handle.shared.lock_inner();
6112 inner.next_corr = Some(u64::MAX);
6113 assert_eq!(next_module_control_corr(&mut inner), Some(u64::MAX));
6114 assert_eq!(next_module_control_corr(&mut inner), None);
6115 }
6116}
6117
6118#[cfg(test)]
6121mod module_close_tests {
6122 use std::time::Instant;
6123
6124 use tokio::{io::DuplexStream, sync::Notify};
6125
6126 use super::stand_in::{serve_against_stand_in_with_ops, Served};
6127 use super::*;
6128
6129 #[derive(Clone, Copy)]
6131 enum Hold {
6132 UntilCancelled,
6134 IgnoringCancellation,
6136 }
6137
6138 struct HoldingHandler {
6139 hold: Hold,
6140 entered: Arc<Notify>,
6141 token: Arc<Mutex<Option<CancellationToken>>>,
6142 }
6143
6144 #[async_trait]
6145 impl ModuleHandler for HoldingHandler {
6146 async fn handle(&self, ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
6147 *self.token.lock().unwrap() = Some(ctx.cancellation_token());
6148 self.entered.notify_one();
6149 match self.hold {
6150 Hold::UntilCancelled => {
6151 ctx.cancelled().await;
6152 HandlerOutcome::Error {
6153 code: "cancelled".to_string(),
6154 message: "request cancelled".to_string(),
6155 }
6156 }
6157 Hold::IgnoringCancellation => std::future::pending().await,
6158 }
6159 }
6160 }
6161
6162 async fn serve_against_stand_in<H: ModuleHandler>(handler: H) -> Served {
6164 serve_against_stand_in_with_ops(|_| handler, &[]).await
6165 }
6166
6167 async fn send(daemon: &mut DuplexStream, frame: Frame) {
6168 write_frame(daemon, &frame).await.unwrap();
6169 daemon.flush().await.unwrap();
6170 }
6171
6172 async fn hold_one_request(served: &mut Served, entered: &Notify) {
6175 let bind = serde_json::to_vec(&ModuleControlRequest::RouteBind {
6176 route_channel: 7,
6177 epoch: 1,
6178 target: RouteTarget::ToolProvider {
6179 module_id: "close-test".to_string(),
6180 },
6181 identity: BindIdentity::new("test".to_string(), "close".to_string()),
6182 principal: None,
6183 consumer_capabilities: None,
6184 role_versions: None,
6185 admission_facts: None,
6186 scope: None,
6187 call_mode: None,
6188 })
6189 .unwrap();
6190 send(
6191 &mut served.daemon,
6192 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, bind).unwrap(),
6193 )
6194 .await;
6195 loop {
6196 let frame = read_frame(&mut served.daemon).await.unwrap().unwrap();
6197 if frame.header.channel == 0 && frame.header.corr == 2 {
6198 assert_eq!(
6199 frame.header.ty,
6200 FrameType::Response,
6201 "route.bind must be acked"
6202 );
6203 break;
6204 }
6205 }
6206 send(
6207 &mut served.daemon,
6208 Frame::build(FrameType::Request, data_flags(), 7, 1, 10, b"hold".to_vec()).unwrap(),
6209 )
6210 .await;
6211 timeout(Duration::from_secs(2), entered.notified())
6212 .await
6213 .expect("the handler must receive the request");
6214 }
6215
6216 fn goodbye() -> Frame {
6217 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 0, Vec::new()).unwrap()
6218 }
6219
6220 fn holding(
6221 hold: Hold,
6222 ) -> (
6223 HoldingHandler,
6224 Arc<Notify>,
6225 Arc<Mutex<Option<CancellationToken>>>,
6226 ) {
6227 let entered = Arc::new(Notify::new());
6228 let token = Arc::new(Mutex::new(None));
6229 let handler = HoldingHandler {
6230 hold,
6231 entered: Arc::clone(&entered),
6232 token: Arc::clone(&token),
6233 };
6234 (handler, entered, token)
6235 }
6236
6237 #[tokio::test]
6238 async fn goodbye_cancels_an_open_request_and_the_serve_future_completes() {
6239 let (handler, entered, token) = holding(Hold::UntilCancelled);
6240 let mut served = serve_against_stand_in(handler).await;
6241 hold_one_request(&mut served, &entered).await;
6242
6243 let started = Instant::now();
6244 send(&mut served.daemon, goodbye()).await;
6245 let result = timeout(Duration::from_secs(5), &mut served.serve)
6246 .await
6247 .expect("a held request must not keep the serve future alive after GOODBYE");
6248 result.unwrap().unwrap();
6249 assert!(
6250 token.lock().unwrap().as_ref().unwrap().is_cancelled(),
6251 "GOODBYE must cancel the request's token"
6252 );
6253 assert!(
6256 started.elapsed() < DEFAULT_WRITER_DRAIN_TIMEOUT,
6257 "closing took {:?}",
6258 started.elapsed()
6259 );
6260 }
6261
6262 #[tokio::test]
6263 async fn a_handler_ignoring_cancellation_cannot_keep_the_serve_future_alive() {
6264 let (handler, entered, _token) = holding(Hold::IgnoringCancellation);
6265 let mut served = serve_against_stand_in(handler).await;
6266 hold_one_request(&mut served, &entered).await;
6267
6268 let started = Instant::now();
6269 send(&mut served.daemon, goodbye()).await;
6270 let result = timeout(
6271 DEFAULT_WRITER_DRAIN_TIMEOUT + Duration::from_secs(3),
6272 &mut served.serve,
6273 )
6274 .await
6275 .expect("the writer drain must be bounded");
6276 result.unwrap().unwrap();
6277 assert!(started.elapsed() >= DEFAULT_WRITER_DRAIN_TIMEOUT);
6278 }
6279
6280 #[tokio::test]
6281 async fn closed_resolves_after_goodbye() {
6282 let served = serve_against_stand_in(EchoHandler).await;
6283 let mut served = served;
6284 assert!(!served.handle.is_closed());
6285 let closed = served.handle.closed();
6286 send(&mut served.daemon, goodbye()).await;
6287 timeout(Duration::from_secs(2), closed)
6288 .await
6289 .expect("closed() must resolve after GOODBYE");
6290 assert!(served.handle.is_closed());
6291 }
6292
6293 #[tokio::test]
6294 async fn closed_resolves_after_eof() {
6295 let Served {
6296 daemon,
6297 handle,
6298 serve: _serve,
6299 } = serve_against_stand_in(EchoHandler).await;
6300 assert!(!handle.is_closed());
6301 drop(daemon);
6302 timeout(Duration::from_secs(2), handle.closed())
6303 .await
6304 .expect("closed() must resolve after EOF");
6305 assert!(handle.is_closed());
6306 }
6307
6308 struct EchoHandler;
6309
6310 #[async_trait]
6311 impl ModuleHandler for EchoHandler {
6312 async fn handle(&self, _ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome {
6313 HandlerOutcome::Response(body)
6314 }
6315 }
6316
6317 #[derive(Clone, Default)]
6320 struct EndRecorder(Arc<Mutex<Vec<ConnectionEnd>>>);
6321
6322 #[async_trait]
6323 impl ModuleHandler for EndRecorder {
6324 async fn handle(&self, _ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome {
6325 HandlerOutcome::Response(body)
6326 }
6327 async fn on_connection_end(&self, end: ConnectionEnd) {
6328 self.0.lock().unwrap().push(end);
6329 }
6330 }
6331
6332 async fn reported_end(recorder: &EndRecorder, served: &mut Served) -> Vec<ConnectionEnd> {
6333 timeout(Duration::from_secs(5), &mut served.serve)
6334 .await
6335 .expect("serving must end")
6336 .unwrap()
6337 .unwrap();
6338 recorder.0.lock().unwrap().clone()
6339 }
6340
6341 #[tokio::test]
6342 async fn a_daemon_goodbye_is_reported_as_goodbye() {
6343 let recorder = EndRecorder::default();
6344 let mut served = serve_against_stand_in(recorder.clone()).await;
6345 send(&mut served.daemon, goodbye()).await;
6346 assert_eq!(
6347 reported_end(&recorder, &mut served).await,
6348 vec![ConnectionEnd::Goodbye]
6349 );
6350 }
6351
6352 #[tokio::test]
6353 async fn a_daemon_closing_without_goodbye_is_reported_as_eof() {
6354 let recorder = EndRecorder::default();
6355 let mut served = serve_against_stand_in(recorder.clone()).await;
6356 served.daemon.shutdown().await.unwrap();
6357 assert_eq!(
6358 reported_end(&recorder, &mut served).await,
6359 vec![ConnectionEnd::Eof]
6360 );
6361 }
6362
6363 #[derive(Debug, Clone, PartialEq)]
6365 enum Seen {
6366 Draining(RouteCloseReason, SystemTime),
6367 End(ConnectionEnd),
6368 }
6369
6370 #[derive(Clone, Default)]
6373 struct DrainRecorder {
6374 seen: Arc<Mutex<Vec<Seen>>>,
6375 park: bool,
6376 }
6377
6378 #[async_trait]
6379 impl ModuleHandler for DrainRecorder {
6380 async fn handle(&self, _ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome {
6381 HandlerOutcome::Response(body)
6382 }
6383 async fn on_draining(&self, reason: RouteCloseReason, deadline: SystemTime) {
6384 self.seen
6385 .lock()
6386 .unwrap()
6387 .push(Seen::Draining(reason, deadline));
6388 if self.park {
6389 std::future::pending::<()>().await;
6390 }
6391 }
6392 async fn on_connection_end(&self, end: ConnectionEnd) {
6393 self.seen.lock().unwrap().push(Seen::End(end));
6394 }
6395 }
6396
6397 fn control_push(body: &[u8]) -> Frame {
6398 Frame::build(FrameType::Push, control_flags(), 0, 0, 0, body.to_vec()).unwrap()
6399 }
6400
6401 fn draining_push(reason: subc_protocol::RouteCloseReason, deadline_ms: u64) -> Frame {
6402 control_push(
6403 &serde_json::to_vec(&ModuleControlCommand::Draining {
6404 reason,
6405 deadline_ms,
6406 })
6407 .unwrap(),
6408 )
6409 }
6410
6411 async fn ping_is_answered(served: &mut Served, corr: u64) {
6414 send(
6415 &mut served.daemon,
6416 Frame::build(FrameType::Ping, control_flags(), 0, 0, corr, Vec::new()).unwrap(),
6417 )
6418 .await;
6419 let pong = timeout(Duration::from_secs(2), read_frame(&mut served.daemon))
6420 .await
6421 .expect("the module must answer PING")
6422 .unwrap()
6423 .expect("the module must not close the connection");
6424 assert_eq!(pong.header.ty, FrameType::Pong);
6425 assert_eq!(pong.header.corr, corr);
6426 }
6427
6428 #[tokio::test]
6432 async fn a_drain_notice_calls_on_draining_before_the_goodbye_that_follows_it() {
6433 let recorder = DrainRecorder::default();
6434 let mut served = serve_against_stand_in(recorder.clone()).await;
6435 let deadline_ms = 4_102_444_800_000;
6438 write_frame(
6439 &mut served.daemon,
6440 &draining_push(subc_protocol::RouteCloseReason::Restart, deadline_ms),
6441 )
6442 .await
6443 .unwrap();
6444 write_frame(&mut served.daemon, &goodbye()).await.unwrap();
6445 served.daemon.flush().await.unwrap();
6446
6447 timeout(Duration::from_secs(5), &mut served.serve)
6448 .await
6449 .expect("serving must end after GOODBYE")
6450 .unwrap()
6451 .unwrap();
6452 assert_eq!(
6453 *recorder.seen.lock().unwrap(),
6454 vec![
6455 Seen::Draining(
6456 RouteCloseReason::Restart,
6457 UNIX_EPOCH + Duration::from_millis(deadline_ms)
6458 ),
6459 Seen::End(ConnectionEnd::Goodbye),
6460 ]
6461 );
6462 }
6463
6464 #[tokio::test]
6467 async fn a_drain_hook_that_never_returns_does_not_stop_the_frame_reader() {
6468 let recorder = DrainRecorder {
6469 park: true,
6470 ..DrainRecorder::default()
6471 };
6472 let mut served = serve_against_stand_in(recorder.clone()).await;
6473 send(
6474 &mut served.daemon,
6475 draining_push(subc_protocol::RouteCloseReason::Reload, 1),
6476 )
6477 .await;
6478 ping_is_answered(&mut served, 41).await;
6479 assert!(matches!(
6480 recorder.seen.lock().unwrap().as_slice(),
6481 [Seen::Draining(RouteCloseReason::Reload, _)]
6482 ));
6483
6484 send(&mut served.daemon, goodbye()).await;
6485 timeout(Duration::from_secs(5), &mut served.serve)
6486 .await
6487 .expect("a parked drain hook must not keep the serve future alive")
6488 .unwrap()
6489 .unwrap();
6490 }
6491
6492 #[tokio::test]
6496 async fn an_undecodable_channel_zero_push_is_ignored_and_the_connection_stays_up() {
6497 let recorder = DrainRecorder::default();
6498 let mut served = serve_against_stand_in(recorder.clone()).await;
6499 send(
6500 &mut served.daemon,
6501 control_push(br#"{"op":"module.some_future_command","x":1}"#),
6502 )
6503 .await;
6504 send(&mut served.daemon, control_push(b"not json")).await;
6505 send(
6506 &mut served.daemon,
6507 control_push(br#"{"op":"module.draining","reason":"restart"}"#),
6508 )
6509 .await;
6510 ping_is_answered(&mut served, 42).await;
6511 assert!(!served.handle.is_closed());
6512 assert!(!served.serve.is_finished());
6513 assert!(recorder.seen.lock().unwrap().is_empty());
6514 }
6515
6516 #[tokio::test]
6517 async fn a_module_closing_its_own_connection_is_reported_as_closed() {
6518 let recorder = EndRecorder::default();
6519 let mut served = serve_against_stand_in(recorder.clone()).await;
6520 served.handle.close_connection();
6521 assert_eq!(
6522 reported_end(&recorder, &mut served).await,
6523 vec![ConnectionEnd::Closed]
6524 );
6525 }
6526}
6527
6528#[cfg(test)]
6529mod operator_confirm_tests;
6530
6531#[cfg(test)]
6532mod binary_reply_tests;
6533
6534#[cfg(test)]
6535mod serve_lifecycle_tests;
6536
6537#[cfg(test)]
6538mod request_pool_tests;
6539
6540#[cfg(test)]
6541mod detailed_error_body_tests {
6542 use serde_json::json;
6543
6544 use subc_protocol::ErrorBody;
6545
6546 #[test]
6550 fn detailed_error_body_keeps_code_message_and_detail() {
6551 let body = ErrorBody::new("bad_request", "invalid envelope")
6552 .with_detail(json!({"reason": "missing_server"}));
6553
6554 assert_eq!(
6555 serde_json::to_value(body).unwrap(),
6556 json!({
6557 "code": "bad_request",
6558 "message": "invalid envelope",
6559 "detail": {"reason": "missing_server"},
6560 })
6561 );
6562 }
6563
6564 #[test]
6567 fn detail_less_error_body_is_byte_identical_to_the_pre_detail_wire() {
6568 let old_wire = r#"{"code":"cancelled","message":"caller cancelled"}"#;
6569 let parsed: ErrorBody = serde_json::from_str(old_wire).unwrap();
6570 assert_eq!(parsed.detail, None);
6571 assert_eq!(serde_json::to_string(&parsed).unwrap(), old_wire);
6572 }
6573}
6574
6575static RETAINED_LAUNCH_NONCE: std::sync::OnceLock<String> = std::sync::OnceLock::new();
6578
6579pub fn retain_launch_nonce_for_hello(launch_nonce: String) -> Result<(), String> {
6582 RETAINED_LAUNCH_NONCE
6583 .set(launch_nonce)
6584 .map_err(|_| "launch nonce was already retained for this process".to_string())
6585}
6586
6587fn retained_launch_nonce() -> Option<String> {
6588 RETAINED_LAUNCH_NONCE.get().cloned()
6589}
6590
6591pub fn launch_nonce_source() -> Option<LaunchNonceSource> {
6597 launch_nonce()
6598 .ok()
6599 .flatten()
6600 .map(|nonce| LaunchNonceSource::from_wire_name(nonce.source().as_str()))
6601}
6602
6603fn stamp_launch_nonce_source(manifest: &mut ModuleManifest, sent: Option<&str>) {
6611 let Some(provenance) = manifest.provenance.as_mut() else {
6612 return;
6613 };
6614 if provenance.launch_nonce_source.is_some() {
6615 return;
6616 }
6617 if let Ok(Some(nonce)) = launch_nonce() {
6618 let launched = registration::is_launched_module(&manifest.module_id).unwrap_or(false);
6621 if launched && (registration::supervised() || sent == Some(nonce.value())) {
6622 provenance.launch_nonce_source =
6623 Some(LaunchNonceSource::from_wire_name(nonce.source().as_str()));
6624 }
6625 }
6626}
6627
6628#[cfg(test)]
6629mod checked_increment_tests {
6630 use super::checked_increment;
6631 use std::sync::atomic::{AtomicU64, Ordering};
6632
6633 #[test]
6634 fn returns_the_previous_value_and_stops_at_the_maximum() {
6635 let counter = AtomicU64::new(7);
6636 assert_eq!(checked_increment(&counter), Some(7));
6637 assert_eq!(counter.load(Ordering::Relaxed), 8);
6638
6639 let full = AtomicU64::new(u64::MAX);
6640 assert_eq!(checked_increment(&full), None);
6641 assert_eq!(
6642 full.load(Ordering::Relaxed),
6643 u64::MAX,
6644 "an exhausted counter is left unchanged"
6645 );
6646 }
6647}