#![forbid(unsafe_code)]
pub mod activity;
pub mod consumer;
mod operator_app;
pub mod policy_cache;
#[cfg(any(test, feature = "test-support"))]
mod stand_in;
#[cfg(feature = "test-support")]
pub mod test_support;
pub use activity::*;
pub use consumer::{
is_retryable_route_open_code, CallError, CallOptions, CallReply, CancelHandle, CancelOutcome,
CatalogList, CloseRouteOptions, ConnectionState, ConsumerCapabilityError, ConsumerError,
ConsumerOptions, ControlPush, CorrelationId, HealthProbeResponse, NotSentCause,
OutcomeUnknownCause, PendingReply, PushEvent, RequestBody, RequestReservation, RetryBackoff,
ReverseRequestContext, ReverseRequestError, ReverseRequestRegistrationError,
ReverseRequestRegistry, RouteCloseDisposition, RouteCloseReason, RouteEndReason,
RoutePollResult, ScopeBindProofRefusal, ScopeSelector, ServerDescription, SpawnStreamError,
SpawnSubscription, StderrTailResponse, SubcConsumer, SubscribeOptions, Subscription,
SubscriptionClosed, SubscriptionEnd, SupervisorAck, SupervisorEntry, SupervisorHealthResponse,
SupervisorListResponse, SupervisorProvenanceResponse, DEFAULT_CALL_TIMEOUT,
DEFAULT_LIVENESS_PROBE_WINDOW, DEFAULT_ROUTE_RETRY_DEADLINE, SPAWN_CURSOR_INCARNATION_MISMATCH,
SPAWN_CURSOR_TOO_OLD, SPAWN_SUBSCRIBER_LAGGED,
};
pub use operator_app::{operator_app_candidates, operator_app_path, OperatorAppPathError};
pub use policy_cache::{
PolicyResolveError, PolicyResolver, PolicyResolverConfig, PolicyResolverFootprint,
PolicyVerdict, ProjectRef, Subject, DEFAULT_POLICY_RESOLVER_MODULE_ID,
};
use std::{
collections::{HashMap, VecDeque},
env,
error::Error,
ffi::OsString,
fmt,
future::Future,
io,
panic::{catch_unwind, AssertUnwindSafe},
path::{Path, PathBuf},
pin::Pin,
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc, Mutex,
},
time::{Duration, SystemTime, UNIX_EPOCH},
};
pub use async_trait::async_trait;
pub use subc_control::{CatalogEntry, ConsumerIdentity};
mod scope_served_by;
pub use scope_served_by::{ScopeServedByAnswer, ScopeServedByReply};
mod request_pools;
use request_pools::{wait_for_permit, DispatchPools, PermitWaitOutcome, WaitingRequest};
pub use request_pools::{RequestClass, RequestPool, RequestPoolSnapshot, RequestPools};
pub use subc_os::launch_nonce;
use subc_protocol::{
call_mode::derive_call_mode,
error_codes,
manifest::ModuleManifest,
scope::{
ScopeEnd, ScopeEndResult, ScopeEnded, ScopeRecord, ScopeRecordResult, ScopeStamp,
ScopeStatus, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
},
session::{
ModuleControlCommand, ModuleControlRequest, ModuleControlRequestFromModule,
ModuleControlResponse, ModuleControlResponseToModule, OperatorConfirmKind,
OperatorConfirmReply, OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK,
MODULE_TO_SUBC_OP_CATALOG_UPDATE,
},
BindIdentity, ErrorBody, Flags, Frame, FrameBuildError, FrameType, ModuleHelloAckBody,
ModuleHelloBody, Principal, Priority, RouteTarget, PROTOCOL_VERSION, SUBC_MODULE_ID_ENV,
};
pub use subc_protocol::{
call_mode::{CallMode, CALL_MODES_CAPABILITY},
manifest::{
build_provenance, CapabilityDeclarations, CapabilityNeed, CapabilityRequirement,
ExecutionMode, LaunchNonceSource, ManifestProvenance, ProvenanceFormError, ProviderRole,
Tool, PROVENANCE_SENTINELS,
},
session::{HealthReport, HealthStatus},
AdmissionClass, MachineId, SUBC_PROTOCOL_CRATE_VERSION,
};
pub use subc_transport::connection_file::{
discover, discovery_candidates, Discovered, DiscoveryError,
};
use subc_transport::{
authenticate_client, connection_file, read_frame, write_frame, AuthError, ConnectionFileError,
FrameIoError,
};
use tokio::{
io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufWriter},
net::TcpStream,
sync::{mpsc, oneshot},
time::{timeout, timeout_at, Instant},
};
use tokio_util::{
sync::{CancellationToken, WaitForCancellationFuture},
task::TaskTracker,
};
const AUTH_DEADLINE: Duration = Duration::from_secs(2);
const CATALOG_UPDATE_TIMEOUT: Duration = Duration::from_secs(10);
const OPERATOR_CONFIRM_TIMEOUT: Duration = Duration::from_secs(290);
const OPERATOR_CONFIRM_OP: &str = "operator.confirm";
const EGRESS_BUFFER: usize = 64;
const HANDLER_TASK_CAPACITY: usize = 64;
const DISPATCH_SATURATION_WAIT: Duration = Duration::from_secs(2);
const HELLO_CORR: u64 = 1;
const DEFAULT_WRITER_DRAIN_TIMEOUT: Duration = Duration::from_secs(2);
const DRAINING_HOOK_START_LIMIT: Duration = Duration::from_secs(2);
static NEXT_MODULE_CONNECTION_TOKEN: AtomicU64 = AtomicU64::new(1);
const DETACHED_CONNECTION_TOKEN: u64 = 0;
type RequestKey = (u16, u32, u64);
type InFlight = Arc<Mutex<HashMap<RequestKey, CancellationToken>>>;
#[derive(Clone, Copy)]
pub struct RouteHandle {
pub channel: u16,
pub epoch: u32,
connection_token: u64,
reverse_request_registry_id: u64,
}
impl RouteHandle {
pub fn detached(channel: u16, epoch: u32) -> Self {
Self::new(channel, epoch, DETACHED_CONNECTION_TOKEN)
}
pub(crate) fn new(channel: u16, epoch: u32, connection_token: u64) -> Self {
Self {
channel,
epoch,
connection_token,
reverse_request_registry_id: 0,
}
}
pub(crate) fn new_consumer(
channel: u16,
epoch: u32,
connection_token: u64,
reverse_requests: ReverseRequestRegistry,
) -> Self {
reverse_requests.seal();
Self {
channel,
epoch,
connection_token,
reverse_request_registry_id: consumer::install_reverse_request_registry(
reverse_requests,
),
}
}
pub(crate) fn connection_token(self) -> u64 {
self.connection_token
}
pub fn on_request<F, Fut>(
&self,
method_family: impl Into<String>,
handler: F,
) -> Result<(), ReverseRequestRegistrationError>
where
F: Fn(Vec<u8>, ReverseRequestContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Vec<u8>> + Send + 'static,
{
consumer::route_reverse_request_registry(self.reverse_request_registry_id)?
.on_request(method_family, handler)
}
pub fn on_request_fallible<F, Fut>(
&self,
method_family: impl Into<String>,
handler: F,
) -> Result<(), ReverseRequestRegistrationError>
where
F: Fn(Vec<u8>, ReverseRequestContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<Vec<u8>, ReverseRequestError>> + Send + 'static,
{
consumer::route_reverse_request_registry(self.reverse_request_registry_id)?
.on_request_fallible(method_family, handler)
}
pub(crate) fn reverse_request_registry_id(self) -> u64 {
self.reverse_request_registry_id
}
}
impl PartialEq for RouteHandle {
fn eq(&self, other: &Self) -> bool {
self.channel == other.channel
&& self.epoch == other.epoch
&& self.connection_token == other.connection_token
}
}
impl Eq for RouteHandle {}
impl std::hash::Hash for RouteHandle {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
std::hash::Hash::hash(&self.channel, state);
std::hash::Hash::hash(&self.epoch, state);
std::hash::Hash::hash(&self.connection_token, state);
}
}
impl fmt::Debug for RouteHandle {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RouteHandle")
.field("channel", &self.channel)
.field("epoch", &self.epoch)
.finish_non_exhaustive()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LiveRootsSnapshot {
pub roots: Vec<subc_protocol::session::LiveRoot>,
pub unknown_root_bindings: u64,
pub total_bindings: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct ScopeSyncReply {
pub generation: u64,
pub results: Vec<ScopeRecordResult>,
pub ended: Vec<ScopeEnded>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct ScopeApplyReply {
pub generation: u64,
pub results: Vec<ScopeRecordResult>,
pub end_results: Vec<ScopeEndResult>,
pub ended: Vec<ScopeEnded>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct ScopeDescribeReply {
pub status: ScopeStatus,
pub scope_epoch: Option<u64>,
pub daemon_incarnation: String,
pub owner_synced: bool,
pub owner_configured: bool,
pub scope: Option<ScopeStamp>,
}
type CatalogUpdateReply =
oneshot::Sender<Result<ModuleControlResponseToModule, CatalogUpdateError>>;
type CatalogUpdateWaiter =
oneshot::Receiver<Result<ModuleControlResponseToModule, CatalogUpdateError>>;
type CatalogUpdateRequest = (u64, mpsc::Sender<Frame>, CatalogUpdateWaiter);
type OperatorConfirmRequestWaiter = (u64, mpsc::Sender<Frame>, oneshot::Receiver<Frame>);
pub type ModuleServeFuture = Pin<Box<dyn Future<Output = Result<(), SubcModuleError>> + Send>>;
#[derive(Clone)]
struct RequestDispatcher {
in_flight: InFlight,
pools: Arc<DispatchPools>,
draining_hooks: Arc<Mutex<Vec<oneshot::Receiver<()>>>>,
undecodable_push_logged: Arc<AtomicBool>,
failure: Arc<Mutex<Option<SubcModuleError>>>,
failed: CancellationToken,
tasks: TaskTracker,
aborts: Arc<Mutex<Vec<tokio::task::AbortHandle>>>,
undelivered_replies: Arc<AtomicU64>,
}
impl RequestDispatcher {
#[cfg(test)]
fn new() -> Self {
Self::with_pools(Arc::new(DispatchPools::new(RequestPools::new())))
}
fn with_pools(pools: Arc<DispatchPools>) -> Self {
Self {
in_flight: Arc::new(Mutex::new(HashMap::new())),
pools,
draining_hooks: Arc::new(Mutex::new(Vec::new())),
undecodable_push_logged: Arc::new(AtomicBool::new(false)),
failure: Arc::new(Mutex::new(None)),
failed: CancellationToken::new(),
tasks: TaskTracker::new(),
aborts: Arc::new(Mutex::new(Vec::new())),
undelivered_replies: Arc::new(AtomicU64::new(0)),
}
}
fn fail(&self, error: SubcModuleError) {
let mut failure = self.failure.lock().unwrap_or_else(|e| e.into_inner());
if failure.is_none() {
*failure = Some(error);
}
self.failed.cancel();
}
fn take_failure(&self) -> Option<SubcModuleError> {
self.failure
.lock()
.unwrap_or_else(|e| e.into_inner())
.take()
}
fn spawn(&self, future: impl Future<Output = ()> + Send + 'static) {
let task = self.tasks.spawn(future);
let mut aborts = self.aborts.lock().unwrap_or_else(|e| e.into_inner());
aborts.retain(|task| !task.is_finished());
aborts.push(task.abort_handle());
}
fn abort_all(&self) {
for task in self
.aborts
.lock()
.unwrap_or_else(|e| e.into_inner())
.drain(..)
{
task.abort();
}
}
fn fold_health(&self, mut report: HealthReport) -> HealthReport {
if let Some(saturation) = self.pools.saturation() {
if report.status == HealthStatus::Ok {
report.status = HealthStatus::Degraded;
}
report.detail = Some(match report.detail {
Some(detail) => format!("{saturation}; {detail}"),
None => saturation,
});
}
report
}
async fn wait_for_draining_hooks_to_start(&self) {
let started = std::mem::take(
&mut *self
.draining_hooks
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()),
);
let deadline = tokio::time::Instant::now() + DRAINING_HOOK_START_LIMIT;
for hook in started {
if tokio::time::timeout_at(deadline, hook).await.is_err() {
return;
}
}
}
fn cancel_all(&self) {
self.pools.close();
let cancelled = match self.in_flight.lock() {
Ok(guard) => guard.values().cloned().collect::<Vec<_>>(),
Err(poisoned) => poisoned.into_inner().values().cloned().collect(),
};
for cancellation in cancelled {
cancellation.cancel();
}
}
}
#[derive(Clone)]
pub struct ModuleHandle {
shared: Arc<ModuleHandleShared>,
}
struct ModuleHandleShared {
negotiated_ver: u8,
daemon_capabilities: Vec<String>,
supports_catalog_update: bool,
supports_live_roots: bool,
supports_scope_sync: bool,
supports_scope_apply: bool,
supports_scope_describe: bool,
supports_operator_confirm: bool,
supports_activity: bool,
connection_runtime: tokio::runtime::Handle,
connection_token: u64,
machine_id: Option<MachineId>,
live_routes: Mutex<HashMap<u16, RouteHandle>>,
dropped_route_frames: AtomicU64,
close_token: CancellationToken,
dispatch_pools: Mutex<Arc<DispatchPools>>,
catalog_send: tokio::sync::Mutex<()>,
inner: Mutex<ModuleHandleState>,
}
struct ModuleHandleState {
writer: Option<mpsc::Sender<Frame>>,
next_corr: Option<u64>,
pending_catalog_updates: HashMap<u64, CatalogUpdateReply>,
pending_operator_confirms: HashMap<u64, oneshot::Sender<Frame>>,
pending_activity_records: HashMap<u64, oneshot::Sender<Frame>>,
pending_scope_served_by: HashMap<u64, oneshot::Sender<Frame>>,
current_provides: Option<Vec<ProviderRole>>,
catalog_changes: VecDeque<CatalogChange>,
closed: bool,
}
struct CatalogChange {
corr: u64,
provides: Vec<ProviderRole>,
outcome: Option<CatalogChangeOutcome>,
}
enum CatalogChangeOutcome {
Accepted,
Rejected,
Unknown,
}
struct CatalogUpdateGuard {
shared: Arc<ModuleHandleShared>,
corr: u64,
}
impl Drop for CatalogUpdateGuard {
fn drop(&mut self) {
self.shared.remove_pending_catalog_update(self.corr);
}
}
impl ModuleHandle {
fn new(
ack: &ModuleHelloAckBody,
writer: mpsc::Sender<Frame>,
connection_token: u64,
close_token: CancellationToken,
connection_runtime: tokio::runtime::Handle,
) -> Self {
Self {
shared: Arc::new(ModuleHandleShared {
negotiated_ver: ack.negotiated_ver,
daemon_capabilities: ack.subc_capabilities.clone(),
supports_catalog_update: ack
.subc_ops
.iter()
.any(|op| op == MODULE_TO_SUBC_OP_CATALOG_UPDATE),
supports_live_roots: ack.subc_ops.iter().any(|op| op == "supervisor.live_roots"),
supports_scope_sync: ack.subc_ops.iter().any(|op| op == SCOPE_SYNC_OP),
supports_scope_apply: ack.subc_ops.iter().any(|op| op == SCOPE_APPLY_OP),
supports_scope_describe: ack.subc_ops.iter().any(|op| op == SCOPE_DESCRIBE_OP),
supports_operator_confirm: ack.subc_ops.iter().any(|op| op == OPERATOR_CONFIRM_OP),
supports_activity: ack
.subc_capabilities
.iter()
.any(|capability| capability == ACTIVITY_LEDGER_CAPABILITY),
connection_runtime,
connection_token,
machine_id: ack
.machine_id
.as_deref()
.and_then(|id| MachineId::parse(id).ok()),
live_routes: Mutex::new(HashMap::new()),
dropped_route_frames: AtomicU64::new(0),
close_token,
dispatch_pools: Mutex::new(Arc::new(DispatchPools::new(RequestPools::new()))),
catalog_send: tokio::sync::Mutex::new(()),
inner: Mutex::new(ModuleHandleState {
writer: Some(writer),
next_corr: Some(HELLO_CORR + 1),
pending_catalog_updates: HashMap::new(),
pending_operator_confirms: HashMap::new(),
pending_activity_records: HashMap::new(),
pending_scope_served_by: HashMap::new(),
current_provides: None,
catalog_changes: VecDeque::new(),
closed: false,
}),
}),
}
}
pub fn machine_id(&self) -> Option<&MachineId> {
self.shared.machine_id.as_ref()
}
pub fn daemon_capabilities(&self) -> &[String] {
&self.shared.daemon_capabilities
}
pub fn daemon_supports(&self, capability: &str) -> bool {
self.shared
.daemon_capabilities
.iter()
.any(|advertised| advertised == capability)
}
pub fn closed(&self) -> impl Future<Output = ()> + Send + 'static {
self.shared.close_token.clone().cancelled_owned()
}
pub fn is_closed(&self) -> bool {
self.shared.close_token.is_cancelled()
}
pub async fn catalog_update(
&self,
provides: Vec<ProviderRole>,
) -> Result<(), CatalogUpdateError> {
self.catalog_update_inner(Some(provides), None, None).await
}
pub async fn catalog_update_with_capabilities(
&self,
provides: Vec<ProviderRole>,
capabilities: CapabilityDeclarations,
) -> Result<(), CatalogUpdateError> {
self.catalog_update_inner(Some(provides), Some(capabilities), None)
.await
}
pub async fn set_ready(&self, ready: bool) -> Result<(), CatalogUpdateError> {
self.catalog_update_inner(None, None, Some(ready)).await
}
async fn catalog_update_inner(
&self,
provides: Option<Vec<ProviderRole>>,
capabilities: Option<CapabilityDeclarations>,
ready: Option<bool>,
) -> Result<(), CatalogUpdateError> {
if !self.shared.supports_catalog_update {
return Err(CatalogUpdateError::NotSupported);
}
let send_guard = self.shared.catalog_send.lock().await;
let provides = match provides {
Some(provides) => provides,
None => {
let inner = self.shared.lock_inner();
if inner.closed {
return Err(CatalogUpdateError::ConnectionClosed);
}
inner
.current_provides
.as_ref()
.filter(|_| inner.catalog_changes.is_empty())
.cloned()
.ok_or_else(|| {
CatalogUpdateError::Protocol(
"current provider roles are unknown or a catalog update is in flight"
.into(),
)
})?
}
};
let body = serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
provides: provides.clone(),
capabilities,
ready,
})
.map_err(|err| {
CatalogUpdateError::Protocol(format!(
"failed to encode catalog.update request body: {err}"
))
})?;
let (corr, writer, rx) = self.shared.begin_catalog_update()?;
let _pending = CatalogUpdateGuard {
shared: self.shared.clone(),
corr,
};
let frame = Frame::build_with_version(
self.shared.negotiated_ver,
FrameType::Request,
control_flags(),
0,
0,
corr,
body,
)
.map_err(|err| {
self.shared.remove_pending_catalog_update(corr);
CatalogUpdateError::Protocol(format!(
"failed to build catalog.update request frame: {err}"
))
})?;
let permit = match writer.reserve().await {
Ok(permit) => permit,
Err(_) => {
self.shared.remove_pending_catalog_update(corr);
return Err(CatalogUpdateError::ConnectionClosed);
}
};
self.shared
.lock_inner()
.catalog_changes
.push_back(CatalogChange {
corr,
provides,
outcome: None,
});
permit.send(frame);
drop(send_guard);
match timeout(CATALOG_UPDATE_TIMEOUT, rx).await {
Ok(Ok(Ok(ModuleControlResponseToModule::CatalogUpdate {}))) => Ok(()),
Ok(Ok(Ok(_))) => Err(CatalogUpdateError::Protocol(
"unexpected catalog.update response".into(),
)),
Ok(Ok(Err(err))) => Err(err),
Ok(Err(_)) => Err(CatalogUpdateError::ConnectionClosed),
Err(_) => {
self.shared.remove_pending_catalog_update(corr);
Err(CatalogUpdateError::Timeout)
}
}
}
pub async fn live_roots(&self) -> Result<LiveRootsSnapshot, CatalogUpdateError> {
if !self.shared.supports_live_roots {
return Err(CatalogUpdateError::NotSupported);
}
let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {})
.map_err(|err| CatalogUpdateError::Protocol(err.to_string()))?;
let (corr, writer, rx) = self.shared.begin_catalog_update()?;
let frame = Frame::build_with_version(
self.shared.negotiated_ver,
FrameType::Request,
control_flags(),
0,
0,
corr,
body,
)
.map_err(|err| {
self.shared.remove_pending_catalog_update(corr);
CatalogUpdateError::Protocol(err.to_string())
})?;
if writer.send(frame).await.is_err() {
self.shared.remove_pending_catalog_update(corr);
return Err(CatalogUpdateError::ConnectionClosed);
}
match timeout(CATALOG_UPDATE_TIMEOUT, rx).await {
Ok(Ok(Ok(ModuleControlResponseToModule::LiveRoots {
roots,
unknown_root_bindings,
total_bindings,
}))) => Ok(LiveRootsSnapshot {
roots,
unknown_root_bindings,
total_bindings,
}),
Ok(Ok(Ok(_))) => Err(CatalogUpdateError::Protocol(
"unexpected live_roots response".into(),
)),
Ok(Ok(Err(err))) => Err(err),
Ok(Err(_)) => Err(CatalogUpdateError::ConnectionClosed),
Err(_) => {
self.shared.remove_pending_catalog_update(corr);
Err(CatalogUpdateError::Timeout)
}
}
}
pub async fn scope_sync(
&self,
generation: u64,
scopes: Vec<ScopeRecord>,
) -> Result<ScopeSyncReply, ScopeCallError> {
if !self.shared.supports_scope_sync {
return Err(ScopeCallError::NotSupported { op: SCOPE_SYNC_OP });
}
let request = ModuleControlRequestFromModule::ScopeSync { generation, scopes };
match self.scope_call(SCOPE_SYNC_OP, &request).await? {
ModuleControlResponseToModule::ScopeSync {
generation,
results,
ended,
} => Ok(ScopeSyncReply {
generation,
results,
ended,
}),
other => Err(ScopeCallError::Protocol(format!(
"unexpected {SCOPE_SYNC_OP} response: {other:?}"
))),
}
}
pub async fn scope_apply(
&self,
generation: u64,
upsert: Vec<ScopeRecord>,
end: Vec<ScopeEnd>,
) -> Result<ScopeApplyReply, ScopeCallError> {
if !self.shared.supports_scope_apply {
return Err(ScopeCallError::NotSupported { op: SCOPE_APPLY_OP });
}
let request = ModuleControlRequestFromModule::ScopeApply {
generation,
upsert,
end,
};
match self.scope_call(SCOPE_APPLY_OP, &request).await? {
ModuleControlResponseToModule::ScopeApply {
generation,
results,
end_results,
ended,
} => Ok(ScopeApplyReply {
generation,
results,
end_results,
ended,
}),
other => Err(ScopeCallError::Protocol(format!(
"unexpected {SCOPE_APPLY_OP} response: {other:?}"
))),
}
}
pub async fn scope_describe(
&self,
owner: Principal,
scope_ref: String,
) -> Result<ScopeDescribeReply, ScopeCallError> {
if !self.shared.supports_scope_describe {
return Err(ScopeCallError::NotSupported {
op: SCOPE_DESCRIBE_OP,
});
}
let request = ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref };
match self.scope_call(SCOPE_DESCRIBE_OP, &request).await? {
ModuleControlResponseToModule::ScopeDescribe {
status,
scope_epoch,
daemon_incarnation,
owner_synced,
owner_configured,
scope,
} => Ok(ScopeDescribeReply {
status,
scope_epoch,
daemon_incarnation,
owner_synced,
owner_configured,
scope,
}),
other => Err(ScopeCallError::Protocol(format!(
"unexpected {SCOPE_DESCRIBE_OP} response: {other:?}"
))),
}
}
pub async fn confirm_operator(
&self,
summary: &str,
caller_route: &RouteHandle,
) -> Result<(), OperatorConfirmError> {
self.confirm_operator_with_deadline(summary, caller_route, OPERATOR_CONFIRM_TIMEOUT)
.await
}
pub async fn confirm_operation(
&self,
summary: &str,
caller_route: &RouteHandle,
operation: &str,
) -> Result<(), OperatorConfirmError> {
if operation.is_empty() {
return Err(OperatorConfirmError::OperationInvalid);
}
self.confirm_operation_with_deadline(
summary,
caller_route,
operation,
OPERATOR_CONFIRM_TIMEOUT,
)
.await
}
pub async fn confirm_enrollment(
&self,
summary: &str,
caller_route: &RouteHandle,
) -> Result<(), OperatorConfirmError> {
self.confirm_inner(
summary,
caller_route,
Some(OperatorConfirmKind::Enrollment),
None,
OPERATOR_CONFIRM_TIMEOUT,
)
.await
}
pub(crate) async fn confirm_operator_with_deadline(
&self,
summary: &str,
caller_route: &RouteHandle,
duration: Duration,
) -> Result<(), OperatorConfirmError> {
self.confirm_inner(summary, caller_route, None, None, duration)
.await
}
async fn confirm_operation_with_deadline(
&self,
summary: &str,
caller_route: &RouteHandle,
operation: &str,
duration: Duration,
) -> Result<(), OperatorConfirmError> {
self.confirm_inner(
summary,
caller_route,
Some(OperatorConfirmKind::Operation),
Some(operation.to_owned()),
duration,
)
.await
}
async fn confirm_inner(
&self,
summary: &str,
caller_route: &RouteHandle,
kind: Option<OperatorConfirmKind>,
operation: Option<String>,
duration: Duration,
) -> Result<(), OperatorConfirmError> {
if caller_route.connection_token() != self.shared.connection_token {
return Err(OperatorConfirmError::NotPermitted);
}
if !self.shared.supports_operator_confirm {
return Err(OperatorConfirmError::Unsupported);
}
let deadline = Instant::now() + duration;
let body = serde_json::to_vec(
&OperatorConfirmRequest::new(summary, caller_route.channel, caller_route.epoch)
.with_kind(kind)
.with_operation(operation),
)
.map_err(|_| OperatorConfirmError::PresenceUnavailable)?;
let (corr, writer, rx) = self.shared.begin_operator_confirm()?;
let mut guard = OperatorConfirmGuard {
shared: Arc::clone(&self.shared),
runtime: self.shared.connection_runtime.clone(),
writer: writer.clone(),
corr,
accepted: false,
armed: true,
};
let frame = Frame::build_with_version(
self.shared.negotiated_ver,
FrameType::Request,
control_flags(),
0,
0,
corr,
body,
)
.map_err(|_| OperatorConfirmError::PresenceUnavailable)?;
tokio::select! {
result = timeout_at(deadline, writer.send(frame)) => {
result.map_err(|_| OperatorConfirmError::PresenceUnavailable)?
.map_err(|_| OperatorConfirmError::PresenceUnavailable)?;
}
_ = self.shared.close_token.cancelled() => {
return Err(OperatorConfirmError::PresenceUnavailable);
}
}
guard.accepted = true;
let reply = timeout_at(deadline, rx)
.await
.map_err(|_| OperatorConfirmError::PresenceUnavailable)?
.map_err(|_| OperatorConfirmError::PresenceUnavailable)?;
guard.armed = false;
match reply.header.ty {
FrameType::Response => {
match serde_json::from_slice::<OperatorConfirmReply>(&reply.body) {
Ok(body) if body.outcome == "confirmed" => Ok(()),
_ => Err(OperatorConfirmError::PresenceUnavailable),
}
}
FrameType::Error => match serde_json::from_slice::<ErrorBody>(&reply.body) {
Ok(body) => Err(match body.code.as_str() {
error_codes::OPERATOR_DECLINED => {
let detail = body.detail.as_ref();
if detail
.and_then(|detail| detail.get("reason"))
.and_then(serde_json::Value::as_str)
== Some("backoff")
{
OperatorConfirmError::Backoff {
retry_after_ms: detail
.and_then(|detail| detail.get("retry_after_ms"))
.and_then(serde_json::Value::as_u64),
}
} else {
OperatorConfirmError::Declined
}
}
error_codes::OPERATOR_PRESENCE_UNAVAILABLE
if body
.detail
.as_ref()
.and_then(|detail| detail.get("reason"))
.and_then(serde_json::Value::as_str)
== Some("unsupported_platform") =>
{
OperatorConfirmError::PresenceUnsupportedPlatform
}
error_codes::OPERATOR_SUMMARY_INVALID => OperatorConfirmError::SummaryInvalid,
error_codes::OPERATOR_REQUEST_NOT_PERMITTED | "not_registered" => {
OperatorConfirmError::NotPermitted
}
"unsupported_control_frame" => OperatorConfirmError::Unsupported,
_ => OperatorConfirmError::PresenceUnavailable,
}),
Err(_) => Err(OperatorConfirmError::PresenceUnavailable),
},
_ => Err(OperatorConfirmError::PresenceUnavailable),
}
}
async fn scope_call(
&self,
op: &'static str,
request: &ModuleControlRequestFromModule,
) -> Result<ModuleControlResponseToModule, ScopeCallError> {
let body = serde_json::to_vec(request).map_err(|err| {
ScopeCallError::Protocol(format!("failed to encode {op} request body: {err}"))
})?;
let (corr, writer, rx) = self
.shared
.begin_catalog_update()
.map_err(ScopeCallError::from_control_error)?;
let frame = Frame::build_with_version(
self.shared.negotiated_ver,
FrameType::Request,
control_flags(),
0,
0,
corr,
body,
)
.map_err(|err| {
self.shared.remove_pending_catalog_update(corr);
ScopeCallError::Protocol(format!("failed to build {op} request frame: {err}"))
})?;
if writer.send(frame).await.is_err() {
self.shared.remove_pending_catalog_update(corr);
return Err(ScopeCallError::ConnectionClosed);
}
match timeout(CATALOG_UPDATE_TIMEOUT, rx).await {
Ok(Ok(reply)) => reply.map_err(ScopeCallError::from_control_error),
Ok(Err(_)) => Err(ScopeCallError::ConnectionClosed),
Err(_) => {
self.shared.remove_pending_catalog_update(corr);
Err(ScopeCallError::Timeout)
}
}
}
pub async fn push(
&self,
handle: &RouteHandle,
body: Vec<u8>,
admission_class: Option<AdmissionClass>,
) -> Result<(), SubcModuleError> {
self.validate_route(*handle)?;
let writer = self
.shared
.lock_inner()
.writer
.clone()
.ok_or(SubcModuleError::WriterClosed)?;
let frame = Frame::build_with_version(
self.shared.negotiated_ver,
FrameType::Push,
data_flags().with_admission_class(admission_class.unwrap_or(AdmissionClass::Normal)),
handle.channel,
handle.epoch,
0,
body,
)
.map_err(SubcModuleError::FrameBuild)?;
send_outbound(&writer, frame).await
}
pub fn dropped_route_frames(&self) -> u64 {
self.shared.dropped_route_frames.load(Ordering::Relaxed)
}
fn validate_route(&self, handle: RouteHandle) -> Result<(), SubcModuleError> {
if handle.connection_token() != self.shared.connection_token {
return Err(SubcModuleError::StaleRouteHandle(handle));
}
let routes = self
.shared
.live_routes
.lock()
.map_err(|_| SubcModuleError::InFlightPoisoned)?;
if routes.get(&handle.channel) == Some(&handle) {
Ok(())
} else {
Err(SubcModuleError::StaleRouteHandle(handle))
}
}
fn install_route(&self, handle: RouteHandle) -> Result<(), SubcModuleError> {
self.shared
.live_routes
.lock()
.map_err(|_| SubcModuleError::InFlightPoisoned)?
.insert(handle.channel, handle);
Ok(())
}
fn installed_route(&self, channel: u16) -> Result<Option<RouteHandle>, SubcModuleError> {
Ok(self
.shared
.live_routes
.lock()
.map_err(|_| SubcModuleError::InFlightPoisoned)?
.get(&channel)
.copied())
}
fn remove_route(&self, handle: RouteHandle) -> Result<bool, SubcModuleError> {
let mut routes = self
.shared
.live_routes
.lock()
.map_err(|_| SubcModuleError::InFlightPoisoned)?;
if routes.get(&handle.channel) == Some(&handle) {
routes.remove(&handle.channel);
Ok(true)
} else {
Ok(false)
}
}
fn validate_ingress(&self, channel: u16, epoch: u32) -> Result<bool, SubcModuleError> {
let handle = RouteHandle::new(channel, epoch, self.shared.connection_token);
let valid = self
.shared
.live_routes
.lock()
.map_err(|_| SubcModuleError::InFlightPoisoned)?
.get(&channel)
== Some(&handle);
if !valid {
self.shared
.dropped_route_frames
.fetch_add(1, Ordering::Relaxed);
}
Ok(valid)
}
fn route_handle(&self, channel: u16, epoch: u32) -> RouteHandle {
RouteHandle::new(channel, epoch, self.shared.connection_token)
}
fn handle_control_reply(&self, frame: Frame) -> bool {
let served_by_reply = self
.shared
.lock_inner()
.pending_scope_served_by
.remove(&frame.header.corr);
if let Some(reply) = served_by_reply {
let _ = reply.send(frame);
return true;
}
let activity_reply = self
.shared
.lock_inner()
.pending_activity_records
.remove(&frame.header.corr);
if let Some(reply) = activity_reply {
let _ = reply.send(frame);
return true;
}
let operator_reply = self
.shared
.lock_inner()
.pending_operator_confirms
.remove(&frame.header.corr);
if let Some(reply) = operator_reply {
let _ = reply.send(frame);
return true;
}
let Some(reply) = self.shared.take_pending_catalog_update(frame.header.corr) else {
return false;
};
let result = match frame.header.ty {
FrameType::Response => serde_json::from_slice::<ModuleControlResponseToModule>(
&frame.body,
)
.map_err(|err| {
CatalogUpdateError::Protocol(format!("invalid module control response body: {err}"))
}),
FrameType::Error => match serde_json::from_slice::<ErrorBody>(&frame.body) {
Ok(body) => Err(match body.code.as_str() {
"catalog_update_frozen_field" => CatalogUpdateError::FrozenField(body),
"not_registered" => CatalogUpdateError::NotRegistered(body),
_ => CatalogUpdateError::Rejected(body),
}),
Err(err) => Err(CatalogUpdateError::Protocol(format!(
"invalid catalog.update error body: {err}"
))),
},
ty => Err(CatalogUpdateError::Protocol(format!(
"unexpected catalog.update terminal frame: {ty:?}"
))),
};
let outcome = match &result {
Ok(ModuleControlResponseToModule::CatalogUpdate {}) => CatalogChangeOutcome::Accepted,
Err(CatalogUpdateError::FrozenField(_))
| Err(CatalogUpdateError::NotRegistered(_))
| Err(CatalogUpdateError::Rejected(_)) => CatalogChangeOutcome::Rejected,
_ => CatalogChangeOutcome::Unknown,
};
self.shared
.finish_catalog_change(frame.header.corr, outcome);
let _ = reply.send(result);
true
}
fn close_connection(&self) {
self.shared.close_connection();
}
}
impl ModuleHandleShared {
fn begin_catalog_update(&self) -> Result<CatalogUpdateRequest, CatalogUpdateError> {
let mut inner = self.lock_inner();
if inner.closed {
return Err(CatalogUpdateError::ConnectionClosed);
}
let Some(writer) = inner.writer.clone() else {
drop(inner);
self.close_connection();
return Err(CatalogUpdateError::ConnectionClosed);
};
let Some(corr) = next_module_control_corr(&mut inner) else {
drop(inner);
self.close_connection();
return Err(CatalogUpdateError::ConnectionClosed);
};
let (tx, rx) = oneshot::channel();
inner.pending_catalog_updates.insert(corr, tx);
Ok((corr, writer, rx))
}
fn begin_operator_confirm(&self) -> Result<OperatorConfirmRequestWaiter, OperatorConfirmError> {
let mut inner = self.lock_inner();
if inner.closed {
return Err(OperatorConfirmError::PresenceUnavailable);
}
let Some(writer) = inner.writer.clone() else {
drop(inner);
self.close_connection();
return Err(OperatorConfirmError::PresenceUnavailable);
};
let Some(corr) = next_module_control_corr(&mut inner) else {
drop(inner);
self.close_connection();
return Err(OperatorConfirmError::PresenceUnavailable);
};
let (tx, rx) = oneshot::channel();
inner.pending_operator_confirms.insert(corr, tx);
Ok((corr, writer, rx))
}
fn take_pending_catalog_update(&self, corr: u64) -> Option<CatalogUpdateReply> {
self.lock_inner().pending_catalog_updates.remove(&corr)
}
fn remove_pending_catalog_update(&self, corr: u64) {
let removed = self
.lock_inner()
.pending_catalog_updates
.remove(&corr)
.is_some();
if removed {
self.finish_catalog_change(corr, CatalogChangeOutcome::Unknown);
}
}
fn finish_catalog_change(&self, corr: u64, outcome: CatalogChangeOutcome) {
let mut inner = self.lock_inner();
if let Some(change) = inner
.catalog_changes
.iter_mut()
.find(|change| change.corr == corr)
{
change.outcome = Some(outcome);
}
while inner
.catalog_changes
.front()
.is_some_and(|change| change.outcome.is_some())
{
let change = inner.catalog_changes.pop_front().expect("completed change");
match change.outcome.expect("completed outcome") {
CatalogChangeOutcome::Accepted => inner.current_provides = Some(change.provides),
CatalogChangeOutcome::Rejected => {}
CatalogChangeOutcome::Unknown => inner.current_provides = None,
}
}
}
fn close_connection(&self) {
let pending = {
let mut inner = self.lock_inner();
if inner.closed {
return;
}
inner.closed = true;
inner.writer = None;
inner.current_provides = None;
inner.catalog_changes.clear();
self.close_token.cancel();
inner.pending_operator_confirms.clear();
inner.pending_activity_records.clear();
inner.pending_scope_served_by.clear();
inner
.pending_catalog_updates
.drain()
.map(|(_, reply)| reply)
.collect::<Vec<_>>()
};
self.clear_live_routes();
for reply in pending {
let _ = reply.send(Err(CatalogUpdateError::ConnectionClosed));
}
}
fn clear_live_routes(&self) {
if let Ok(mut routes) = self.live_routes.lock() {
routes.clear();
}
}
fn lock_inner(&self) -> std::sync::MutexGuard<'_, ModuleHandleState> {
self.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
}
fn next_module_control_corr(inner: &mut ModuleHandleState) -> Option<u64> {
let corr = inner.next_corr?;
inner.next_corr = corr.checked_add(1);
Some(corr)
}
struct OperatorConfirmGuard {
shared: Arc<ModuleHandleShared>,
runtime: tokio::runtime::Handle,
writer: mpsc::Sender<Frame>,
corr: u64,
accepted: bool,
armed: bool,
}
impl Drop for OperatorConfirmGuard {
fn drop(&mut self) {
if !self.armed {
return;
}
self.shared
.lock_inner()
.pending_operator_confirms
.remove(&self.corr);
if !self.accepted {
return;
}
let Ok(cancel) = Frame::build_with_version(
self.shared.negotiated_ver,
FrameType::Cancel,
control_flags(),
0,
0,
self.corr,
Vec::new(),
) else {
return;
};
match self.writer.try_send(cancel) {
Ok(()) | Err(mpsc::error::TrySendError::Closed(_)) => {}
Err(mpsc::error::TrySendError::Full(cancel)) => {
let writer = self.writer.clone();
self.runtime.spawn(async move {
let _ = writer.send(cancel).await;
});
}
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum OperatorConfirmError {
Declined,
Backoff {
retry_after_ms: Option<u64>,
},
PresenceUnavailable,
PresenceUnsupportedPlatform,
SummaryInvalid,
OperationInvalid,
NotPermitted,
Unsupported,
}
impl fmt::Display for OperatorConfirmError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::Declined => "operator declined the write",
Self::Backoff { retry_after_ms: None } => {
"operator confirmation refused without asking: a recent decline holds new prompts for a short time"
}
Self::PresenceUnavailable => "operator confirmation is unavailable",
Self::PresenceUnsupportedPlatform => {
"operator confirmation has no prompt on this platform"
}
Self::SummaryInvalid => "operator confirmation summary is invalid",
Self::OperationInvalid => "operator confirmation operation is invalid",
Self::NotPermitted => "operator confirmation is not permitted for this route",
Self::Unsupported => "daemon does not support operator confirmation",
Self::Backoff {
retry_after_ms: Some(_),
} => return write!(
f,
"operator confirmation refused without asking: a recent decline holds new prompts for {} s",
self.backoff_seconds().expect("backoff retry duration is present")
),
})
}
}
impl OperatorConfirmError {
fn backoff_seconds(&self) -> Option<u64> {
match self {
Self::Backoff {
retry_after_ms: Some(milliseconds),
} => Some(milliseconds / 1_000 + u64::from(!milliseconds.is_multiple_of(1_000))),
_ => None,
}
}
}
impl Error for OperatorConfirmError {}
#[derive(Debug)]
pub enum CatalogUpdateError {
NotSupported,
FrozenField(ErrorBody),
NotRegistered(ErrorBody),
Rejected(ErrorBody),
Timeout,
ConnectionClosed,
Protocol(String),
}
impl fmt::Display for CatalogUpdateError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NotSupported => write!(
f,
"daemon HELLO_ACK did not advertise catalog.update support"
),
Self::FrozenField(body) => {
write!(f, "catalog.update rejected frozen field: {}", body.message)
}
Self::NotRegistered(body) => write!(
f,
"catalog.update requires a registered module: {}",
body.message
),
Self::Rejected(body) => write!(
f,
"catalog.update rejected by subc: {} ({})",
body.code, body.message
),
Self::Timeout => write!(f, "catalog.update timed out waiting for an ACK"),
Self::ConnectionClosed => {
write!(f, "subc connection closed before catalog.update completed")
}
Self::Protocol(message) => write!(f, "catalog.update protocol error: {message}"),
}
}
}
impl Error for CatalogUpdateError {}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum ScopeCallError {
NotSupported { op: &'static str },
Refused { code: String, message: String },
Timeout,
ConnectionClosed,
Protocol(String),
}
impl ScopeCallError {
pub fn code(&self) -> Option<&str> {
match self {
Self::Refused { code, .. } => Some(code),
_ => None,
}
}
fn from_control_error(err: CatalogUpdateError) -> Self {
match err {
CatalogUpdateError::FrozenField(body)
| CatalogUpdateError::NotRegistered(body)
| CatalogUpdateError::Rejected(body) => Self::Refused {
code: body.code,
message: body.message,
},
CatalogUpdateError::Timeout => Self::Timeout,
CatalogUpdateError::ConnectionClosed => Self::ConnectionClosed,
CatalogUpdateError::Protocol(message) => Self::Protocol(message),
CatalogUpdateError::NotSupported => {
Self::Protocol("unexpected NotSupported from the control reply table".into())
}
}
}
}
impl fmt::Display for ScopeCallError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NotSupported { op } => {
write!(f, "daemon HELLO_ACK did not advertise {op} support")
}
Self::Refused { code, message } => {
write!(f, "subc refused the request: {code} ({message})")
}
Self::Timeout => write!(f, "timed out waiting for the daemon's reply"),
Self::ConnectionClosed => {
write!(f, "subc connection closed before the daemon replied")
}
Self::Protocol(message) => write!(f, "scope call protocol error: {message}"),
}
}
}
impl Error for ScopeCallError {}
#[async_trait]
pub trait ModuleHandler: Send + Sync + 'static {
async fn handle(&self, ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome;
async fn on_hello_ack(&self, _ack: &ModuleHelloAckBody) {}
async fn on_bind(&self, _req: &RouteBindRequest) -> BindDecision {
BindDecision::accept()
}
async fn on_bound(&self, _handle: &RouteHandle) {}
async fn health(&self) -> HealthReport {
HealthReport {
detail: Some("no health implementation; inherited default".to_string()),
..HealthReport::ok()
}
}
async fn on_route_gone(&self, _handle: &RouteHandle) {}
fn request_pools(&self) -> RequestPools {
RequestPools::new()
}
async fn on_draining(&self, _reason: RouteCloseReason, _deadline: SystemTime) {}
async fn on_connection_end(&self, _end: ConnectionEnd) {}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum ConnectionEnd {
Goodbye,
Eof,
Reset,
Closed,
}
impl ConnectionEnd {
pub fn as_str(self) -> &'static str {
match self {
Self::Goodbye => "goodbye",
Self::Eof => "eof",
Self::Reset => "reset",
Self::Closed => "closed",
}
}
}
impl std::fmt::Display for ConnectionEnd {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum HandlerOutcome {
Response(Vec<u8>),
Error { code: String, message: String },
ErrorWithDetail {
code: String,
message: String,
detail: serde_json::Value,
},
Streamed,
}
#[derive(Clone)]
pub struct RequestCtx {
handle: RouteHandle,
corr: u64,
ver: u8,
egress: mpsc::Sender<Frame>,
module_handle: ModuleHandle,
cancelled: CancellationToken,
binary_reply: Arc<AtomicBool>,
request_flags: Flags,
request_body: Arc<[u8]>,
}
impl RequestCtx {
pub fn module_handle(&self) -> ModuleHandle {
self.module_handle.clone()
}
pub fn request_payload(
&self,
) -> Result<Option<(serde_json::Value, &[u8])>, subc_protocol::framed_body::FramedBodyError>
{
if !self.request_flags.is_binary() {
return Ok(None);
}
subc_protocol::framed_body::decode(&self.request_body).map(Some)
}
pub fn route_handle(&self) -> RouteHandle {
self.handle
}
pub fn corr(&self) -> u64 {
self.corr
}
pub fn mark_binary_reply(&self) {
self.binary_reply.store(true, Ordering::Relaxed);
}
pub async fn emit(&self, body: Vec<u8>) -> Result<(), SubcModuleError> {
self.emit_with_admission(body, None).await
}
pub async fn emit_with_admission(
&self,
body: Vec<u8>,
admission_class: Option<AdmissionClass>,
) -> Result<(), SubcModuleError> {
self.module_handle.validate_route(self.handle)?;
if self.cancelled.is_cancelled() {
return Ok(());
}
self.send_frame(
FrameType::StreamData,
data_flags().with_admission_class(admission_class.unwrap_or(AdmissionClass::Normal)),
body,
)
.await
}
pub fn cancelled(&self) -> WaitForCancellationFuture<'_> {
self.cancelled.cancelled()
}
pub fn cancellation_token(&self) -> CancellationToken {
self.cancelled.clone()
}
async fn send_frame(
&self,
frame_type: FrameType,
flags: Flags,
body: Vec<u8>,
) -> Result<(), SubcModuleError> {
self.module_handle.validate_route(self.handle)?;
let (frame_type, flags, body) = bounded_terminal(frame_type, flags, body)?;
let frame = Frame::build_with_version(
self.ver,
frame_type,
flags,
self.handle.channel,
self.handle.epoch,
self.corr,
body,
)
.map_err(SubcModuleError::FrameBuild)?;
send_outbound(&self.egress, frame).await
}
}
fn checked_increment(counter: &AtomicU64) -> Option<u64> {
let mut current = counter.load(Ordering::Relaxed);
loop {
let next = current.checked_add(1)?;
match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
Ok(previous) => return Some(previous),
Err(actual) => current = actual,
}
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct RouteBindRequest {
pub handle: RouteHandle,
pub target: RouteTarget,
pub identity: BindIdentity,
pub principal: Option<Principal>,
pub consumer_capabilities: Option<Vec<String>>,
pub role_versions: Option<std::collections::BTreeMap<String, String>>,
pub admission_facts: Option<serde_json::Value>,
pub scope: Option<ScopeStamp>,
pub call_mode: CallMode,
}
impl RouteBindRequest {
pub fn new(handle: RouteHandle, target: RouteTarget, identity: BindIdentity) -> Self {
Self {
handle,
target,
identity,
principal: None,
consumer_capabilities: None,
role_versions: None,
admission_facts: None,
scope: None,
call_mode: derive_call_mode(None),
}
}
pub fn with_principal(mut self, principal: Principal) -> Self {
self.principal = Some(principal);
self
}
pub fn with_consumer_capabilities(mut self, consumer_capabilities: Vec<String>) -> Self {
self.consumer_capabilities = Some(consumer_capabilities);
self
}
pub fn with_role_versions(
mut self,
role_versions: std::collections::BTreeMap<String, String>,
) -> Self {
self.role_versions = Some(role_versions);
self
}
pub fn with_admission_facts(mut self, admission_facts: serde_json::Value) -> Self {
self.admission_facts = Some(admission_facts);
self
}
pub fn with_scope(mut self, scope: ScopeStamp) -> Self {
self.call_mode = derive_call_mode(Some(&scope));
self.scope = Some(scope);
self
}
pub fn with_call_mode(mut self, call_mode: CallMode) -> Self {
self.call_mode = call_mode;
self
}
}
#[derive(Debug, Clone)]
pub struct BindDecision {
kind: BindDecisionKind,
}
impl BindDecision {
pub fn accept() -> Self {
Self {
kind: BindDecisionKind::Accept,
}
}
pub fn reject(code: impl Into<String>, message: impl Into<String>) -> Self {
Self {
kind: BindDecisionKind::Reject {
code: code.into(),
message: message.into(),
},
}
}
}
#[derive(Debug, Clone)]
enum BindDecisionKind {
Accept,
Reject { code: String, message: String },
}
pub async fn serve<H>(manifest: ModuleManifest, handler: H) -> Result<(), SubcModuleError>
where
H: ModuleHandler,
{
ServeOptions::default().serve(manifest, handler).await
}
#[derive(Clone, Debug, Default)]
pub struct ModuleStopHandle {
token: CancellationToken,
}
impl ModuleStopHandle {
pub fn new() -> Self {
Self::default()
}
pub fn stop(&self) {
self.token.cancel();
}
}
#[derive(Clone, Debug)]
#[non_exhaustive]
pub struct ServeOptions {
writer_drain_timeout: Duration,
stop_handle: ModuleStopHandle,
}
impl Default for ServeOptions {
fn default() -> Self {
Self {
writer_drain_timeout: DEFAULT_WRITER_DRAIN_TIMEOUT,
stop_handle: ModuleStopHandle::new(),
}
}
}
impl ServeOptions {
pub fn with_writer_drain_timeout(mut self, timeout: Duration) -> Self {
self.writer_drain_timeout = timeout;
self
}
pub fn with_stop_handle(mut self, handle: ModuleStopHandle) -> Self {
self.stop_handle = handle;
self
}
pub async fn serve<H: ModuleHandler>(
self,
mut manifest: ModuleManifest,
handler: H,
) -> Result<(), SubcModuleError> {
let connection_file = parse_subc_arg(env::args_os().skip(1))?;
if let Some(module_id) = module_id_from_env()? {
manifest.module_id = module_id;
}
self.serve_with(&connection_file, manifest, handler).await
}
pub async fn serve_with<H: ModuleHandler>(
self,
connection_file: &Path,
manifest: ModuleManifest,
handler: H,
) -> Result<(), SubcModuleError> {
if let Some((_handle, future)) =
prepare_connection(connection_file, manifest, handler, self).await?
{
future.await?;
}
Ok(())
}
}
pub async fn serve_from_env_with_handle<H>(
mut manifest: ModuleManifest,
handler: H,
) -> Result<(ModuleHandle, ModuleServeFuture), SubcModuleError>
where
H: ModuleHandler,
{
let connection_file = parse_subc_arg(env::args_os().skip(1))?;
if let Some(module_id) = module_id_from_env()? {
manifest.module_id = module_id;
}
serve_with_handle(&connection_file, manifest, handler).await
}
pub async fn serve_with<H>(
connection_file: &Path,
manifest: ModuleManifest,
handler: H,
) -> Result<(), SubcModuleError>
where
H: ModuleHandler,
{
ServeOptions::default()
.serve_with(connection_file, manifest, handler)
.await
}
pub async fn serve_with_handle<H>(
connection_file: &Path,
manifest: ModuleManifest,
handler: H,
) -> Result<(ModuleHandle, ModuleServeFuture), SubcModuleError>
where
H: ModuleHandler,
{
prepare_connection(connection_file, manifest, handler, ServeOptions::default())
.await?
.ok_or(SubcModuleError::ConnectionClosedBeforeHelloAck)
}
async fn prepare_connection<H: ModuleHandler>(
connection_file: &Path,
manifest: ModuleManifest,
handler: H,
options: ServeOptions,
) -> Result<Option<(ModuleHandle, ModuleServeFuture)>, SubcModuleError> {
let stream = tokio::select! {
biased;
() = options.stop_handle.token.cancelled() => return Ok(None),
stream = connect_to_subc(connection_file) => stream?,
};
register_connection(stream, manifest, handler, options).await
}
async fn register_connection<S, H>(
stream: S,
manifest: ModuleManifest,
handler: H,
options: ServeOptions,
) -> Result<Option<(ModuleHandle, ModuleServeFuture)>, SubcModuleError>
where
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
H: ModuleHandler,
{
let (mut read_half, write_half) = tokio::io::split(stream);
let (tx, rx) = mpsc::channel::<Frame>(EGRESS_BUFFER);
let connection_runtime = tokio::runtime::Handle::current();
let mut writer = tokio::spawn(drain_writer(write_half, rx));
let handler = Arc::new(handler);
let mut hello_ack = None;
let provides = manifest.provides.clone();
let handshake = async {
send_hello(&tx, manifest).await?;
hello_ack = Some(expect_hello_ack(&mut read_half).await?);
handler
.on_hello_ack(hello_ack.as_ref().expect("validated HELLO_ACK"))
.await;
Ok::<_, SubcModuleError>(())
};
let mut writer_result = None;
let handshake_result = tokio::select! {
biased;
() = options.stop_handle.token.cancelled() => None,
result = handshake => Some(result),
result = &mut writer => {
writer_result = Some(result);
Some(Ok(()))
}
};
match handshake_result {
Some(Ok(())) => {}
other => {
let result = match other {
None => {
let deadline = Instant::now() + options.writer_drain_timeout;
let result = timeout_at(deadline, send_goodbye(&tx)).await;
drop(tx);
let flush = finish_writer_and_drain(writer, &mut read_half, deadline).await;
handler.on_connection_end(ConnectionEnd::Closed).await;
match result.unwrap_or(Ok(())) {
Ok(()) | Err(SubcModuleError::WriterClosed) => {}
Err(error) => return Err(error),
}
match flush.0 {
Err(SubcModuleError::FrameIo(FrameIoError::Io(error)))
if is_disconnect(&error) => {}
result => result?,
}
return Ok(None);
}
Some(Err(error)) => error,
Some(Ok(_)) => unreachable!(),
};
drop(tx);
let deadline = Instant::now() + options.writer_drain_timeout;
let _ = finish_writer_and_drain(writer, &mut read_half, deadline).await;
return Err(result);
}
}
let ack = match hello_ack {
Some(ack) => ack,
None => {
return Err(writer_ended(
writer_result.expect("writer ended before HELLO_ACK"),
))
}
};
let writer_end = writer_result.map(writer_connection_end);
let Some(connection_token) = checked_increment(&NEXT_MODULE_CONNECTION_TOKEN) else {
drop(tx);
let deadline = Instant::now() + options.writer_drain_timeout;
if writer_end.is_none() {
let _ = finish_writer_and_drain(writer, &mut read_half, deadline).await;
}
return Err(SubcModuleError::ConnectionTokenExhausted);
};
let close_token = CancellationToken::new();
let handle = ModuleHandle::new(
&ack,
tx.clone(),
connection_token,
close_token,
connection_runtime,
);
handle.shared.lock_inner().current_provides = Some(provides);
let serve_future = connection_serve_future_with_options(
read_half,
tx,
handler,
handle.clone(),
writer,
options,
writer_end,
);
Ok(Some((handle, serve_future)))
}
#[cfg(any(test, feature = "test-support"))]
fn connection_serve_future<R, H>(
read_half: R,
tx: mpsc::Sender<Frame>,
handler: Arc<H>,
handle: ModuleHandle,
writer: tokio::task::JoinHandle<Result<(), WriterFailure>>,
) -> ModuleServeFuture
where
R: AsyncRead + Unpin + Send + 'static,
H: ModuleHandler,
{
connection_serve_future_with_options(
read_half,
tx,
handler,
handle,
writer,
ServeOptions::default(),
None,
)
}
fn connection_serve_future_with_options<R, H>(
mut read_half: R,
tx: mpsc::Sender<Frame>,
handler: Arc<H>,
handle: ModuleHandle,
mut writer: tokio::task::JoinHandle<Result<(), WriterFailure>>,
options: ServeOptions,
writer_end: Option<Result<ConnectionEnd, SubcModuleError>>,
) -> ModuleServeFuture
where
R: AsyncRead + Unpin + Send + 'static,
H: ModuleHandler,
{
let serve_handle = handle.clone();
handle.install_dispatch_pools(handler.request_pools());
Box::pin(async move {
let dispatcher = RequestDispatcher::with_pools(serve_handle.dispatch_pools());
let mut writer_finished = writer_end.is_some();
let mut loop_result = if let Some(end) = writer_end {
end
} else {
tokio::select! {
biased;
result = module_loop(&mut read_half, &tx, Arc::clone(&handler), serve_handle.clone(), &dispatcher, &options.stop_handle) => result,
result = &mut writer => {
writer_finished = true;
writer_connection_end(result)
}
}
};
if matches!(loop_result, Err(SubcModuleError::WriterClosed)) && !writer_finished {
writer_finished = true;
loop_result = writer_connection_end((&mut writer).await);
}
let deadline = Instant::now() + options.writer_drain_timeout;
dispatcher.wait_for_draining_hooks_to_start().await;
dispatcher.tasks.close();
if matches!(loop_result, Ok(ConnectionEnd::Closed))
&& options.stop_handle.token.is_cancelled()
{
let _ = timeout_at(deadline, dispatcher.tasks.wait()).await;
}
let orderly = loop_result.is_ok();
if let Some(error) = dispatcher.take_failure() {
if !orderly || !matches!(error, SubcModuleError::WriterClosed) {
loop_result = Err(error);
}
}
dispatcher.cancel_all();
let _ = timeout_at(deadline, dispatcher.tasks.wait()).await;
let abandoned_replies = dispatcher
.in_flight
.lock()
.unwrap_or_else(|e| e.into_inner())
.len();
if let Some(error) = dispatcher.take_failure() {
if !orderly || !matches!(error, SubcModuleError::WriterClosed) {
loop_result = Err(error);
}
}
dispatcher.abort_all();
dispatcher.tasks.wait().await;
let daemon_eof = matches!(loop_result, Ok(ConnectionEnd::Eof));
let module_ended =
!writer_finished && matches!(loop_result, Ok(ConnectionEnd::Closed) | Err(_));
let deadline = Instant::now() + options.writer_drain_timeout;
if matches!(loop_result, Ok(ConnectionEnd::Closed))
&& options.stop_handle.token.is_cancelled()
{
match timeout_at(deadline, send_goodbye(&tx)).await {
Ok(Ok(())) | Err(_) => {}
Ok(Err(SubcModuleError::WriterClosed)) => {}
Ok(Err(error)) => loop_result = Err(error),
}
}
serve_handle.close_connection();
drop(tx);
let loop_result = match loop_result {
Ok(end) => {
handler.on_connection_end(end).await;
Ok(())
}
Err(error) => Err(error),
};
let (writer_result, undelivered_replies) = if writer_finished {
(Ok(()), 0)
} else if module_ended {
finish_writer_and_drain(writer, &mut read_half, deadline).await
} else {
finish_writer(writer, deadline).await
};
let undelivered_replies = undelivered_replies as u64
+ dispatcher.undelivered_replies.load(Ordering::Relaxed)
+ abandoned_replies as u64;
if daemon_eof && undelivered_replies > 0 {
tracing::warn!(
undelivered_replies,
"daemon EOF: replies could not be delivered during drain"
);
}
match (loop_result, writer_result) {
(Err(SubcModuleError::WriterClosed), Err(writer_error)) => Err(writer_error),
(Err(loop_err), _) => Err(loop_err),
(Ok(()), Ok(())) => Ok(()),
(Ok(()), Err(SubcModuleError::FrameIo(FrameIoError::Io(err))))
if is_disconnect(&err) =>
{
Ok(())
}
(Ok(()), Err(error)) => Err(error),
}
})
}
async fn module_loop<R, H>(
mut reader: R,
egress: &mpsc::Sender<Frame>,
handler: Arc<H>,
module_handle: ModuleHandle,
dispatcher: &RequestDispatcher,
stop: &ModuleStopHandle,
) -> Result<ConnectionEnd, SubcModuleError>
where
R: AsyncRead + Unpin,
H: ModuleHandler,
{
serve_frames(
&mut reader,
egress,
&handler,
&module_handle,
dispatcher,
stop,
)
.await
}
async fn serve_frames<R, H>(
reader: &mut R,
egress: &mpsc::Sender<Frame>,
handler: &Arc<H>,
module_handle: &ModuleHandle,
dispatcher: &RequestDispatcher,
stop: &ModuleStopHandle,
) -> Result<ConnectionEnd, SubcModuleError>
where
R: AsyncRead + Unpin,
H: ModuleHandler,
{
loop {
let read = tokio::select! {
biased;
() = dispatcher.failed.cancelled() => return Err(dispatcher.take_failure().expect("failure signalled")),
() = stop.token.cancelled() => return Ok(ConnectionEnd::Closed),
() = module_handle.shared.close_token.cancelled() => return Ok(ConnectionEnd::Closed),
read = read_frame(reader) => read,
};
let frame = match read {
Ok(Some(frame)) => frame,
Ok(None) => return Ok(ConnectionEnd::Eof),
Err(FrameIoError::Io(error)) if is_disconnect(&error) => {
return Ok(ConnectionEnd::Reset);
}
Err(err) => return Err(SubcModuleError::FrameIo(err)),
};
if !handle_frame(
frame,
egress,
Arc::clone(handler),
dispatcher.clone(),
module_handle.clone(),
)
.await?
{
return Ok(ConnectionEnd::Goodbye);
}
}
}
async fn handle_frame<H>(
frame: Frame,
egress: &mpsc::Sender<Frame>,
handler: Arc<H>,
dispatcher: RequestDispatcher,
module_handle: ModuleHandle,
) -> Result<bool, SubcModuleError>
where
H: ModuleHandler,
{
if frame.header.channel != 0
&& !module_handle.validate_ingress(frame.header.channel, frame.header.epoch)?
{
return Ok(true);
}
match frame.header.ty {
FrameType::Ping if frame.header.channel == 0 => {
let pong = Frame::build_with_version(
frame.header.ver,
FrameType::Pong,
frame.header.flags,
0,
0,
frame.header.corr,
Vec::new(),
)
.map_err(SubcModuleError::FrameBuild)?;
send_outbound(egress, pong).await?;
Ok(true)
}
FrameType::Goodbye if frame.header.channel == 0 => Ok(false),
FrameType::Goodbye => {
let handle = module_handle.route_handle(frame.header.channel, frame.header.epoch);
if module_handle.remove_route(handle)? {
cancel_handle(&dispatcher.in_flight, handle)?;
handler.on_route_gone(&handle).await;
}
Ok(true)
}
FrameType::Response if frame.header.channel == 0 => {
let _ = module_handle.handle_control_reply(frame);
Ok(true)
}
FrameType::Error if frame.header.channel == 0 => {
let _ = module_handle.handle_control_reply(frame);
Ok(true)
}
FrameType::Cancel => {
handle_cancel(frame, &dispatcher.in_flight)?;
Ok(true)
}
FrameType::Request if frame.header.channel == 0 => {
handle_control_request(frame, egress, handler, dispatcher, module_handle.clone())
.await?;
Ok(true)
}
FrameType::Request => {
spawn_data_request(frame, egress.clone(), handler, dispatcher, module_handle)?;
Ok(true)
}
FrameType::Push if frame.header.channel == 0 => {
handle_control_push(&frame, handler, &dispatcher);
Ok(true)
}
_ => Ok(true),
}
}
fn handle_control_push<H>(frame: &Frame, handler: Arc<H>, dispatcher: &RequestDispatcher)
where
H: ModuleHandler,
{
match serde_json::from_slice::<ModuleControlCommand>(&frame.body) {
Ok(ModuleControlCommand::Draining {
reason,
deadline_ms,
}) => {
let started = spawn_draining_hook(
handler,
sdk_route_close_reason(reason),
drain_deadline(deadline_ms),
dispatcher,
);
dispatcher
.draining_hooks
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.push(started);
}
Err(error) => {
if !dispatcher
.undecodable_push_logged
.swap(true, Ordering::Relaxed)
{
let op = serde_json::from_slice::<serde_json::Value>(&frame.body)
.ok()
.and_then(|body| body.get("op")?.as_str().map(str::to_string));
tracing::warn!(
op = op.as_deref().unwrap_or("<none>"),
error = %error,
"ignoring a channel-0 push from subc that this SDK cannot decode; \
further undecodable pushes on this connection are ignored without logging"
);
}
}
}
}
fn spawn_draining_hook<H>(
handler: Arc<H>,
reason: RouteCloseReason,
deadline: SystemTime,
dispatcher: &RequestDispatcher,
) -> oneshot::Receiver<()>
where
H: ModuleHandler,
{
let (started_tx, started_rx) = oneshot::channel();
dispatcher.spawn(async move {
let mut hook = handler.on_draining(reason, deadline);
let mut started_tx = Some(started_tx);
std::future::poll_fn(|cx| {
let poll = hook.as_mut().poll(cx);
if let Some(started) = started_tx.take() {
let _ = started.send(());
}
poll
})
.await;
});
started_rx
}
fn sdk_route_close_reason(reason: subc_protocol::RouteCloseReason) -> RouteCloseReason {
match serde_json::to_value(reason) {
Ok(serde_json::Value::String(wire)) => RouteCloseReason::from_wire(&wire),
_ => RouteCloseReason::Unknown(format!("{reason:?}")),
}
}
fn drain_deadline(deadline_ms: u64) -> SystemTime {
UNIX_EPOCH
.checked_add(Duration::from_millis(deadline_ms))
.unwrap_or_else(SystemTime::now)
}
fn spawn_data_request<H>(
frame: Frame,
egress: mpsc::Sender<Frame>,
handler: Arc<H>,
dispatcher: RequestDispatcher,
module_handle: ModuleHandle,
) -> Result<(), SubcModuleError>
where
H: ModuleHandler,
{
let handle = module_handle.route_handle(frame.header.channel, frame.header.epoch);
let corr = frame.header.corr;
let cancellation = CancellationToken::new();
{
let mut guard = lock_in_flight(&dispatcher.in_flight)?;
guard.insert((handle.channel, handle.epoch, corr), cancellation.clone());
}
let ctx = RequestCtx {
handle,
corr,
ver: frame.header.ver,
egress,
module_handle,
cancelled: cancellation,
binary_reply: Arc::new(AtomicBool::new(false)),
request_flags: frame.header.flags,
request_body: Arc::from(frame.body.as_slice()),
};
let body = frame.body;
let in_flight = Arc::clone(&dispatcher.in_flight);
let (pool, op) = dispatcher.pools.classify(&body);
let pool = pool.clone();
let wait = pool.start_waiting((handle.channel, handle.epoch, corr));
dispatcher.clone().spawn(async move {
let waiting = WaitingRequest {
op,
channel: handle.channel,
corr,
};
let permit = match wait_for_permit(&pool, wait, waiting, &ctx.cancelled).await {
PermitWaitOutcome::Acquired(permit) => Some(permit),
PermitWaitOutcome::Cancelled => None,
PermitWaitOutcome::Closed => {
dispatcher
.undelivered_replies
.fetch_add(1, Ordering::Relaxed);
if let Ok(mut guard) = in_flight.lock() {
guard.remove(&(handle.channel, handle.epoch, corr));
}
return;
}
};
let outcome = if permit.is_none() || ctx.cancelled.is_cancelled() {
Ok(HandlerOutcome::Error {
code: "cancelled".to_string(),
message: "request cancelled".to_string(),
})
} else {
catch_handler_panic(async { handler.handle(ctx.clone(), body).await }).await
};
let panicked = outcome.is_err();
let outcome = outcome.unwrap_or_else(|()| HandlerOutcome::Error {
code: "internal".to_string(),
message: "module handler panicked".to_string(),
});
match send_handler_outcome(&ctx, outcome).await {
Err(SubcModuleError::StaleRouteHandle(_)) if !panicked => {}
Err(error) => {
if matches!(error, SubcModuleError::WriterClosed) {
dispatcher
.undelivered_replies
.fetch_add(1, Ordering::Relaxed);
}
dispatcher.fail(error);
}
Ok(()) if panicked => dispatcher.fail(SubcModuleError::HandlerPanic),
Ok(()) => {}
}
if let Ok(mut guard) = in_flight.lock() {
guard.remove(&(handle.channel, handle.epoch, corr));
}
});
Ok(())
}
fn spawn_health_request<H>(
frame: Frame,
egress: mpsc::Sender<Frame>,
handler: Arc<H>,
dispatcher: RequestDispatcher,
) -> Result<(), SubcModuleError>
where
H: ModuleHandler,
{
let channel = frame.header.channel;
let epoch = frame.header.epoch;
let corr = frame.header.corr;
let ver = frame.header.ver;
let cancellation = CancellationToken::new();
{
let mut guard = lock_in_flight(&dispatcher.in_flight)?;
guard.insert((channel, epoch, corr), cancellation.clone());
}
let in_flight = Arc::clone(&dispatcher.in_flight);
dispatcher.clone().spawn(async move {
if !cancellation.is_cancelled() {
let result = async {
let report = catch_handler_panic(async { handler.health().await }).await;
let panicked = report.is_err();
let (ty, body) = match report {
Ok(report) => (
FrameType::Response,
serde_json::to_vec(&ModuleControlResponse::from(
dispatcher.fold_health(report),
))?,
),
Err(()) => (
FrameType::Error,
serde_json::to_vec(&ErrorBody::new("internal", "module handler panicked"))?,
),
};
let (ty, flags, body) = bounded_terminal(ty, control_flags(), body)?;
let frame = Frame::build_with_version(ver, ty, flags, channel, epoch, corr, body)
.map_err(SubcModuleError::FrameBuild)?;
send_outbound(&egress, frame).await?;
if panicked {
return Err(SubcModuleError::HandlerPanic);
}
Ok(())
}
.await;
if let Err(error) = result {
if matches!(error, SubcModuleError::WriterClosed) {
dispatcher
.undelivered_replies
.fetch_add(1, Ordering::Relaxed);
}
dispatcher.fail(error);
}
}
if let Ok(mut guard) = in_flight.lock() {
guard.remove(&(channel, epoch, corr));
}
});
Ok(())
}
async fn send_handler_outcome(
ctx: &RequestCtx,
outcome: HandlerOutcome,
) -> Result<(), SubcModuleError> {
match outcome {
HandlerOutcome::Response(body) => {
let flags = Flags::new(
ctx.binary_reply.load(Ordering::Relaxed),
Priority::Interactive,
false,
);
ctx.send_frame(FrameType::Response, flags, body).await
}
HandlerOutcome::Error { code, message } => {
let body = serde_json::to_vec(&ErrorBody::new(code, message))
.map_err(SubcModuleError::Json)?;
ctx.send_frame(FrameType::Error, data_flags(), body).await
}
HandlerOutcome::ErrorWithDetail {
code,
message,
detail,
} => {
let body = serde_json::to_vec(&ErrorBody::new(code, message).with_detail(detail))
.map_err(SubcModuleError::Json)?;
ctx.send_frame(FrameType::Error, data_flags(), body).await
}
HandlerOutcome::Streamed => {
ctx.send_frame(FrameType::StreamEnd, data_flags(), Vec::new())
.await
}
}
}
async fn catch_handler_panic<F: Future>(future: F) -> Result<F::Output, ()> {
let mut future = std::pin::pin!(future);
std::future::poll_fn(
|cx| match catch_unwind(AssertUnwindSafe(|| future.as_mut().poll(cx))) {
Ok(poll) => poll.map(Ok),
Err(_) => std::task::Poll::Ready(Err(())),
},
)
.await
}
fn bounded_terminal(
ty: FrameType,
flags: Flags,
body: Vec<u8>,
) -> Result<(FrameType, Flags, Vec<u8>), SubcModuleError> {
let size = subc_protocol::HEADER_LEN + body.len();
let limit = subc_protocol::MAX_DELIVERED_FRAME_LEN;
if matches!(ty, FrameType::Response | FrameType::Error) && size > limit {
let body = serde_json::to_vec(
&ErrorBody::new("frame_too_large", "module reply exceeds frame limit")
.with_detail(serde_json::json!({ "size": size, "limit": limit })),
)?;
return Ok((FrameType::Error, flags, body));
}
Ok((ty, flags, body))
}
fn handle_cancel(frame: Frame, in_flight: &InFlight) -> Result<(), SubcModuleError> {
let cancellation = {
let guard = lock_in_flight(in_flight)?;
guard
.get(&(frame.header.channel, frame.header.epoch, frame.header.corr))
.cloned()
};
if let Some(cancellation) = cancellation {
cancellation.cancel();
}
Ok(())
}
fn cancel_handle(in_flight: &InFlight, handle: RouteHandle) -> Result<(), SubcModuleError> {
let cancelled = {
let mut guard = lock_in_flight(in_flight)?;
let keys = guard
.keys()
.copied()
.filter(|(channel, epoch, _)| *channel == handle.channel && *epoch == handle.epoch)
.collect::<Vec<_>>();
keys.into_iter()
.filter_map(|key| guard.remove(&key))
.collect::<Vec<_>>()
};
for cancellation in cancelled {
cancellation.cancel();
}
Ok(())
}
async fn handle_control_request<H>(
frame: Frame,
egress: &mpsc::Sender<Frame>,
handler: Arc<H>,
dispatcher: RequestDispatcher,
module_handle: ModuleHandle,
) -> Result<(), SubcModuleError>
where
H: ModuleHandler,
{
let request = match serde_json::from_slice::<ModuleControlRequest>(&frame.body) {
Ok(request) => request,
Err(error) => {
let body = serde_json::to_vec(&ErrorBody::new(
"invalid_request",
format!("control request could not be decoded by this module: {error}"),
))
.map_err(SubcModuleError::Json)?;
let refusal = Frame::build_with_version(
frame.header.ver,
FrameType::Error,
control_flags(),
0,
0,
frame.header.corr,
body,
)
.map_err(SubcModuleError::FrameBuild)?;
send_outbound(egress, refusal).await?;
return Ok(());
}
};
match request {
ModuleControlRequest::RouteBind {
route_channel,
epoch,
target,
identity,
principal,
consumer_capabilities,
role_versions,
admission_facts,
scope,
call_mode,
} => {
if let Some(stale) = module_handle.installed_route(route_channel)? {
if epoch <= stale.epoch {
let body = serde_json::to_vec(&ErrorBody::new(
"route_rejected",
format!(
"route.bind epoch {epoch} does not supersede installed epoch {} on channel {route_channel}",
stale.epoch
),
))
.map_err(SubcModuleError::Json)?;
let reject = Frame::build_with_version(
frame.header.ver,
FrameType::Error,
control_flags(),
0,
0,
frame.header.corr,
body,
)
.map_err(SubcModuleError::FrameBuild)?;
send_outbound(egress, reject).await?;
return Ok(());
}
if module_handle.remove_route(stale)? {
cancel_handle(&dispatcher.in_flight, stale)?;
handler.on_route_gone(&stale).await;
}
}
let handle = module_handle.route_handle(route_channel, epoch);
let req = RouteBindRequest {
handle,
target,
identity,
principal,
consumer_capabilities,
role_versions,
admission_facts,
call_mode: call_mode.unwrap_or_else(|| derive_call_mode(scope.as_ref())),
scope,
};
let decision = handler.on_bind(&req).await;
match decision.kind {
BindDecisionKind::Accept => {
let response = match serde_json::to_vec(&ModuleControlResponse::RouteBindAck {})
.map_err(SubcModuleError::Json)
.and_then(|body| {
Frame::build_with_version(
frame.header.ver,
FrameType::Response,
control_flags(),
0,
0,
frame.header.corr,
body,
)
.map_err(SubcModuleError::FrameBuild)
}) {
Ok(response) => response,
Err(err) => {
handler.on_route_gone(&handle).await;
return Err(err);
}
};
if let Err(err) = send_outbound(egress, response).await {
handler.on_route_gone(&handle).await;
return Err(err);
}
if let Err(err) = module_handle.install_route(handle) {
handler.on_route_gone(&handle).await;
return Err(err);
}
handler.on_bound(&handle).await;
}
BindDecisionKind::Reject { code, message } => {
let result = serde_json::to_vec(&ErrorBody::new(code, message))
.map_err(SubcModuleError::Json)
.and_then(|body| {
Frame::build_with_version(
frame.header.ver,
FrameType::Error,
control_flags(),
0,
0,
frame.header.corr,
body,
)
.map_err(SubcModuleError::FrameBuild)
});
let result = match result {
Ok(response) => send_outbound(egress, response).await,
Err(err) => Err(err),
};
handler.on_route_gone(&handle).await;
result?;
}
}
}
ModuleControlRequest::HealthCheck {} => {
spawn_health_request(frame, egress.clone(), handler, dispatcher)?;
}
}
Ok(())
}
async fn send_hello(
egress: &mpsc::Sender<Frame>,
mut manifest: ModuleManifest,
) -> Result<(), SubcModuleError> {
let launch_nonce = match retained_launch_nonce() {
Some(retained) => Some(retained),
None => launch_nonce()
.map_err(SubcModuleError::LaunchNonce)?
.map(|nonce| nonce.value().to_string()),
};
stamp_launch_nonce_source(&mut manifest, launch_nonce.as_deref());
let body = serde_json::to_vec(&ModuleHelloBody {
manifest,
protocol_ver: PROTOCOL_VERSION,
control_ops: Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
launch_nonce,
})
.map_err(SubcModuleError::Json)?;
let frame = Frame::build(FrameType::Hello, control_flags(), 0, 0, HELLO_CORR, body)
.map_err(SubcModuleError::FrameBuild)?;
send_outbound(egress, frame).await
}
async fn expect_hello_ack<R>(reader: &mut R) -> Result<ModuleHelloAckBody, SubcModuleError>
where
R: AsyncRead + Unpin,
{
let Some(frame) = read_frame(reader).await.map_err(SubcModuleError::FrameIo)? else {
return Err(SubcModuleError::ConnectionClosedBeforeHelloAck);
};
match frame.header.ty {
FrameType::HelloAck => serde_json::from_slice(&frame.body).map_err(SubcModuleError::Json),
FrameType::Error => {
let body =
serde_json::from_slice::<ErrorBody>(&frame.body).map_err(SubcModuleError::Json)?;
Err(SubcModuleError::HelloRejected { body })
}
ty => Err(SubcModuleError::UnexpectedHelloAck { ty }),
}
}
async fn connect_to_subc(connection_file_path: &Path) -> Result<TcpStream, SubcModuleError> {
let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
SubcModuleError::ConnectionFile {
path: connection_file_path.to_path_buf(),
source,
}
})?;
let endpoint = conn
.endpoints
.first()
.ok_or_else(|| SubcModuleError::NoEndpoint {
path: connection_file_path.to_path_buf(),
})?;
let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
let mut stream = connect_endpoint(&endpoint.host, endpoint.port)
.await
.map_err(|source| SubcModuleError::Connect {
path: connection_file_path.to_path_buf(),
endpoint: endpoint_label.clone(),
source,
})?;
let _ = stream.set_nodelay(true);
authenticate_client(&mut stream, &conn, AUTH_DEADLINE)
.await
.map_err(|source| SubcModuleError::Auth {
path: connection_file_path.to_path_buf(),
endpoint: endpoint_label,
source,
})?;
Ok(stream)
}
async fn connect_endpoint(host: &str, port: u16) -> io::Result<TcpStream> {
let host = host
.strip_prefix('[')
.and_then(|h| h.strip_suffix(']'))
.unwrap_or(host);
TcpStream::connect((host, port)).await
}
async fn send_goodbye(tx: &mpsc::Sender<Frame>) -> Result<(), SubcModuleError> {
let frame = Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 0, Vec::new())
.map_err(SubcModuleError::FrameBuild)?;
send_outbound(tx, frame).await
}
fn is_disconnect(error: &io::Error) -> bool {
matches!(
error.kind(),
io::ErrorKind::ConnectionReset
| io::ErrorKind::ConnectionAborted
| io::ErrorKind::BrokenPipe
)
}
fn writer_ended(
result: Result<Result<(), WriterFailure>, tokio::task::JoinError>,
) -> SubcModuleError {
match result {
Ok(Ok(())) => SubcModuleError::WriterClosed,
Ok(Err(error)) => SubcModuleError::FrameIo(error.error),
Err(error) => SubcModuleError::WriterTask(error),
}
}
fn writer_connection_end(
result: Result<Result<(), WriterFailure>, tokio::task::JoinError>,
) -> Result<ConnectionEnd, SubcModuleError> {
match writer_ended(result) {
SubcModuleError::FrameIo(FrameIoError::Io(error)) if is_disconnect(&error) => {
Ok(ConnectionEnd::Reset)
}
error => Err(error),
}
}
async fn finish_writer(
mut writer: tokio::task::JoinHandle<Result<(), WriterFailure>>,
deadline: Instant,
) -> (Result<(), SubcModuleError>, usize) {
match timeout_at(deadline, &mut writer).await {
Ok(Ok(Ok(()))) => (Ok(()), 0),
Ok(Ok(Err(failure))) => (
Err(SubcModuleError::FrameIo(failure.error)),
failure.undelivered_replies,
),
Ok(Err(error)) => (Err(SubcModuleError::WriterTask(error)), 0),
Err(_) => {
writer.abort();
let _ = writer.await;
(Ok(()), 0)
}
}
}
struct WriterFailure {
error: FrameIoError,
undelivered_replies: usize,
}
async fn finish_writer_and_drain<R>(
writer: tokio::task::JoinHandle<Result<(), WriterFailure>>,
reader: &mut R,
deadline: Instant,
) -> (Result<(), SubcModuleError>, usize)
where
R: AsyncRead + Unpin,
{
let flushed = finish_writer(writer, deadline).await;
if flushed.0.is_ok() {
drain_until_peer_closes(reader, deadline).await;
}
flushed
}
async fn drain_until_peer_closes<R>(reader: &mut R, deadline: Instant)
where
R: AsyncRead + Unpin,
{
if Instant::now() >= deadline {
return;
}
let mut discard = [0u8; 4096];
let _ = timeout_at(deadline, async {
while let Ok(read) = reader.read(&mut discard).await {
if read == 0 {
break;
}
}
})
.await;
}
fn is_reply(frame: &Frame) -> bool {
matches!(
frame.header.ty,
FrameType::Response | FrameType::Error | FrameType::StreamEnd
)
}
async fn drain_writer<W>(write_half: W, mut rx: mpsc::Receiver<Frame>) -> Result<(), WriterFailure>
where
W: AsyncWrite + Unpin,
{
let mut writer = BufWriter::new(write_half);
let mut pending_replies = 0;
let result = async {
while let Some(frame) = rx.recv().await {
pending_replies += usize::from(is_reply(&frame));
write_frame(&mut writer, &frame).await?;
while let Ok(frame) = rx.try_recv() {
pending_replies += usize::from(is_reply(&frame));
write_frame(&mut writer, &frame).await?;
}
writer.flush().await.map_err(FrameIoError::Io)?;
pending_replies = 0;
}
writer.flush().await.map_err(FrameIoError::Io)?;
let _ = writer.shutdown().await;
Ok(())
}
.await;
result.map_err(|error| {
rx.close();
while let Ok(frame) = rx.try_recv() {
pending_replies += usize::from(is_reply(&frame));
}
WriterFailure {
error,
undelivered_replies: pending_replies,
}
})
}
async fn send_outbound(egress: &mpsc::Sender<Frame>, frame: Frame) -> Result<(), SubcModuleError> {
egress
.send(frame)
.await
.map_err(|_| SubcModuleError::WriterClosed)
}
fn parse_subc_arg(args: impl IntoIterator<Item = OsString>) -> Result<PathBuf, SubcModuleError> {
let mut args = args.into_iter();
while let Some(arg) = args.next() {
if arg == "--subc" {
let value = args.next().ok_or(SubcModuleError::MissingSubcValue)?;
return Ok(PathBuf::from(value));
}
if let Some(raw) = arg.to_str().and_then(|arg| arg.strip_prefix("--subc=")) {
if raw.is_empty() {
return Err(SubcModuleError::MissingSubcValue);
}
return Ok(PathBuf::from(raw));
}
}
Err(SubcModuleError::MissingSubcArg)
}
fn module_id_from_env() -> Result<Option<String>, SubcModuleError> {
match env::var(SUBC_MODULE_ID_ENV) {
Ok(value) if !value.trim().is_empty() => Ok(Some(value)),
Ok(_) => Err(SubcModuleError::EmptyModuleIdEnv),
Err(env::VarError::NotPresent) => Ok(None),
Err(env::VarError::NotUnicode(value)) => {
Err(SubcModuleError::NonUnicodeModuleIdEnv { value })
}
}
}
fn lock_in_flight(
in_flight: &InFlight,
) -> Result<std::sync::MutexGuard<'_, HashMap<RequestKey, CancellationToken>>, SubcModuleError> {
in_flight
.lock()
.map_err(|_| SubcModuleError::InFlightPoisoned)
}
fn control_flags() -> Flags {
Flags::new(false, Priority::Passive, false)
}
fn data_flags() -> Flags {
Flags::new(false, Priority::Interactive, false)
}
#[derive(Debug)]
#[non_exhaustive]
pub enum SubcModuleError {
MissingSubcArg,
MissingSubcValue,
EmptyModuleIdEnv,
NonUnicodeModuleIdEnv {
value: OsString,
},
ConnectionFile {
path: PathBuf,
source: ConnectionFileError,
},
NoEndpoint {
path: PathBuf,
},
Connect {
path: PathBuf,
endpoint: String,
source: io::Error,
},
Auth {
path: PathBuf,
endpoint: String,
source: AuthError,
},
FrameIo(FrameIoError),
FrameBuild(FrameBuildError),
Json(serde_json::Error),
WriterClosed,
HandlerPanic,
StaleRouteHandle(RouteHandle),
ConnectionTokenExhausted,
WriterTask(tokio::task::JoinError),
InFlightPoisoned,
ConnectionClosedBeforeHelloAck,
UnexpectedHelloAck {
ty: FrameType,
},
HelloRejected {
body: ErrorBody,
},
LaunchNonce(launch_nonce::LaunchNonceError),
}
impl fmt::Display for SubcModuleError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::MissingSubcArg => write!(f, "missing required --subc <connection-file> argument"),
Self::MissingSubcValue => write!(f, "--subc requires a connection-file path value"),
Self::EmptyModuleIdEnv => write!(f, "{SUBC_MODULE_ID_ENV} must not be empty when set"),
Self::NonUnicodeModuleIdEnv { value } => write!(
f,
"{SUBC_MODULE_ID_ENV} must be valid UTF-8, got '{}'",
value.to_string_lossy()
),
Self::ConnectionFile { path, source } => write!(
f,
"failed to read subc connection file '{}': {source}",
path.display()
),
Self::NoEndpoint { path } => write!(
f,
"subc connection file '{}' has no endpoints",
path.display()
),
Self::Connect {
path,
endpoint,
source,
} => write!(
f,
"failed to connect to subc endpoint {endpoint} from '{}': {source}",
path.display()
),
Self::Auth {
path,
endpoint,
source,
} => write!(
f,
"failed to authenticate to subc endpoint {endpoint} from '{}': {source}",
path.display()
),
Self::FrameIo(err) => fmt::Display::fmt(err, f),
Self::FrameBuild(err) => write!(f, "frame build error: {err}"),
Self::Json(err) => write!(f, "JSON error: {err}"),
Self::WriterClosed => write!(f, "module writer task closed"),
Self::HandlerPanic => write!(f, "module handler panicked"),
Self::StaleRouteHandle(handle) => write!(f, "stale route handle: {handle:?}"),
Self::ConnectionTokenExhausted => write!(f, "module connection token exhausted"),
Self::WriterTask(err) => write!(f, "module writer task failed: {err}"),
Self::InFlightPoisoned => write!(f, "in-flight registry lock poisoned"),
Self::ConnectionClosedBeforeHelloAck => write!(f, "connection closed before HELLO_ACK"),
Self::UnexpectedHelloAck { ty } => write!(f, "expected HELLO_ACK, got {ty:?}"),
Self::HelloRejected { body } => write!(
f,
"HELLO rejected by subc: {} ({})",
body.code, body.message
),
Self::LaunchNonce(err) => write!(f, "launch nonce unavailable: {err}"),
}
}
}
impl Error for SubcModuleError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::ConnectionFile { source, .. } => Some(source),
Self::Connect { source, .. } => Some(source),
Self::Auth { source, .. } => Some(source),
Self::FrameIo(err) => Some(err),
Self::FrameBuild(err) => Some(err),
Self::Json(err) => Some(err),
Self::LaunchNonce(err) => Some(err),
Self::WriterTask(err) => Some(err),
Self::MissingSubcArg
| Self::MissingSubcValue
| Self::EmptyModuleIdEnv
| Self::NonUnicodeModuleIdEnv { .. }
| Self::NoEndpoint { .. }
| Self::WriterClosed
| Self::HandlerPanic
| Self::StaleRouteHandle(_)
| Self::ConnectionTokenExhausted
| Self::InFlightPoisoned
| Self::ConnectionClosedBeforeHelloAck
| Self::UnexpectedHelloAck { .. }
| Self::HelloRejected { .. } => None,
}
}
}
impl From<serde_json::Error> for SubcModuleError {
fn from(err: serde_json::Error) -> Self {
Self::Json(err)
}
}
#[cfg(test)]
mod call_mode_tests;
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicUsize, Ordering};
use serde_json::json;
use subc_protocol::manifest::{Concurrency, ExecutionMode, IdentityScope, Tool};
use tokio::sync::Semaphore;
use tokio::time::{advance, timeout};
use super::*;
struct EchoHandler;
#[async_trait]
impl ModuleHandler for EchoHandler {
async fn handle(&self, _ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome {
HandlerOutcome::Response(body)
}
}
struct BlockingHandler {
entered: Arc<AtomicUsize>,
release: Semaphore,
report: HealthReport,
}
#[async_trait]
impl ModuleHandler for BlockingHandler {
async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
self.entered.fetch_add(1, Ordering::SeqCst);
self.release.acquire().await.unwrap().forget();
HandlerOutcome::Streamed
}
async fn health(&self) -> HealthReport {
self.report.clone()
}
}
fn health_request(corr: u64) -> Frame {
Frame::build(
FrameType::Request,
control_flags(),
0,
0,
corr,
serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).unwrap(),
)
.unwrap()
}
fn data_request(channel: u16, corr: u64) -> Frame {
Frame::build(
FrameType::Request,
data_flags(),
channel,
1,
corr,
b"opaque".to_vec(),
)
.unwrap()
}
fn catalog_update_response(corr: u64) -> Frame {
Frame::build(
FrameType::Response,
control_flags(),
0,
0,
corr,
serde_json::to_vec(&ModuleControlResponseToModule::CatalogUpdate {}).unwrap(),
)
.unwrap()
}
fn test_module_handle(subc_ops: &[&str]) -> (ModuleHandle, mpsc::Receiver<Frame>) {
let (tx, rx) = mpsc::channel(4);
let ack = ModuleHelloAckBody {
negotiated_ver: PROTOCOL_VERSION,
subc_ops: subc_ops.iter().map(|op| (*op).to_string()).collect(),
subc_capabilities: Vec::new(),
storage: None,
machine_id: None,
};
(
ModuleHandle::new(
&ack,
tx,
1,
CancellationToken::new(),
tokio::runtime::Handle::current(),
),
rx,
)
}
#[test]
fn module_handle_exposes_the_acked_machine_id_and_none_otherwise() {
let runtime = tokio::runtime::Runtime::new().unwrap();
let handle_for = |machine_id: Option<&str>| {
let (tx, _rx) = mpsc::channel(1);
let ack = ModuleHelloAckBody {
negotiated_ver: PROTOCOL_VERSION,
subc_ops: Vec::new(),
subc_capabilities: Vec::new(),
storage: None,
machine_id: machine_id.map(str::to_owned),
};
ModuleHandle::new(
&ack,
tx,
1,
CancellationToken::new(),
runtime.handle().clone(),
)
};
assert_eq!(
handle_for(Some("0123456789abcdef0123456789abcdef"))
.machine_id()
.map(MachineId::as_str),
Some("0123456789abcdef0123456789abcdef")
);
assert_eq!(handle_for(None).machine_id(), None);
assert_eq!(handle_for(Some("not-a-machine-id")).machine_id(), None);
}
fn test_provider_role(tool_names: &[&str]) -> ProviderRole {
ProviderRole::ToolProvider {
tools: tool_names
.iter()
.map(|name| Tool {
name: (*name).to_string(),
description: None,
execution_mode: ExecutionMode::Pure,
schema: json!({"type": "object"}),
})
.collect(),
identity_scope: vec![IdentityScope::Project],
concurrency: Concurrency::ModuleManaged,
emits_push: false,
sub_supervises: false,
}
}
#[tokio::test]
async fn catalog_update_fails_fast_when_hello_ack_does_not_advertise_support() {
let (handle, mut rx) = test_module_handle(&[]);
let error = handle
.catalog_update(vec![test_provider_role(&["a"])])
.await
.unwrap_err();
assert!(matches!(error, CatalogUpdateError::NotSupported));
assert!(matches!(
handle.set_ready(true).await,
Err(CatalogUpdateError::NotSupported)
));
assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
}
#[tokio::test]
async fn catalog_update_demuxes_multiple_in_flight_requests() {
let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
let first_handle = handle.clone();
let second_handle = handle.clone();
let first = tokio::spawn(async move {
first_handle
.catalog_update(vec![test_provider_role(&["a"])])
.await
});
let second = tokio::spawn(async move {
second_handle
.catalog_update(vec![test_provider_role(&["b"])])
.await
});
let first_frame = timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
let second_frame = timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
assert_eq!(first_frame.header.ty, FrameType::Request);
assert_eq!(first_frame.header.channel, 0);
assert_eq!(second_frame.header.ty, FrameType::Request);
assert_eq!(second_frame.header.channel, 0);
assert_ne!(first_frame.header.corr, second_frame.header.corr);
let (earlier, later) =
if json_body(&first_frame.body)["provides"] == json!([test_provider_role(&["a"])]) {
(first, second)
} else {
(second, first)
};
assert!(handle.handle_control_reply(catalog_update_response(second_frame.header.corr)));
assert!(later.await.unwrap().is_ok());
assert!(handle.handle_control_reply(catalog_update_response(first_frame.header.corr)));
assert!(earlier.await.unwrap().is_ok());
let expected = json_body(&second_frame.body)["provides"].clone();
let (result, ()) = tokio::join!(handle.set_ready(true), async {
let frame = rx.recv().await.unwrap();
assert_eq!(json_body(&frame.body)["provides"], expected);
assert_eq!(json_body(&frame.body)["ready"], true);
handle.handle_control_reply(catalog_update_response(frame.header.corr));
});
result.unwrap();
}
fn json_body(body: &[u8]) -> serde_json::Value {
serde_json::from_slice(body).unwrap()
}
async fn ack_catalog_update(
handle: &ModuleHandle,
rx: &mut mpsc::Receiver<Frame>,
provides: &[ProviderRole],
ready: Option<bool>,
capabilities: Option<&CapabilityDeclarations>,
) {
let frame = rx.recv().await.unwrap();
let mut expected = json!({"op": "catalog.update", "provides": provides});
if let Some(ready) = ready {
expected["ready"] = json!(ready);
}
if let Some(capabilities) = capabilities {
expected["capabilities"] = json!(capabilities);
}
assert_eq!(json_body(&frame.body), expected);
let mut wire = format!(
r#"{{"op":"catalog.update","provides":{}"#,
serde_json::to_string(provides).unwrap()
);
if let Some(capabilities) = capabilities {
wire.push_str(&format!(
r#","capabilities":{}"#,
serde_json::to_string(capabilities).unwrap()
));
}
if let Some(ready) = ready {
wire.push_str(&format!(r#","ready":{ready}"#));
}
wire.push('}');
assert_eq!(frame.body, wire.as_bytes());
assert!(handle.handle_control_reply(catalog_update_response(frame.header.corr)));
}
#[tokio::test]
async fn catalog_update_set_ready_preserves_registered_and_replaced_roles() {
let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
let initial = vec![test_provider_role(&["initial", "second"])];
handle.shared.lock_inner().current_provides = Some(initial.clone());
let (result, ()) = tokio::join!(
handle.set_ready(true),
ack_catalog_update(&handle, &mut rx, &initial, Some(true), None)
);
result.unwrap();
let replaced = vec![test_provider_role(&["replacement"])];
let (result, ()) = tokio::join!(
handle.catalog_update(replaced.clone()),
ack_catalog_update(&handle, &mut rx, &replaced, None, None)
);
result.unwrap();
let (result, ()) = tokio::join!(
handle.set_ready(false),
ack_catalog_update(&handle, &mut rx, &replaced, Some(false), None)
);
result.unwrap();
let attested = vec![test_provider_role(&["attested"])];
let capabilities = CapabilityDeclarations {
provides: vec!["test.capability".into()],
requires: Vec::new(),
must_never_reach: Vec::new(),
};
let (result, ()) = tokio::join!(
handle.catalog_update_with_capabilities(attested.clone(), capabilities.clone()),
ack_catalog_update(&handle, &mut rx, &attested, None, Some(&capabilities))
);
result.unwrap();
let (result, ()) = tokio::join!(
handle.set_ready(true),
ack_catalog_update(&handle, &mut rx, &attested, Some(true), None)
);
result.unwrap();
}
#[tokio::test]
async fn catalog_update_set_ready_refuses_unknown_or_in_flight_roles() {
let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
assert!(matches!(
handle.set_ready(true).await,
Err(CatalogUpdateError::Protocol(_))
));
assert!(rx.try_recv().is_err());
let updating = handle.clone();
let update = tokio::spawn(async move {
updating
.catalog_update(vec![test_provider_role(&["new"])])
.await
});
let frame = rx.recv().await.unwrap();
assert!(matches!(
handle.set_ready(true).await,
Err(CatalogUpdateError::Protocol(_))
));
assert!(rx.try_recv().is_err());
update.abort();
assert!(update.await.unwrap_err().is_cancelled());
assert!(!handle.handle_control_reply(catalog_update_response(frame.header.corr)));
assert!(matches!(
handle.set_ready(true).await,
Err(CatalogUpdateError::Protocol(_))
));
let roles = vec![test_provider_role(&["recovered"])];
let (result, ()) = tokio::join!(
handle.catalog_update(roles.clone()),
ack_catalog_update(&handle, &mut rx, &roles, None, None)
);
result.unwrap();
let (result, ()) = tokio::join!(
handle.set_ready(true),
ack_catalog_update(&handle, &mut rx, &roles, Some(true), None)
);
result.unwrap();
}
#[tokio::test(start_paused = true)]
async fn catalog_update_set_ready_timeout_does_not_replay_stale_roles() {
let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
handle.shared.lock_inner().current_provides = Some(vec![test_provider_role(&["old"])]);
assert!(matches!(
handle
.catalog_update(vec![test_provider_role(&["new"])])
.await,
Err(CatalogUpdateError::Timeout)
));
let frame = rx.recv().await.unwrap();
assert!(!handle.handle_control_reply(catalog_update_response(frame.header.corr)));
assert!(matches!(
handle.set_ready(true).await,
Err(CatalogUpdateError::Protocol(_))
));
assert!(rx.try_recv().is_err());
}
#[tokio::test]
async fn catalog_update_set_ready_rejection_retains_last_accepted_roles() {
let (handle, mut rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
let initial = vec![test_provider_role(&["kept"])];
handle.shared.lock_inner().current_provides = Some(initial.clone());
let (result, ()) = tokio::join!(
handle.catalog_update(vec![test_provider_role(&["rejected"])]),
async {
let frame = rx.recv().await.unwrap();
handle.handle_control_reply(control_reply(
FrameType::Error,
frame.header.corr,
br#"{"code":"catalog_update_frozen_field","message":"frozen"}"#.to_vec(),
));
}
);
assert!(matches!(result, Err(CatalogUpdateError::FrozenField(_))));
let (result, ()) = tokio::join!(
handle.set_ready(true),
ack_catalog_update(&handle, &mut rx, &initial, Some(true), None)
);
result.unwrap();
}
fn control_reply(ty: FrameType, corr: u64, body: Vec<u8>) -> Frame {
Frame::build(ty, control_flags(), 0, 0, corr, body).unwrap()
}
#[tokio::test]
async fn scope_served_by_requires_capability_and_decodes_open_replies() {
let (unsupported, mut rx) = test_module_handle(&[subc_control::SCOPE_SERVED_BY_OP]);
assert_eq!(
unsupported
.scope_served_by("s".into(), 1, "tool".into())
.await,
Err(ScopeCallError::NotSupported {
op: subc_control::SCOPE_SERVED_BY_OP
})
);
assert!(rx.try_recv().is_err());
let (tx, mut rx) = mpsc::channel(4);
let ack = ModuleHelloAckBody {
negotiated_ver: PROTOCOL_VERSION,
subc_ops: vec![],
subc_capabilities: vec![subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.into()],
storage: None,
machine_id: None,
};
let handle = ModuleHandle::new(
&ack,
tx,
1,
CancellationToken::new(),
tokio::runtime::Handle::current(),
);
for answer in ["served", "not_served", "unknown", "future"] {
let caller = handle.clone();
let pending =
tokio::spawn(
async move { caller.scope_served_by("s".into(), 7, "tool".into()).await },
);
let request = rx.recv().await.unwrap();
let decoded: subc_control::ScopeServedByRequest =
serde_json::from_slice(&request.body).unwrap();
assert_eq!(
(
decoded.scope_ref.as_str(),
decoded.scope_epoch,
decoded.module_id.as_str()
),
("s", 7, "tool")
);
assert!(handle.handle_control_reply(control_reply(FrameType::Response, request.header.corr,
serde_json::to_vec(&serde_json::json!({"op":"scope.served_by", "answer":answer, "daemon_incarnation":"d"})).unwrap())));
let reply = pending.await.unwrap().unwrap();
assert_eq!(reply.daemon_incarnation, "d");
assert_eq!(serde_json::to_value(reply.answer).unwrap(), answer);
}
let caller = handle.clone();
let pending =
tokio::spawn(async move { caller.scope_served_by("s".into(), 7, "tool".into()).await });
let request = rx.recv().await.unwrap();
assert!(handle.handle_control_reply(control_reply(
FrameType::Error,
request.header.corr,
serde_json::to_vec(&serde_json::json!({"code":"not_registered", "message":"gone"}))
.unwrap()
)));
assert_eq!(
pending.await.unwrap().unwrap_err().code(),
Some("not_registered")
);
}
fn owner() -> Principal {
Principal::Reserved {
module_id: "owner".to_string(),
}
}
#[tokio::test]
async fn scope_ops_fail_fast_when_hello_ack_does_not_advertise_them() {
let (handle, mut rx) = test_module_handle(&[]);
assert_eq!(
handle.scope_sync(1, Vec::new()).await.unwrap_err(),
ScopeCallError::NotSupported { op: SCOPE_SYNC_OP }
);
assert_eq!(
handle
.scope_describe(owner(), "s".to_string())
.await
.unwrap_err(),
ScopeCallError::NotSupported {
op: SCOPE_DESCRIBE_OP
}
);
assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
let (describe_only, mut rx) = test_module_handle(&[SCOPE_DESCRIBE_OP]);
assert_eq!(
describe_only.scope_sync(1, Vec::new()).await.unwrap_err(),
ScopeCallError::NotSupported { op: SCOPE_SYNC_OP }
);
assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
let (sync_only, mut rx) = test_module_handle(&[SCOPE_SYNC_OP]);
assert_eq!(
sync_only
.scope_describe(owner(), "s".to_string())
.await
.unwrap_err(),
ScopeCallError::NotSupported {
op: SCOPE_DESCRIBE_OP
}
);
assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
}
#[tokio::test]
async fn scope_apply_sends_nothing_without_its_own_advertised_op() {
for ops in [&[][..], &[SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP][..]] {
let (handle, mut rx) = test_module_handle(ops);
assert_eq!(
handle.scope_apply(2, Vec::new(), Vec::new()).await,
Err(ScopeCallError::NotSupported { op: SCOPE_APPLY_OP })
);
assert!(timeout(Duration::from_millis(75), rx.recv()).await.is_err());
}
}
#[tokio::test]
async fn scope_apply_sends_the_delta_and_preserves_per_entry_outcomes() {
let (handle, mut rx) = test_module_handle(&[SCOPE_APPLY_OP]);
let apply = tokio::spawn({
let handle = handle.clone();
async move {
handle
.scope_apply(
2,
vec![ScopeRecord::new(
"run",
1,
subc_protocol::scope::ScopeKind::Ephemeral,
)
.with_expires_at_ms(Some(12345))
.with_workdir(Some("/session/folder".into()))
.with_project_id(Some("project:1".into()))],
vec![ScopeEnd::new("old", 3), ScopeEnd::new("missing", 1)],
)
.await
}
});
let request = timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
assert_eq!(request.header.ty, FrameType::Request);
assert_eq!(request.header.channel, 0);
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&request.body).unwrap(),
serde_json::json!({
"op": "scope.apply", "generation": 2,
"upsert": [{"ref": "run", "scope_epoch": 1, "kind": "ephemeral", "expires_at_ms": 12345,
"workdir": "/session/folder", "project_id": "project:1"}],
"end": [{"ref": "old", "scope_epoch": 3}, {"ref": "missing", "scope_epoch": 1}]
})
);
let body = serde_json::to_vec(&serde_json::json!({
"op": "scope.apply", "generation": 2,
"results": [{"ref": "run", "scope_epoch": 1, "outcome": "refused", "code": "scope_expired"}],
"end_results": [
{"ref": "old", "scope_epoch": 3, "outcome": "ended"},
{"ref": "missing", "scope_epoch": 1, "outcome": "not_live"}
],
"ended": [{"ref": "old", "scope_epoch": 3}]
}))
.unwrap();
assert!(handle.handle_control_reply(control_reply(
FrameType::Response,
request.header.corr,
body
)));
let reply = apply.await.unwrap().unwrap();
assert_eq!(reply.generation, 2);
assert_eq!(reply.results.len(), 1);
assert_eq!(reply.results[0].scope_ref, "run");
assert_eq!(
reply.results[0].outcome,
subc_protocol::scope::ScopeRecordOutcome::Refused
);
assert_eq!(reply.results[0].code.as_deref(), Some("scope_expired"));
assert_eq!(reply.end_results.len(), 2);
assert_eq!(reply.end_results[0].scope_ref, "old");
assert_eq!(reply.end_results[0].scope_epoch, 3);
assert_eq!(
reply.end_results[0].outcome,
subc_protocol::scope::ScopeEndOutcome::Ended
);
assert_eq!(reply.end_results[1].scope_ref, "missing");
assert_eq!(
reply.end_results[1].outcome,
subc_protocol::scope::ScopeEndOutcome::NotLive
);
assert_eq!(reply.ended.len(), 1);
assert_eq!(reply.ended[0].scope_ref, "old");
assert_eq!(reply.ended[0].scope_epoch, 3);
}
#[tokio::test]
async fn scope_apply_rejects_a_reply_for_another_op() {
let (handle, mut rx) = test_module_handle(&[SCOPE_APPLY_OP]);
let apply = tokio::spawn({
let handle = handle.clone();
async move { handle.scope_apply(2, Vec::new(), Vec::new()).await }
});
let request = timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
let body = serde_json::to_vec(&ModuleControlResponseToModule::ScopeSync {
generation: 2,
results: Vec::new(),
ended: Vec::new(),
})
.unwrap();
assert!(handle.handle_control_reply(control_reply(
FrameType::Response,
request.header.corr,
body
)));
assert!(matches!(
apply.await.unwrap(),
Err(ScopeCallError::Protocol(_))
));
}
#[tokio::test]
async fn scope_apply_preserves_whole_call_refusal_codes() {
let (handle, mut rx) = test_module_handle(&[SCOPE_APPLY_OP]);
for code in [
subc_protocol::error_codes::SCOPE_SYNC_REQUIRED,
subc_protocol::error_codes::SCOPE_SYNC_STALE,
subc_protocol::error_codes::SCOPE_SYNC_NOT_AUTHORITY,
"not_registered",
] {
let apply = tokio::spawn({
let handle = handle.clone();
async move { handle.scope_apply(2, Vec::new(), Vec::new()).await }
});
let request = timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
let body = serde_json::to_vec(&ErrorBody::new(code, "why")).unwrap();
assert!(handle.handle_control_reply(control_reply(
FrameType::Error,
request.header.corr,
body
)));
let error = apply.await.unwrap().unwrap_err();
assert_eq!(
error,
ScopeCallError::Refused {
code: code.to_string(),
message: "why".to_string()
}
);
assert_eq!(error.code(), Some(code));
}
}
#[tokio::test]
async fn a_scope_reply_of_another_op_is_a_protocol_error() {
let (handle, mut rx) = test_module_handle(&[SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP]);
let sync = tokio::spawn({
let handle = handle.clone();
async move { handle.scope_sync(1, Vec::new()).await }
});
let request = timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
let describe_body = serde_json::to_vec(&ModuleControlResponseToModule::ScopeDescribe {
status: ScopeStatus::NotLive,
scope_epoch: None,
daemon_incarnation: "i".to_string(),
owner_synced: false,
owner_configured: false,
scope: None,
})
.unwrap();
assert!(handle.handle_control_reply(control_reply(
FrameType::Response,
request.header.corr,
describe_body
)));
assert!(matches!(
sync.await.unwrap(),
Err(ScopeCallError::Protocol(_))
));
let describe = tokio::spawn({
let handle = handle.clone();
async move { handle.scope_describe(owner(), "s".to_string()).await }
});
let request = timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
let sync_body = serde_json::to_vec(&ModuleControlResponseToModule::ScopeSync {
generation: 1,
results: Vec::new(),
ended: Vec::new(),
})
.unwrap();
assert!(handle.handle_control_reply(control_reply(
FrameType::Response,
request.header.corr,
sync_body
)));
assert!(matches!(
describe.await.unwrap(),
Err(ScopeCallError::Protocol(_))
));
}
#[tokio::test]
async fn a_scope_error_frame_keeps_the_daemons_code_and_message() {
let (handle, mut rx) = test_module_handle(&[SCOPE_SYNC_OP]);
for code in [
subc_protocol::error_codes::SCOPE_SYNC_STALE,
subc_protocol::error_codes::SCOPE_SYNC_NOT_AUTHORITY,
"not_registered",
] {
let sync = tokio::spawn({
let handle = handle.clone();
async move { handle.scope_sync(1, Vec::new()).await }
});
let request = timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
let body = serde_json::to_vec(&ErrorBody::new(code, "why")).unwrap();
assert!(handle.handle_control_reply(control_reply(
FrameType::Error,
request.header.corr,
body
)));
let error = sync.await.unwrap().unwrap_err();
assert_eq!(
error,
ScopeCallError::Refused {
code: code.to_string(),
message: "why".to_string()
}
);
assert_eq!(error.code(), Some(code));
}
}
#[tokio::test]
async fn an_undecodable_control_request_is_refused_and_the_module_keeps_serving() {
let (tx, mut rx) = mpsc::channel(4);
let handler = Arc::new(EchoHandler);
let dispatcher = RequestDispatcher::new();
let (module_handle, _unused_rx) = test_module_handle(&[]);
let mut body: serde_json::Value =
serde_json::from_slice(&route_bind_frame(9, 1, 41).body).unwrap();
body["scope"] = serde_json::json!({
"owner": { "kind": "reserved", "module_id": "prefrontal-core" },
"ref": "s",
"scope_epoch": 1,
"kind": "head",
"attributes": { "a_field_from_a_newer_daemon": "x" },
"owner_authorized": true
});
let body = serde_json::to_vec(&body).unwrap();
let bind = Frame::build(FrameType::Request, control_flags(), 0, 0, 41, body).unwrap();
let kept_serving = handle_frame(
bind,
&tx,
Arc::clone(&handler),
dispatcher.clone(),
module_handle.clone(),
)
.await
.expect("an undecodable control request must not end the module");
assert!(kept_serving);
let refusal = timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
assert_eq!(refusal.header.ty, FrameType::Error);
assert_eq!(refusal.header.corr, 41);
let error: ErrorBody = serde_json::from_slice(&refusal.body).unwrap();
assert_eq!(error.code, "invalid_request");
assert!(
error.message.contains("a_field_from_a_newer_daemon"),
"{}",
error.message
);
assert!(
handle_frame(health_request(42), &tx, handler, dispatcher, module_handle)
.await
.unwrap()
);
let health = timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
assert_eq!(health.header.ty, FrameType::Response);
assert_eq!(health.header.corr, 42);
}
#[tokio::test]
async fn default_health_check_answers_ok() {
let (tx, mut rx) = mpsc::channel(4);
let handler = Arc::new(EchoHandler);
let dispatcher = RequestDispatcher::new();
let (module_handle, _unused_rx) = test_module_handle(&[]);
assert!(
handle_frame(health_request(77), &tx, handler, dispatcher, module_handle)
.await
.unwrap()
);
let response = timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
assert_eq!(response.header.ty, FrameType::Response);
assert_eq!(response.header.channel, 0);
assert_eq!(response.header.corr, 77);
let parsed = serde_json::from_slice::<ModuleControlResponse>(&response.body).unwrap();
let ModuleControlResponse::HealthCheck { status, detail, .. } = parsed else {
panic!("expected a health.check response");
};
assert_eq!(status, HealthStatus::Ok);
assert!(
detail
.as_deref()
.is_some_and(|d| d.contains("no health implementation")),
"the inherited default must identify itself, got {detail:?}"
);
}
struct DispatchHarness {
tx: mpsc::Sender<Frame>,
rx: mpsc::Receiver<Frame>,
handler: Arc<BlockingHandler>,
dispatcher: RequestDispatcher,
module_handle: ModuleHandle,
}
impl DispatchHarness {
fn new(report: HealthReport) -> Self {
let (tx, rx) = mpsc::channel(HANDLER_TASK_CAPACITY + 4);
let (module_handle, _unused_rx) = test_module_handle(&[]);
module_handle
.install_route(RouteHandle::new(7, 1, 1))
.unwrap();
Self {
tx,
rx,
handler: Arc::new(BlockingHandler {
entered: Arc::new(AtomicUsize::new(0)),
release: Semaphore::new(0),
report,
}),
dispatcher: RequestDispatcher::new(),
module_handle,
}
}
async fn request(&self, corr: u64) {
handle_frame(
data_request(7, corr),
&self.tx,
Arc::clone(&self.handler),
self.dispatcher.clone(),
self.module_handle.clone(),
)
.await
.unwrap();
}
async fn wait_for_entered(&self, count: usize) {
timeout(Duration::from_secs(1), async {
while self.handler.entered.load(Ordering::SeqCst) != count {
tokio::task::yield_now().await;
}
})
.await
.expect("data handlers should acquire the available slots");
}
async fn saturate(&self) {
for corr in 1..=HANDLER_TASK_CAPACITY as u64 {
self.request(corr).await;
}
self.wait_for_entered(HANDLER_TASK_CAPACITY).await;
self.request(HANDLER_TASK_CAPACITY as u64 + 1).await;
tokio::task::yield_now().await;
assert_eq!(self.dispatcher.pools.default.permits.available_permits(), 0);
assert_eq!(
self.dispatcher.pools.default.waiting.lock().unwrap().len(),
1
);
assert_eq!(
self.handler.entered.load(Ordering::SeqCst),
HANDLER_TASK_CAPACITY
);
}
async fn health(&mut self, corr: u64) -> HealthReport {
handle_frame(
health_request(corr),
&self.tx,
Arc::clone(&self.handler),
self.dispatcher.clone(),
self.module_handle.clone(),
)
.await
.unwrap();
let frame = timeout(Duration::from_millis(100), self.rx.recv())
.await
.expect("health.check must answer within 100 ms while data slots are busy")
.unwrap();
assert_eq!(frame.header.ty, FrameType::Response);
assert_eq!(frame.header.channel, 0);
assert_eq!(frame.header.corr, corr);
serde_json::from_slice::<ModuleControlResponse>(&frame.body)
.unwrap()
.health_report()
.unwrap()
}
async fn release(&mut self, count: usize) {
self.handler.release.add_permits(count);
for _ in 0..count {
let frame = timeout(Duration::from_secs(1), self.rx.recv())
.await
.unwrap()
.unwrap();
assert_eq!(frame.header.ty, FrameType::StreamEnd);
assert_eq!(frame.header.channel, 7);
}
assert!(self
.dispatcher
.pools
.default
.waiting
.lock()
.unwrap()
.is_empty());
assert!(self.dispatcher.in_flight.lock().unwrap().is_empty());
assert_eq!(
self.dispatcher.pools.default.permits.available_permits(),
HANDLER_TASK_CAPACITY
);
}
}
#[tokio::test(start_paused = true)]
async fn health_check_answers_degraded_under_saturation_and_recovers() {
let own_report = HealthReport {
status: HealthStatus::Ok,
detail: Some("module gauges healthy".to_string()),
metrics: Some(json!({"transforms": 64})),
};
let mut harness = DispatchHarness::new(own_report.clone());
harness.saturate().await;
assert_eq!(harness.health(899).await, own_report);
advance(Duration::from_millis(7300)).await;
let report = harness.health(900).await;
assert_eq!(report.status, HealthStatus::Degraded);
assert_eq!(report.metrics, own_report.metrics);
assert_eq!(
report.detail.as_deref(),
Some("request dispatch saturated: 64/64 in use, oldest waiting 7.3 s; module gauges healthy")
);
harness.release(HANDLER_TASK_CAPACITY + 1).await;
assert_eq!(harness.health(901).await, own_report);
}
#[tokio::test(start_paused = true)]
async fn health_check_preserves_failing_under_saturation() {
let mut harness = DispatchHarness::new(HealthReport {
status: HealthStatus::Failing,
detail: Some("module dispatch heartbeat stale".to_string()),
metrics: Some(json!({"heartbeat_age_s": 12})),
});
harness.saturate().await;
advance(Duration::from_millis(7300)).await;
let report = harness.health(900).await;
assert_eq!(report.status, HealthStatus::Failing);
assert_eq!(report.metrics, Some(json!({"heartbeat_age_s": 12})));
assert_eq!(
report.detail.as_deref(),
Some("request dispatch saturated: 64/64 in use, oldest waiting 7.3 s; module dispatch heartbeat stale")
);
harness.release(HANDLER_TASK_CAPACITY + 1).await;
}
#[tokio::test(start_paused = true)]
async fn health_check_passes_module_report_through_with_free_permits() {
for status in [
HealthStatus::Ok,
HealthStatus::Degraded,
HealthStatus::Failing,
] {
let own_report = HealthReport {
status,
detail: Some("module's own assessment".to_string()),
metrics: Some(json!({"gauge": 42})),
};
let mut harness = DispatchHarness::new(own_report.clone());
harness.request(1).await;
harness.wait_for_entered(1).await;
advance(Duration::from_millis(7300)).await;
assert_eq!(harness.health(900).await, own_report);
harness.release(1).await;
}
}
struct CorrBlockingHandler {
entered: Arc<Mutex<Vec<u64>>>,
release_first: Arc<Semaphore>,
}
#[async_trait]
impl ModuleHandler for CorrBlockingHandler {
async fn handle(&self, ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
let corr = ctx.corr();
self.entered.lock().unwrap().push(corr);
if corr == 1 {
self.release_first.acquire().await.unwrap().forget();
}
HandlerOutcome::Response(Vec::new())
}
}
#[tokio::test]
async fn cancelled_capacity_queued_data_request_is_answered_without_waiting_for_a_slot() {
let (tx, mut rx) = mpsc::channel(4);
let entered = Arc::new(Mutex::new(Vec::new()));
let release_first = Arc::new(Semaphore::new(0));
let handler = Arc::new(CorrBlockingHandler {
entered: Arc::clone(&entered),
release_first: Arc::clone(&release_first),
});
let dispatcher = RequestDispatcher::with_pools(Arc::new(
DispatchPools::with_default_limit(1, RequestPools::new()),
));
let (module_handle, _unused_rx) = test_module_handle(&[]);
module_handle
.install_route(RouteHandle::new(7, 1, 1))
.unwrap();
handle_frame(
data_request(7, 1),
&tx,
Arc::clone(&handler),
dispatcher.clone(),
module_handle.clone(),
)
.await
.unwrap();
timeout(Duration::from_secs(1), async {
while entered.lock().unwrap().as_slice() != [1] {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
handle_frame(
data_request(7, 2),
&tx,
Arc::clone(&handler),
dispatcher.clone(),
module_handle.clone(),
)
.await
.unwrap();
timeout(Duration::from_secs(1), async {
while !dispatcher
.in_flight
.lock()
.unwrap()
.contains_key(&(7, 1, 2))
{
tokio::task::yield_now().await;
}
})
.await
.unwrap();
handle_frame(
Frame::build(FrameType::Cancel, data_flags(), 7, 1, 2, Vec::new()).unwrap(),
&tx,
Arc::clone(&handler),
dispatcher.clone(),
module_handle,
)
.await
.unwrap();
assert!(
dispatcher
.in_flight
.lock()
.unwrap()
.get(&(7, 1, 2))
.unwrap()
.is_cancelled(),
"cancel must land while the second request waits for handler capacity"
);
let cancelled = timeout(Duration::from_secs(1), rx.recv())
.await
.expect("a request cancelled while waiting must be answered without waiting for a slot")
.unwrap();
assert_eq!(cancelled.header.ty, FrameType::Error);
assert_eq!(cancelled.header.channel, 7);
assert_eq!(cancelled.header.epoch, 1);
assert_eq!(cancelled.header.corr, 2);
assert_eq!(
serde_json::from_slice::<ErrorBody>(&cancelled.body).unwrap(),
ErrorBody {
code: "cancelled".to_string(),
message: "request cancelled".to_string(),
detail: None,
}
);
timeout(Duration::from_secs(1), async {
while dispatcher
.in_flight
.lock()
.unwrap()
.contains_key(&(7, 1, 2))
{
tokio::task::yield_now().await;
}
})
.await
.unwrap();
assert_eq!(dispatcher.pools.default.permits.available_permits(), 0);
assert!(dispatcher.pools.default.waiting.lock().unwrap().is_empty());
assert_eq!(*entered.lock().unwrap(), vec![1]);
release_first.add_permits(1);
let response = timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
assert_eq!(response.header.ty, FrameType::Response);
assert_eq!(response.header.corr, 1);
timeout(Duration::from_secs(1), async {
while !dispatcher.in_flight.lock().unwrap().is_empty() {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
assert_eq!(*entered.lock().unwrap(), vec![1]);
assert_eq!(dispatcher.pools.default.permits.available_permits(), 1);
assert!(timeout(Duration::from_millis(50), rx.recv()).await.is_err());
}
#[tokio::test]
async fn cancelled_terminal_is_not_sent_after_route_teardown() {
let (tx, mut rx) = mpsc::channel(1);
let (module_handle, _unused_rx) = test_module_handle(&[]);
let handle = RouteHandle::new(7, 1, 1);
module_handle.install_route(handle).unwrap();
let ctx = RequestCtx {
handle,
corr: 2,
ver: PROTOCOL_VERSION,
egress: tx,
module_handle: module_handle.clone(),
cancelled: CancellationToken::new(),
binary_reply: Arc::new(AtomicBool::new(false)),
request_flags: Flags::new(false, Priority::Interactive, false),
request_body: Arc::from([]),
};
assert!(module_handle.remove_route(handle).unwrap());
let result = send_handler_outcome(
&ctx,
HandlerOutcome::Error {
code: "cancelled".to_string(),
message: "request cancelled".to_string(),
},
)
.await;
assert!(matches!(
result,
Err(SubcModuleError::StaleRouteHandle(stale)) if stale == handle
));
assert!(rx.try_recv().is_err());
}
struct CountingHandler {
calls: Arc<AtomicUsize>,
}
#[async_trait]
impl ModuleHandler for CountingHandler {
async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
self.calls.fetch_add(1, Ordering::SeqCst);
HandlerOutcome::Response(Vec::new())
}
}
#[tokio::test]
async fn endpoint_validation_drops_stale_request_before_handler_dispatch() {
let (tx, mut rx) = mpsc::channel(4);
let calls = Arc::new(AtomicUsize::new(0));
let handler = Arc::new(CountingHandler {
calls: Arc::clone(&calls),
});
let dispatcher = RequestDispatcher::new();
let (module_handle, _unused_rx) = test_module_handle(&[]);
module_handle
.install_route(RouteHandle::new(7, 2, 1))
.unwrap();
let stale = Frame::build(FrameType::Request, data_flags(), 7, 1, 55, Vec::new()).unwrap();
assert!(
handle_frame(stale, &tx, handler, dispatcher, module_handle.clone())
.await
.unwrap()
);
tokio::task::yield_now().await;
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert_eq!(module_handle.dropped_route_frames(), 1);
assert!(rx.try_recv().is_err());
}
struct BindOrderingHandler {
module_handle: ModuleHandle,
bind_emit_rejected: Arc<AtomicUsize>,
bound: Arc<AtomicUsize>,
cleanup: Arc<AtomicUsize>,
}
#[async_trait]
impl ModuleHandler for BindOrderingHandler {
async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
HandlerOutcome::Response(Vec::new())
}
async fn on_bind(&self, req: &RouteBindRequest) -> BindDecision {
if matches!(
self.module_handle
.push(&req.handle, b"too-early".to_vec(), None)
.await,
Err(SubcModuleError::StaleRouteHandle(_))
) {
self.bind_emit_rejected.fetch_add(1, Ordering::SeqCst);
}
BindDecision::accept()
}
async fn on_bound(&self, handle: &RouteHandle) {
self.bound.fetch_add(1, Ordering::SeqCst);
self.module_handle
.push(handle, b"bound".to_vec(), Some(AdmissionClass::Expedite))
.await
.unwrap();
}
async fn on_route_gone(&self, _handle: &RouteHandle) {
self.cleanup.fetch_add(1, Ordering::SeqCst);
}
}
fn route_bind_frame(channel: u16, epoch: u32, corr: u64) -> Frame {
let body = serde_json::to_vec(&ModuleControlRequest::RouteBind {
route_channel: channel,
epoch,
target: RouteTarget::ToolProvider {
module_id: "provider".to_string(),
},
identity: BindIdentity::new(
PathBuf::from("/tmp/project"),
"test".to_string(),
"bind".to_string(),
),
principal: None,
consumer_capabilities: None,
role_versions: None,
admission_facts: None,
scope: None,
call_mode: None,
})
.unwrap();
Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
}
#[tokio::test]
async fn on_bound_runs_only_after_ack_queue_and_handle_install() {
let (module_handle, mut rx) = test_module_handle(&[]);
let tx = module_handle.shared.lock_inner().writer.clone().unwrap();
let bind_emit_rejected = Arc::new(AtomicUsize::new(0));
let bound = Arc::new(AtomicUsize::new(0));
let cleanup = Arc::new(AtomicUsize::new(0));
let handler = Arc::new(BindOrderingHandler {
module_handle: module_handle.clone(),
bind_emit_rejected: Arc::clone(&bind_emit_rejected),
bound: Arc::clone(&bound),
cleanup: Arc::clone(&cleanup),
});
assert!(handle_frame(
route_bind_frame(8, 4, 90),
&tx,
handler,
RequestDispatcher::new(),
module_handle.clone(),
)
.await
.unwrap());
assert_eq!(bind_emit_rejected.load(Ordering::SeqCst), 1);
assert_eq!(bound.load(Ordering::SeqCst), 1);
assert_eq!(cleanup.load(Ordering::SeqCst), 0);
let ack = rx.recv().await.unwrap();
let push = rx.recv().await.unwrap();
assert_eq!(ack.header.ty, FrameType::Response);
assert_eq!(ack.header.channel, 0);
assert_eq!(push.header.ty, FrameType::Push);
assert_eq!((push.header.channel, push.header.epoch), (8, 4));
assert_eq!(
push.header.flags.admission_class(),
Some(AdmissionClass::Expedite)
);
let captured = RouteHandle::new(8, 4, 1);
assert!(module_handle.remove_route(captured).unwrap());
let stale = module_handle
.push(&captured, Vec::new(), None)
.await
.unwrap_err();
assert!(matches!(stale, SubcModuleError::StaleRouteHandle(_)));
assert!(rx.try_recv().is_err());
}
struct RejectingHandler {
bound: Arc<AtomicUsize>,
cleanup: Arc<AtomicUsize>,
}
#[async_trait]
impl ModuleHandler for RejectingHandler {
async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
HandlerOutcome::Response(Vec::new())
}
async fn on_bind(&self, _req: &RouteBindRequest) -> BindDecision {
BindDecision::reject("no", "rejected")
}
async fn on_bound(&self, _handle: &RouteHandle) {
self.bound.fetch_add(1, Ordering::SeqCst);
}
async fn on_route_gone(&self, _handle: &RouteHandle) {
self.cleanup.fetch_add(1, Ordering::SeqCst);
}
}
#[tokio::test]
async fn rejected_bind_cleans_up_without_installing_or_calling_on_bound() {
let (module_handle, mut rx) = test_module_handle(&[]);
let tx = module_handle.shared.lock_inner().writer.clone().unwrap();
let bound = Arc::new(AtomicUsize::new(0));
let cleanup = Arc::new(AtomicUsize::new(0));
let handler = Arc::new(RejectingHandler {
bound: Arc::clone(&bound),
cleanup: Arc::clone(&cleanup),
});
handle_frame(
route_bind_frame(6, 3, 91),
&tx,
handler,
RequestDispatcher::new(),
module_handle.clone(),
)
.await
.unwrap();
assert_eq!(rx.recv().await.unwrap().header.ty, FrameType::Error);
assert_eq!(bound.load(Ordering::SeqCst), 0);
assert_eq!(cleanup.load(Ordering::SeqCst), 1);
assert!(matches!(
module_handle.validate_route(RouteHandle::new(6, 3, 1)),
Err(SubcModuleError::StaleRouteHandle(_))
));
}
struct RebindCountingHandler {
bound: Arc<AtomicUsize>,
cleanup: Arc<AtomicUsize>,
}
#[async_trait]
impl ModuleHandler for RebindCountingHandler {
async fn handle(&self, _ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
HandlerOutcome::Response(Vec::new())
}
async fn on_bound(&self, _handle: &RouteHandle) {
self.bound.fetch_add(1, Ordering::SeqCst);
}
async fn on_route_gone(&self, _handle: &RouteHandle) {
self.cleanup.fetch_add(1, Ordering::SeqCst);
}
}
#[tokio::test]
async fn rebind_on_installed_channel_replaces_on_higher_epoch_only() {
let (module_handle, mut rx) = test_module_handle(&[]);
let tx = module_handle.shared.lock_inner().writer.clone().unwrap();
let bound = Arc::new(AtomicUsize::new(0));
let cleanup = Arc::new(AtomicUsize::new(0));
let handler = Arc::new(RebindCountingHandler {
bound: Arc::clone(&bound),
cleanup: Arc::clone(&cleanup),
});
handle_frame(
route_bind_frame(8, 4, 90),
&tx,
Arc::clone(&handler),
RequestDispatcher::new(),
module_handle.clone(),
)
.await
.unwrap();
assert_eq!(rx.recv().await.unwrap().header.ty, FrameType::Response);
assert_eq!(
(bound.load(Ordering::SeqCst), cleanup.load(Ordering::SeqCst)),
(1, 0)
);
handle_frame(
route_bind_frame(8, 4, 91),
&tx,
Arc::clone(&handler),
RequestDispatcher::new(),
module_handle.clone(),
)
.await
.unwrap();
let reject = rx.recv().await.unwrap();
assert_eq!(reject.header.ty, FrameType::Error);
assert_eq!(
(bound.load(Ordering::SeqCst), cleanup.load(Ordering::SeqCst)),
(1, 0)
);
module_handle
.validate_route(RouteHandle::new(8, 4, 1))
.expect("epoch-4 install must survive the rejected rebind");
handle_frame(
route_bind_frame(8, 3, 92),
&tx,
Arc::clone(&handler),
RequestDispatcher::new(),
module_handle.clone(),
)
.await
.unwrap();
assert_eq!(rx.recv().await.unwrap().header.ty, FrameType::Error);
assert_eq!(
(bound.load(Ordering::SeqCst), cleanup.load(Ordering::SeqCst)),
(1, 0)
);
handle_frame(
route_bind_frame(8, 5, 93),
&tx,
Arc::clone(&handler),
RequestDispatcher::new(),
module_handle.clone(),
)
.await
.unwrap();
assert_eq!(rx.recv().await.unwrap().header.ty, FrameType::Response);
assert_eq!(
(bound.load(Ordering::SeqCst), cleanup.load(Ordering::SeqCst)),
(2, 1)
);
assert!(matches!(
module_handle.validate_route(RouteHandle::new(8, 4, 1)),
Err(SubcModuleError::StaleRouteHandle(_))
));
module_handle
.validate_route(RouteHandle::new(8, 5, 1))
.expect("epoch-5 install must be live after implicit replace");
}
#[tokio::test]
async fn module_control_corr_is_monotonic_and_exhausts_without_wrap() {
let (module_handle, _rx) = test_module_handle(&[MODULE_TO_SUBC_OP_CATALOG_UPDATE]);
let mut inner = module_handle.shared.lock_inner();
inner.next_corr = Some(u64::MAX);
assert_eq!(next_module_control_corr(&mut inner), Some(u64::MAX));
assert_eq!(next_module_control_corr(&mut inner), None);
}
}
#[cfg(test)]
mod module_close_tests {
use std::time::Instant;
use tokio::{io::DuplexStream, sync::Notify};
use super::stand_in::{serve_against_stand_in_with_ops, Served};
use super::*;
#[derive(Clone, Copy)]
enum Hold {
UntilCancelled,
IgnoringCancellation,
}
struct HoldingHandler {
hold: Hold,
entered: Arc<Notify>,
token: Arc<Mutex<Option<CancellationToken>>>,
}
#[async_trait]
impl ModuleHandler for HoldingHandler {
async fn handle(&self, ctx: RequestCtx, _body: Vec<u8>) -> HandlerOutcome {
*self.token.lock().unwrap() = Some(ctx.cancellation_token());
self.entered.notify_one();
match self.hold {
Hold::UntilCancelled => {
ctx.cancelled().await;
HandlerOutcome::Error {
code: "cancelled".to_string(),
message: "request cancelled".to_string(),
}
}
Hold::IgnoringCancellation => std::future::pending().await,
}
}
}
async fn serve_against_stand_in<H: ModuleHandler>(handler: H) -> Served {
serve_against_stand_in_with_ops(|_| handler, &[]).await
}
async fn send(daemon: &mut DuplexStream, frame: Frame) {
write_frame(daemon, &frame).await.unwrap();
daemon.flush().await.unwrap();
}
async fn hold_one_request(served: &mut Served, entered: &Notify) {
let bind = serde_json::to_vec(&ModuleControlRequest::RouteBind {
route_channel: 7,
epoch: 1,
target: RouteTarget::ToolProvider {
module_id: "close-test".to_string(),
},
identity: BindIdentity::new(
PathBuf::from("/tmp/project"),
"test".to_string(),
"close".to_string(),
),
principal: None,
consumer_capabilities: None,
role_versions: None,
admission_facts: None,
scope: None,
call_mode: None,
})
.unwrap();
send(
&mut served.daemon,
Frame::build(FrameType::Request, control_flags(), 0, 0, 2, bind).unwrap(),
)
.await;
loop {
let frame = read_frame(&mut served.daemon).await.unwrap().unwrap();
if frame.header.channel == 0 && frame.header.corr == 2 {
assert_eq!(
frame.header.ty,
FrameType::Response,
"route.bind must be acked"
);
break;
}
}
send(
&mut served.daemon,
Frame::build(FrameType::Request, data_flags(), 7, 1, 10, b"hold".to_vec()).unwrap(),
)
.await;
timeout(Duration::from_secs(2), entered.notified())
.await
.expect("the handler must receive the request");
}
fn goodbye() -> Frame {
Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 0, Vec::new()).unwrap()
}
fn holding(
hold: Hold,
) -> (
HoldingHandler,
Arc<Notify>,
Arc<Mutex<Option<CancellationToken>>>,
) {
let entered = Arc::new(Notify::new());
let token = Arc::new(Mutex::new(None));
let handler = HoldingHandler {
hold,
entered: Arc::clone(&entered),
token: Arc::clone(&token),
};
(handler, entered, token)
}
#[tokio::test]
async fn goodbye_cancels_an_open_request_and_the_serve_future_completes() {
let (handler, entered, token) = holding(Hold::UntilCancelled);
let mut served = serve_against_stand_in(handler).await;
hold_one_request(&mut served, &entered).await;
let started = Instant::now();
send(&mut served.daemon, goodbye()).await;
let result = timeout(Duration::from_secs(5), &mut served.serve)
.await
.expect("a held request must not keep the serve future alive after GOODBYE");
result.unwrap().unwrap();
assert!(
token.lock().unwrap().as_ref().unwrap().is_cancelled(),
"GOODBYE must cancel the request's token"
);
assert!(
started.elapsed() < DEFAULT_WRITER_DRAIN_TIMEOUT,
"closing took {:?}",
started.elapsed()
);
}
#[tokio::test]
async fn a_handler_ignoring_cancellation_cannot_keep_the_serve_future_alive() {
let (handler, entered, _token) = holding(Hold::IgnoringCancellation);
let mut served = serve_against_stand_in(handler).await;
hold_one_request(&mut served, &entered).await;
let started = Instant::now();
send(&mut served.daemon, goodbye()).await;
let result = timeout(
DEFAULT_WRITER_DRAIN_TIMEOUT + Duration::from_secs(3),
&mut served.serve,
)
.await
.expect("the writer drain must be bounded");
result.unwrap().unwrap();
assert!(started.elapsed() >= DEFAULT_WRITER_DRAIN_TIMEOUT);
}
#[tokio::test]
async fn closed_resolves_after_goodbye() {
let served = serve_against_stand_in(EchoHandler).await;
let mut served = served;
assert!(!served.handle.is_closed());
let closed = served.handle.closed();
send(&mut served.daemon, goodbye()).await;
timeout(Duration::from_secs(2), closed)
.await
.expect("closed() must resolve after GOODBYE");
assert!(served.handle.is_closed());
}
#[tokio::test]
async fn closed_resolves_after_eof() {
let Served {
daemon,
handle,
serve: _serve,
} = serve_against_stand_in(EchoHandler).await;
assert!(!handle.is_closed());
drop(daemon);
timeout(Duration::from_secs(2), handle.closed())
.await
.expect("closed() must resolve after EOF");
assert!(handle.is_closed());
}
struct EchoHandler;
#[async_trait]
impl ModuleHandler for EchoHandler {
async fn handle(&self, _ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome {
HandlerOutcome::Response(body)
}
}
#[derive(Clone, Default)]
struct EndRecorder(Arc<Mutex<Vec<ConnectionEnd>>>);
#[async_trait]
impl ModuleHandler for EndRecorder {
async fn handle(&self, _ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome {
HandlerOutcome::Response(body)
}
async fn on_connection_end(&self, end: ConnectionEnd) {
self.0.lock().unwrap().push(end);
}
}
async fn reported_end(recorder: &EndRecorder, served: &mut Served) -> Vec<ConnectionEnd> {
timeout(Duration::from_secs(5), &mut served.serve)
.await
.expect("serving must end")
.unwrap()
.unwrap();
recorder.0.lock().unwrap().clone()
}
#[tokio::test]
async fn a_daemon_goodbye_is_reported_as_goodbye() {
let recorder = EndRecorder::default();
let mut served = serve_against_stand_in(recorder.clone()).await;
send(&mut served.daemon, goodbye()).await;
assert_eq!(
reported_end(&recorder, &mut served).await,
vec![ConnectionEnd::Goodbye]
);
}
#[tokio::test]
async fn a_daemon_closing_without_goodbye_is_reported_as_eof() {
let recorder = EndRecorder::default();
let mut served = serve_against_stand_in(recorder.clone()).await;
served.daemon.shutdown().await.unwrap();
assert_eq!(
reported_end(&recorder, &mut served).await,
vec![ConnectionEnd::Eof]
);
}
#[derive(Debug, Clone, PartialEq)]
enum Seen {
Draining(RouteCloseReason, SystemTime),
End(ConnectionEnd),
}
#[derive(Clone, Default)]
struct DrainRecorder {
seen: Arc<Mutex<Vec<Seen>>>,
park: bool,
}
#[async_trait]
impl ModuleHandler for DrainRecorder {
async fn handle(&self, _ctx: RequestCtx, body: Vec<u8>) -> HandlerOutcome {
HandlerOutcome::Response(body)
}
async fn on_draining(&self, reason: RouteCloseReason, deadline: SystemTime) {
self.seen
.lock()
.unwrap()
.push(Seen::Draining(reason, deadline));
if self.park {
std::future::pending::<()>().await;
}
}
async fn on_connection_end(&self, end: ConnectionEnd) {
self.seen.lock().unwrap().push(Seen::End(end));
}
}
fn control_push(body: &[u8]) -> Frame {
Frame::build(FrameType::Push, control_flags(), 0, 0, 0, body.to_vec()).unwrap()
}
fn draining_push(reason: subc_protocol::RouteCloseReason, deadline_ms: u64) -> Frame {
control_push(
&serde_json::to_vec(&ModuleControlCommand::Draining {
reason,
deadline_ms,
})
.unwrap(),
)
}
async fn ping_is_answered(served: &mut Served, corr: u64) {
send(
&mut served.daemon,
Frame::build(FrameType::Ping, control_flags(), 0, 0, corr, Vec::new()).unwrap(),
)
.await;
let pong = timeout(Duration::from_secs(2), read_frame(&mut served.daemon))
.await
.expect("the module must answer PING")
.unwrap()
.expect("the module must not close the connection");
assert_eq!(pong.header.ty, FrameType::Pong);
assert_eq!(pong.header.corr, corr);
}
#[tokio::test]
async fn a_drain_notice_calls_on_draining_before_the_goodbye_that_follows_it() {
let recorder = DrainRecorder::default();
let mut served = serve_against_stand_in(recorder.clone()).await;
let deadline_ms = 4_102_444_800_000;
write_frame(
&mut served.daemon,
&draining_push(subc_protocol::RouteCloseReason::Restart, deadline_ms),
)
.await
.unwrap();
write_frame(&mut served.daemon, &goodbye()).await.unwrap();
served.daemon.flush().await.unwrap();
timeout(Duration::from_secs(5), &mut served.serve)
.await
.expect("serving must end after GOODBYE")
.unwrap()
.unwrap();
assert_eq!(
*recorder.seen.lock().unwrap(),
vec![
Seen::Draining(
RouteCloseReason::Restart,
UNIX_EPOCH + Duration::from_millis(deadline_ms)
),
Seen::End(ConnectionEnd::Goodbye),
]
);
}
#[tokio::test]
async fn a_drain_hook_that_never_returns_does_not_stop_the_frame_reader() {
let recorder = DrainRecorder {
park: true,
..DrainRecorder::default()
};
let mut served = serve_against_stand_in(recorder.clone()).await;
send(
&mut served.daemon,
draining_push(subc_protocol::RouteCloseReason::Reload, 1),
)
.await;
ping_is_answered(&mut served, 41).await;
assert!(matches!(
recorder.seen.lock().unwrap().as_slice(),
[Seen::Draining(RouteCloseReason::Reload, _)]
));
send(&mut served.daemon, goodbye()).await;
timeout(Duration::from_secs(5), &mut served.serve)
.await
.expect("a parked drain hook must not keep the serve future alive")
.unwrap()
.unwrap();
}
#[tokio::test]
async fn an_undecodable_channel_zero_push_is_ignored_and_the_connection_stays_up() {
let recorder = DrainRecorder::default();
let mut served = serve_against_stand_in(recorder.clone()).await;
send(
&mut served.daemon,
control_push(br#"{"op":"module.some_future_command","x":1}"#),
)
.await;
send(&mut served.daemon, control_push(b"not json")).await;
send(
&mut served.daemon,
control_push(br#"{"op":"module.draining","reason":"restart"}"#),
)
.await;
ping_is_answered(&mut served, 42).await;
assert!(!served.handle.is_closed());
assert!(!served.serve.is_finished());
assert!(recorder.seen.lock().unwrap().is_empty());
}
#[tokio::test]
async fn a_module_closing_its_own_connection_is_reported_as_closed() {
let recorder = EndRecorder::default();
let mut served = serve_against_stand_in(recorder.clone()).await;
served.handle.close_connection();
assert_eq!(
reported_end(&recorder, &mut served).await,
vec![ConnectionEnd::Closed]
);
}
}
#[cfg(test)]
mod operator_confirm_tests;
#[cfg(test)]
mod binary_reply_tests;
#[cfg(test)]
mod serve_lifecycle_tests;
#[cfg(test)]
mod request_pool_tests;
#[cfg(test)]
mod detailed_error_body_tests {
use serde_json::json;
use subc_protocol::ErrorBody;
#[test]
fn detailed_error_body_keeps_code_message_and_detail() {
let body = ErrorBody::new("bad_request", "invalid envelope")
.with_detail(json!({"reason": "missing_server"}));
assert_eq!(
serde_json::to_value(body).unwrap(),
json!({
"code": "bad_request",
"message": "invalid envelope",
"detail": {"reason": "missing_server"},
})
);
}
#[test]
fn detail_less_error_body_is_byte_identical_to_the_pre_detail_wire() {
let old_wire = r#"{"code":"cancelled","message":"caller cancelled"}"#;
let parsed: ErrorBody = serde_json::from_str(old_wire).unwrap();
assert_eq!(parsed.detail, None);
assert_eq!(serde_json::to_string(&parsed).unwrap(), old_wire);
}
}
static RETAINED_LAUNCH_NONCE: std::sync::OnceLock<String> = std::sync::OnceLock::new();
pub fn retain_launch_nonce_for_hello(launch_nonce: String) -> Result<(), String> {
RETAINED_LAUNCH_NONCE
.set(launch_nonce)
.map_err(|_| "launch nonce was already retained for this process".to_string())
}
fn retained_launch_nonce() -> Option<String> {
RETAINED_LAUNCH_NONCE.get().cloned()
}
pub fn launch_nonce_source() -> Option<LaunchNonceSource> {
launch_nonce()
.ok()
.flatten()
.map(|nonce| LaunchNonceSource::from_wire_name(nonce.source().as_str()))
}
fn stamp_launch_nonce_source(manifest: &mut ModuleManifest, sent: Option<&str>) {
let Some(provenance) = manifest.provenance.as_mut() else {
return;
};
if provenance.launch_nonce_source.is_some() {
return;
}
if let Ok(Some(nonce)) = launch_nonce() {
if sent == Some(nonce.value()) {
provenance.launch_nonce_source =
Some(LaunchNonceSource::from_wire_name(nonce.source().as_str()));
}
}
}
#[cfg(test)]
mod checked_increment_tests {
use super::checked_increment;
use std::sync::atomic::{AtomicU64, Ordering};
#[test]
fn returns_the_previous_value_and_stops_at_the_maximum() {
let counter = AtomicU64::new(7);
assert_eq!(checked_increment(&counter), Some(7));
assert_eq!(counter.load(Ordering::Relaxed), 8);
let full = AtomicU64::new(u64::MAX);
assert_eq!(checked_increment(&full), None);
assert_eq!(
full.load(Ordering::Relaxed),
u64::MAX,
"an exhausted counter is left unchanged"
);
}
}