use std::{
collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque},
error::Error,
fmt,
future::Future,
io,
path::{Path, PathBuf},
pin::Pin,
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc, Mutex, MutexGuard, OnceLock,
},
task::{Context, Poll},
time::{Duration, SystemTime, UNIX_EPOCH},
};
use sha2::{Digest, Sha256};
pub use subc_control::SupervisorEntry;
use subc_control::{
CatalogEntry, ClientControlRequest, ClientControlResponse, ConsumerIdentity, PollKind,
RouteOpenReplyExt, RouteOpenRequestExt, ROUTE_CLOSE_REASONS_CAPABILITY,
};
pub use subc_control::{LiveSpawn, SpawnCursor, SpawnEvent, SpawnEventKind, SpawnSnapshot};
pub use subc_control::{RouteOpenTarget, RouteOpenTargetKind};
pub use subc_protocol::scope::ScopeSelector;
use subc_protocol::{
error_codes, manifest::is_valid_capability_identifier, AdmissionClass, BindIdentity, ErrorBody,
Flags, Frame, FrameBuildError, FrameType, Priority, RouteTarget, FLAG_SUBSCRIPTION,
SUBC_MODULE_ID_ENV,
};
use crate::{
ActivityCursor, ActivityQueryRequest, ActivityQueryResponse, ActivitySubscribeRequest,
ActivitySubscription, RouteHandle,
};
use subc_transport::{
authenticate_client, connection_file, read_frame, write_frame, AuthError, ConnectionFileError,
FrameIoError,
};
use tokio::{
io::{AsyncRead, AsyncWrite, AsyncWriteExt, BufWriter},
net::TcpStream,
sync::{mpsc, oneshot, watch, Notify, OwnedSemaphorePermit, Semaphore},
task::JoinHandle,
time::{sleep, sleep_until, timeout_at, Instant},
};
use tokio_util::sync::CancellationToken;
const DEFAULT_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(2);
const CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(2);
pub const DEFAULT_CALL_TIMEOUT: Duration = Duration::from_secs(30);
pub const DEFAULT_ROUTE_RETRY_DEADLINE: Duration = Duration::from_secs(90);
pub const MAX_ROUTE_OPENS_IN_FLIGHT: usize = 8;
const DEFAULT_RESTORED_DEBOUNCE: Duration = Duration::from_millis(250);
pub const DEFAULT_LIVENESS_PROBE_WINDOW: Duration = Duration::from_secs(2);
const EGRESS_BUFFER: usize = 128;
const DEFAULT_ROUTE_WINDOW: usize = 1024;
const DEFAULT_SUBSCRIPTION_EVENT_BUFFER: usize = 128;
const DEFAULT_PUSH_EVENT_BUFFER: usize = 128;
const REVERSE_REQUEST_UNHANDLED: &str = "reverse_request_unhandled";
const SPAWN_EVENT_BUFFER: usize = 8192;
const ACTIVITY_EVENT_BUFFER: usize = 2048;
#[cfg(test)]
#[path = "activity_consumer_tests.rs"]
mod activity_consumer_tests;
mod cancellation;
mod slots;
use slots::*;
mod reservation;
pub use reservation::{CorrelationId, PendingReply, RequestBody, RequestReservation};
mod catalog;
mod supervisor_ops;
use catalog::decode_route_open_response;
pub use catalog::CatalogSubscription;
#[cfg(test)]
mod reader_skew_tests;
#[cfg(test)]
mod connection_ops_tests;
pub use cancellation::{CancelHandle, CancelOutcome};
use cancellation::{CancelRegistration, UnaryCancel};
pub use supervisor_ops::{
HealthProbeResponse, ServerDescription, StderrTailResponse, SupervisorAck,
SupervisorHealthResponse, SupervisorProvenanceResponse,
};
pub const SPAWN_CURSOR_INCARNATION_MISMATCH: &str = "spawn_cursor_incarnation_mismatch";
pub const SPAWN_CURSOR_TOO_OLD: &str = "spawn_cursor_too_old";
pub const SPAWN_SUBSCRIBER_LAGGED: &str = "spawn_subscriber_lagged";
type ReverseRequestFuture =
Pin<Box<dyn Future<Output = Result<Vec<u8>, ReverseRequestError>> + Send + 'static>>;
type ReverseRequestHandler =
Arc<dyn Fn(Vec<u8>, ReverseRequestContext) -> ReverseRequestFuture + Send + Sync + 'static>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ReverseRequestContext {
pub corr: u64,
pub method: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ReverseRequestError {
message: String,
}
impl ReverseRequestError {
pub fn new(message: impl Into<String>) -> Self {
Self {
message: message.into(),
}
}
}
impl fmt::Display for ReverseRequestError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.message)
}
}
impl Error for ReverseRequestError {}
#[derive(Default)]
struct ReverseRequestRegistryState {
handlers: HashMap<String, ReverseRequestHandler>,
declared_families: Option<BTreeSet<String>>,
consumer_capabilities: BTreeSet<String>,
}
#[derive(Clone, Default)]
pub struct ReverseRequestRegistry {
inner: Arc<Mutex<ReverseRequestRegistryState>>,
}
impl fmt::Debug for ReverseRequestRegistry {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ReverseRequestRegistry")
.field("capabilities", &self.capabilities())
.field("consumer_capabilities", &self.consumer_capabilities())
.finish()
}
}
impl PartialEq for ReverseRequestRegistry {
fn eq(&self, other: &Self) -> bool {
self.capabilities() == other.capabilities()
&& self.consumer_capabilities() == other.consumer_capabilities()
}
}
impl Eq for ReverseRequestRegistry {}
impl ReverseRequestRegistry {
pub fn new() -> Self {
Self::default()
}
pub fn on_request<F, Fut>(
&self,
method_family: impl Into<String>,
handler: F,
) -> Result<(), ReverseRequestRegistrationError>
where
F: Fn(Vec<u8>, ReverseRequestContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Vec<u8>> + Send + 'static,
{
self.on_request_fallible(method_family, move |body, ctx| {
let future = handler(body, ctx);
async move { Ok(future.await) }
})
}
pub fn on_request_fallible<F, Fut>(
&self,
method_family: impl Into<String>,
handler: F,
) -> Result<(), ReverseRequestRegistrationError>
where
F: Fn(Vec<u8>, ReverseRequestContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<Vec<u8>, ReverseRequestError>> + Send + 'static,
{
let method_family = method_family.into();
if !is_valid_method_family(&method_family) {
return Err(ReverseRequestRegistrationError::InvalidFamily(
method_family,
));
}
let mut state = self
.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if state
.declared_families
.as_ref()
.is_some_and(|families| !families.contains(&method_family))
{
return Err(ReverseRequestRegistrationError::NotDeclared(method_family));
}
state.handlers.insert(
method_family,
Arc::new(move |body, ctx| Box::pin(handler(body, ctx))),
);
Ok(())
}
fn capabilities(&self) -> Vec<String> {
let state = self
.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let mut capabilities = state.handlers.keys().cloned().collect::<Vec<_>>();
capabilities.sort();
capabilities
}
fn declare_consumer_capability(
&self,
capability: String,
) -> Result<(), ConsumerCapabilityError> {
if capability.is_empty()
|| capability.len() > 64
|| !capability.bytes().all(|byte| (0x21..=0x7e).contains(&byte))
{
return Err(ConsumerCapabilityError { capability });
}
self.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.consumer_capabilities
.insert(capability);
Ok(())
}
fn consumer_capabilities(&self) -> Vec<String> {
self.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.consumer_capabilities
.iter()
.cloned()
.collect()
}
fn handler(&self, method_family: &str) -> Option<ReverseRequestHandler> {
self.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.handlers
.get(method_family)
.cloned()
}
pub(crate) fn seal(&self) {
let mut state = self
.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if state.declared_families.is_none() {
state.declared_families = Some(state.handlers.keys().cloned().collect());
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ReverseRequestRegistrationError {
InvalidFamily(String),
NotDeclared(String),
NotConsumerRoute,
}
impl fmt::Display for ReverseRequestRegistrationError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidFamily(family) => {
write!(f, "invalid reverse-request method family {family:?}")
}
Self::NotDeclared(family) => write!(
f,
"reverse-request capability {family:?} was not registered before route.open"
),
Self::NotConsumerRoute => {
f.write_str("route handle does not belong to a consumer route")
}
}
}
}
impl Error for ReverseRequestRegistrationError {}
fn is_valid_method_family(value: &str) -> bool {
let mut segments = value.split(['.', '_', '-']);
segments.next().is_some_and(|segment| {
segment
.as_bytes()
.first()
.is_some_and(u8::is_ascii_lowercase)
&& segment
.bytes()
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit())
}) && segments.all(|segment| {
!segment.is_empty()
&& segment
.bytes()
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit())
})
}
static NEXT_REVERSE_REQUEST_REGISTRY_ID: AtomicU64 = AtomicU64::new(1);
static REVERSE_REQUEST_REGISTRIES: OnceLock<Mutex<HashMap<u64, ReverseRequestRegistry>>> =
OnceLock::new();
fn reverse_request_registries() -> &'static Mutex<HashMap<u64, ReverseRequestRegistry>> {
REVERSE_REQUEST_REGISTRIES.get_or_init(|| Mutex::new(HashMap::new()))
}
pub(crate) fn install_reverse_request_registry(registry: ReverseRequestRegistry) -> u64 {
let id = crate::checked_increment(&NEXT_REVERSE_REQUEST_REGISTRY_ID)
.expect("reverse-request route registry id exhausted");
reverse_request_registries()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.insert(id, registry);
id
}
pub(crate) fn route_reverse_request_registry(
id: u64,
) -> Result<ReverseRequestRegistry, ReverseRequestRegistrationError> {
if id == 0 {
return Err(ReverseRequestRegistrationError::NotConsumerRoute);
}
reverse_request_registries()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.get(&id)
.cloned()
.ok_or(ReverseRequestRegistrationError::NotConsumerRoute)
}
fn release_reverse_request_registry(handle: RouteHandle) {
let id = handle.reverse_request_registry_id();
if id != 0 {
reverse_request_registries()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.remove(&id);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RetryBackoff {
pub base: Duration,
pub cap: Duration,
pub max_attempts: usize,
}
impl Default for RetryBackoff {
fn default() -> Self {
Self {
base: Duration::from_millis(100),
cap: Duration::from_secs(2),
max_attempts: 6,
}
}
}
impl RetryBackoff {
fn delay_after_attempt(self, attempt: usize) -> Duration {
let mut delay = self.base;
for _ in 1..attempt {
delay = (delay * 2).min(self.cap);
}
delay
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ConsumerTransport {
Tcp,
Local,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ConsumerOptions {
pub handshake_timeout: Duration,
pub call_timeout: Duration,
pub reconnect_backoff: RetryBackoff,
pub restored_debounce: Duration,
pub liveness_probe_window: Duration,
}
impl Default for ConsumerOptions {
fn default() -> Self {
Self {
handshake_timeout: DEFAULT_HANDSHAKE_TIMEOUT,
call_timeout: DEFAULT_CALL_TIMEOUT,
reconnect_backoff: RetryBackoff::default(),
restored_debounce: DEFAULT_RESTORED_DEBOUNCE,
liveness_probe_window: DEFAULT_LIVENESS_PROBE_WINDOW,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ConsumerCapabilityError {
capability: String,
}
impl ConsumerCapabilityError {
pub fn capability(&self) -> &str {
&self.capability
}
}
impl fmt::Display for ConsumerCapabilityError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"invalid consumer capability {:?}: expected 1-64 printable ASCII bytes without whitespace",
self.capability
)
}
}
impl Error for ConsumerCapabilityError {}
#[derive(Debug, Clone)]
pub struct CloseRouteOptions {
pub scope: Option<ScopeSelector>,
pub drain: bool,
pub drain_timeout: Duration,
pub consumer_identity: Option<ConsumerIdentity>,
pub reverse_requests: ReverseRequestRegistry,
pub role_versions: Option<BTreeMap<String, String>>,
}
impl Default for CloseRouteOptions {
fn default() -> Self {
Self {
scope: None,
drain: false,
drain_timeout: DEFAULT_CALL_TIMEOUT,
consumer_identity: None,
reverse_requests: ReverseRequestRegistry::new(),
role_versions: None,
}
}
}
impl CloseRouteOptions {
pub fn with_consumer_capability(
self,
capability: impl Into<String>,
) -> Result<Self, ConsumerCapabilityError> {
self.reverse_requests
.declare_consumer_capability(capability.into())?;
Ok(self)
}
pub fn consumer_capabilities(&self) -> Vec<String> {
self.reverse_requests.consumer_capabilities()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CallOptions {
pub scope: Option<ScopeSelector>,
pub timeout: Duration,
pub priority: Priority,
pub admission_class: AdmissionClass,
pub route_retry: RetryBackoff,
pub route_retry_deadline: Duration,
pub consumer_identity: Option<ConsumerIdentity>,
pub reverse_requests: ReverseRequestRegistry,
pub role_versions: Option<BTreeMap<String, String>>,
}
impl Default for CallOptions {
fn default() -> Self {
Self {
scope: None,
timeout: DEFAULT_CALL_TIMEOUT,
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
route_retry: RetryBackoff::default(),
route_retry_deadline: DEFAULT_ROUTE_RETRY_DEADLINE,
consumer_identity: None,
reverse_requests: ReverseRequestRegistry::new(),
role_versions: None,
}
}
}
impl CallOptions {
fn reject_scope(&self) -> Result<(), CallError> {
if self.scope.is_some() {
return Err(CallError::not_sent(
"CallOptions.scope is not supported here; select the scope with open_route_scoped or a managed call",
));
}
Ok(())
}
pub fn with_consumer_capability(
self,
capability: impl Into<String>,
) -> Result<Self, ConsumerCapabilityError> {
self.reverse_requests
.declare_consumer_capability(capability.into())?;
Ok(self)
}
pub fn consumer_capabilities(&self) -> Vec<String> {
self.reverse_requests.consumer_capabilities()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SubscribeOptions {
pub scope: Option<ScopeSelector>,
pub priority: Priority,
pub admission_class: AdmissionClass,
pub event_buffer: usize,
pub route_retry: RetryBackoff,
pub route_retry_deadline: Duration,
pub route_open_timeout: Duration,
pub consumer_identity: Option<ConsumerIdentity>,
pub reverse_requests: ReverseRequestRegistry,
pub role_versions: Option<BTreeMap<String, String>>,
}
impl Default for SubscribeOptions {
fn default() -> Self {
Self {
scope: None,
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
event_buffer: DEFAULT_SUBSCRIPTION_EVENT_BUFFER,
route_retry: RetryBackoff::default(),
route_retry_deadline: DEFAULT_ROUTE_RETRY_DEADLINE,
route_open_timeout: DEFAULT_CALL_TIMEOUT,
consumer_identity: None,
reverse_requests: ReverseRequestRegistry::new(),
role_versions: None,
}
}
}
impl SubscribeOptions {
pub fn with_consumer_capability(
self,
capability: impl Into<String>,
) -> Result<Self, ConsumerCapabilityError> {
self.reverse_requests
.declare_consumer_capability(capability.into())?;
Ok(self)
}
pub fn consumer_capabilities(&self) -> Vec<String> {
self.reverse_requests.consumer_capabilities()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ConnectionState {
Dropped,
Restored { epoch: u64 },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RoutePollResult {
pub handle: RouteHandle,
pub status: Option<String>,
pub live: Option<bool>,
}
#[derive(Debug, Clone, serde::Deserialize, PartialEq)]
pub struct CatalogList {
pub generation: u64,
#[serde(default)]
pub modules: Vec<CatalogEntry>,
#[serde(default)]
pub subc_ops: Vec<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct SupervisorListResponse {
pub generation: u64,
pub modules: Vec<SupervisorEntry>,
pub daemon: Option<subc_control::DaemonResources>,
pub agent_commands: Option<Vec<subc_control::AgentCommandResources>>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PushEvent {
pub handle: RouteHandle,
pub body: Vec<u8>,
}
enum DroppedPush {
Delivered,
NoReceiver,
ReceiverFull,
}
#[derive(Debug, Clone)]
pub struct ControlPush {
pub op: String,
pub body: serde_json::Value,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum RouteCloseReason {
Reload,
Restart,
AuthorityChanged,
Disable,
Crash,
CapabilityDenied,
ScopeEnded,
ScopeCarrierRemoved,
ScopeDelegationChanged,
ScopeParentEnded,
Unknown(String),
}
pub const ACCEPTED_CLOSE_REASONS: &[&str] = &[
"reload",
"restart",
"authority_changed",
"disable",
"crash",
"capability_denied",
"scope_ended",
"scope_carrier_removed",
"scope_delegation_changed",
"scope_parent_ended",
];
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum RouteEndReason {
Daemon(RouteCloseReason),
ClosedByCaller,
ConnectionLost,
Unknown,
}
#[derive(Debug, Clone)]
struct RouteEnd {
reason: RouteEndReason,
terminal: Option<bool>,
}
impl From<RouteEndReason> for RouteEnd {
fn from(reason: RouteEndReason) -> Self {
Self {
reason,
terminal: None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RouteCloseDisposition {
MayReopen,
MustNotReopen,
}
impl RouteCloseReason {
pub fn from_wire(reason: &str) -> Self {
match reason {
"reload" => Self::Reload,
"restart" => Self::Restart,
"authority_changed" => Self::AuthorityChanged,
"disable" => Self::Disable,
"crash" => Self::Crash,
"capability_denied" => Self::CapabilityDenied,
"scope_ended" => Self::ScopeEnded,
"scope_carrier_removed" => Self::ScopeCarrierRemoved,
"scope_delegation_changed" => Self::ScopeDelegationChanged,
"scope_parent_ended" => Self::ScopeParentEnded,
other => Self::Unknown(other.to_string()),
}
}
pub fn disposition(&self) -> RouteCloseDisposition {
match self {
Self::Reload | Self::Restart | Self::AuthorityChanged => {
RouteCloseDisposition::MayReopen
}
Self::Disable
| Self::Crash
| Self::CapabilityDenied
| Self::ScopeEnded
| Self::ScopeCarrierRemoved
| Self::ScopeDelegationChanged
| Self::ScopeParentEnded
| Self::Unknown(_) => RouteCloseDisposition::MustNotReopen,
}
}
}
impl ControlPush {
pub fn route_close_reason(&self) -> Option<RouteCloseReason> {
matches!(self.op.as_str(), "route.closing" | "route.closed")
.then(|| {
self.body
.get("reason")?
.as_str()
.map(RouteCloseReason::from_wire)
})
.flatten()
}
}
pub struct SubcConsumer {
shared: Arc<Shared>,
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ConnectionFacts {
local: bool,
operator_app: Option<subc_protocol::peer::OperatorApp>,
}
impl ConnectionFacts {
pub fn local(&self) -> bool {
self.local
}
pub fn operator_app(&self) -> Option<&subc_protocol::peer::OperatorApp> {
self.operator_app.as_ref()
}
}
#[non_exhaustive]
#[derive(Debug)]
pub enum AppOpenError {
Refused { code: String, message: String },
Call(CallError),
}
impl fmt::Display for AppOpenError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Refused { code, message } => write!(f, "{code}: {message}"),
Self::Call(err) => err.fmt(f),
}
}
}
impl Error for AppOpenError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Call(err) => Some(err),
Self::Refused { .. } => None,
}
}
}
impl From<CallError> for AppOpenError {
fn from(err: CallError) -> Self {
Self::Call(err)
}
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CallReply {
body: Vec<u8>,
binary: bool,
}
impl CallReply {
pub fn body(&self) -> &[u8] {
&self.body
}
pub fn into_body(self) -> Vec<u8> {
self.body
}
pub fn is_binary(&self) -> bool {
self.binary
}
}
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub enum SubscriptionEnd {
Response(Vec<u8>),
StreamEnd,
Error(ErrorBody),
RouteClosed {
reason: RouteEndReason,
},
ConnectionLost,
Cancelled,
Backpressure,
}
pub struct Subscription {
events: mpsc::Receiver<Vec<u8>>,
closed: SubscriptionClosed,
ended: watch::Receiver<Option<SubscriptionEnd>>,
cancel: SubscriptionCancel,
}
impl Subscription {
pub fn events(&mut self) -> &mut mpsc::Receiver<Vec<u8>> {
&mut self.events
}
pub fn closed(&mut self) -> &mut SubscriptionClosed {
&mut self.closed
}
pub async fn ended(&mut self) -> SubscriptionEnd {
loop {
if let Some(end) = self.ended.borrow().clone() {
return end;
}
if self.ended.changed().await.is_err() {
return SubscriptionEnd::ConnectionLost;
}
}
}
pub fn unsubscribe(&self) -> Result<(), CallError> {
self.cancel.unsubscribe()
}
}
impl Drop for Subscription {
fn drop(&mut self) {
let _ = self.cancel.unsubscribe();
}
}
pub struct SpawnSubscription {
inner: Subscription,
finished: bool,
}
impl SpawnSubscription {
pub async fn next(&mut self) -> Result<Option<SpawnEvent>, SpawnStreamError> {
if self.finished {
return Ok(None);
}
if let Some(body) = self.inner.events().recv().await {
return serde_json::from_slice::<SpawnEvent>(&body)
.map(Some)
.map_err(|err| {
self.finished = true;
let _ = self.inner.unsubscribe();
SpawnStreamError::Decode(err.to_string())
});
}
let terminal = self.inner.closed().await;
self.finished = true;
match terminal {
Ok(()) => Ok(None),
Err(err) => Err(SpawnStreamError::from_call_error(err)),
}
}
pub fn unsubscribe(&self) -> Result<(), CallError> {
self.inner.unsubscribe()
}
}
#[derive(Debug)]
pub enum SpawnStreamError {
CursorIncarnationMismatch {
current_daemon_incarnation: String,
body: Box<ErrorBody>,
},
CursorTooOld {
oldest_retained_cursor: SpawnCursor,
body: Box<ErrorBody>,
},
SubscriberLagged {
first_undelivered_cursor: SpawnCursor,
body: Box<ErrorBody>,
},
Call(CallError),
Decode(String),
}
impl SpawnStreamError {
fn from_call_error(err: CallError) -> Self {
let CallError::Module(body) = err else {
return Self::Call(err);
};
let detail = |key: &str| body.detail.as_ref().and_then(|detail| detail.get(key));
let cursor = |key: &str| {
detail(key).and_then(|value| serde_json::from_value::<SpawnCursor>(value.clone()).ok())
};
match body.code.as_str() {
SPAWN_CURSOR_INCARNATION_MISMATCH => {
match detail("current_daemon_incarnation").and_then(|value| value.as_str()) {
Some(current) => Self::CursorIncarnationMismatch {
current_daemon_incarnation: current.to_string(),
body: Box::new(body),
},
None => Self::Call(CallError::Module(body)),
}
}
SPAWN_CURSOR_TOO_OLD => match cursor("oldest_retained_cursor") {
Some(oldest_retained_cursor) => Self::CursorTooOld {
oldest_retained_cursor,
body: Box::new(body),
},
None => Self::Call(CallError::Module(body)),
},
SPAWN_SUBSCRIBER_LAGGED => match cursor("first_undelivered_cursor") {
Some(first_undelivered_cursor) => Self::SubscriberLagged {
first_undelivered_cursor,
body: Box::new(body),
},
None => Self::Call(CallError::Module(body)),
},
_ => Self::Call(CallError::Module(body)),
}
}
pub fn code(&self) -> Option<&str> {
match self {
Self::CursorIncarnationMismatch { .. } => Some(SPAWN_CURSOR_INCARNATION_MISMATCH),
Self::CursorTooOld { .. } => Some(SPAWN_CURSOR_TOO_OLD),
Self::SubscriberLagged { .. } => Some(SPAWN_SUBSCRIBER_LAGGED),
Self::Call(err) => err.code(),
Self::Decode(_) => None,
}
}
}
impl fmt::Display for SpawnStreamError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::CursorIncarnationMismatch {
current_daemon_incarnation,
..
} => write!(
f,
"{SPAWN_CURSOR_INCARNATION_MISMATCH}: the daemon is incarnation \
{current_daemon_incarnation}"
),
Self::CursorTooOld {
oldest_retained_cursor,
..
} => write!(
f,
"{SPAWN_CURSOR_TOO_OLD}: the oldest retained event is seq {}",
oldest_retained_cursor.seq
),
Self::SubscriberLagged {
first_undelivered_cursor,
..
} => write!(
f,
"{SPAWN_SUBSCRIBER_LAGGED}: the first undelivered event is seq {}",
first_undelivered_cursor.seq
),
Self::Call(err) => write!(f, "spawn stream ended: {err}"),
Self::Decode(message) => write!(f, "spawn event did not decode: {message}"),
}
}
}
impl Error for SpawnStreamError {}
pub struct SubscriptionClosed {
rx: oneshot::Receiver<Result<(), CallError>>,
}
impl Unpin for SubscriptionClosed {}
impl Future for SubscriptionClosed {
type Output = Result<(), CallError>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let this = self.get_mut();
match Pin::new(&mut this.rx).poll(cx) {
Poll::Ready(Ok(result)) => Poll::Ready(result),
Poll::Ready(Err(_)) => Poll::Ready(Err(CallError::outcome_unknown(
"subscription closed result channel dropped",
OutcomeUnknownCause::CompletionFailed,
))),
Poll::Pending => Poll::Pending,
}
}
}
struct SubscriptionCancel {
shared: Arc<Shared>,
position: Arc<SubscriptionPosition>,
priority: Priority,
cancelled: AtomicBool,
}
impl SubscriptionCancel {
fn new(shared: Arc<Shared>, position: Arc<SubscriptionPosition>, priority: Priority) -> Self {
Self {
shared,
position,
priority,
cancelled: AtomicBool::new(false),
}
}
fn unsubscribe(&self) -> Result<(), CallError> {
let key = *self
.position
.key
.lock()
.unwrap_or_else(|err| err.into_inner());
let handle = RouteHandle::new(key.channel, key.epoch, key.generation);
self.shared.validate_current_handle(handle)?;
if self.cancelled.swap(true, Ordering::AcqRel) {
return Ok(());
}
self.shared
.unsubscribe_subscription(&self.position, self.priority)
}
}
impl SubcConsumer {
pub async fn connect(
connection_file: &Path,
opts: ConsumerOptions,
) -> Result<Self, ConsumerError> {
Self::connect_with_transport(connection_file, opts, ConsumerTransport::Tcp, true).await
}
pub async fn connect_local(
connection_file: &Path,
opts: ConsumerOptions,
) -> Result<Self, ConsumerError> {
Self::connect_with_transport(connection_file, opts, ConsumerTransport::Local, true).await
}
async fn connect_with_transport(
connection_file: &Path,
opts: ConsumerOptions,
transport: ConsumerTransport,
identity_from_environment: bool,
) -> Result<Self, ConsumerError> {
let opened =
open_connection(connection_file, opts.handshake_timeout, transport, None).await?;
let mut shared = Shared::with_transport(connection_file.to_path_buf(), opts, transport);
shared.identity_from_environment = identity_from_environment;
let shared = Arc::new(shared);
shared.install_initial(opened)?;
Ok(Self { shared })
}
pub async fn connect_default(opts: ConsumerOptions) -> Result<Self, ConsumerError> {
let discovered = connection_file::discover(None)
.map_err(|source| ConsumerError::Discovery { source })?;
let opened =
open_connection_with_info(&discovered.path, &discovered.info, opts.handshake_timeout)
.await?;
let shared = Arc::new(Shared::new(discovered.path, opts));
shared.install_initial(opened)?;
Ok(Self { shared })
}
pub async fn connect_default_local(opts: ConsumerOptions) -> Result<Self, ConsumerError> {
let discovered = connection_file::discover(None)
.map_err(|source| ConsumerError::Discovery { source })?;
Self::connect_local(&discovered.path, opts).await
}
pub async fn connect_operator_app(opts: ConsumerOptions) -> Result<Self, ConsumerError> {
Self::connect_operator_app_resolving(
opts,
ConsumerTransport::Local,
crate::operator_app_candidates,
)
.await
}
pub async fn connect_operator_app_plain(opts: ConsumerOptions) -> Result<Self, ConsumerError> {
Self::connect_operator_app_resolving(
opts,
ConsumerTransport::Tcp,
crate::operator_app_candidates,
)
.await
}
async fn connect_operator_app_resolving(
opts: ConsumerOptions,
transport: ConsumerTransport,
resolve: impl FnOnce() -> Result<Vec<PathBuf>, crate::OperatorAppPathError>,
) -> Result<Self, ConsumerError> {
let candidates = resolve().map_err(|source| ConsumerError::ConnectionFile {
path: PathBuf::new(),
source: ConnectionFileError::Io {
op: "resolve installed daemon connection-file path",
path: PathBuf::new(),
source: io::Error::other(source),
},
})?;
let path = candidates[0].clone();
let opened =
open_connection(&path, opts.handshake_timeout, transport, Some(&candidates)).await?;
let mut shared = Shared::with_transport(path, opts, transport);
shared.operator_app_candidates = Some(candidates);
shared.identity_from_environment = false;
let shared = Arc::new(shared);
shared.install_initial(opened)?;
Ok(Self { shared })
}
pub async fn open_route(
&self,
target: RouteTarget,
identity: BindIdentity,
opts: CallOptions,
) -> Result<RouteHandle, CallError> {
opts.reject_scope()?;
let deadline = Instant::now() + opts.timeout;
let consumer_identity = self.shared.route_open_consumer_identity(&opts)?;
let consumer_capabilities = route_open_consumer_capabilities(&opts);
let role_versions = route_open_role_versions(opts.role_versions.as_ref());
let key = RouteKey::new(
&target,
&identity,
consumer_identity.as_ref(),
consumer_capabilities.as_deref(),
)
.with_role_versions(role_versions.as_ref());
let params = RouteOpenParams {
target: &target,
identity: &identity,
consumer_identity: &consumer_identity,
consumer_capabilities: &consumer_capabilities,
role_versions: &role_versions,
reverse_requests: &opts.reverse_requests,
scope: None,
};
self.shared
.ensure_route(&key, ¶ms, &opts, deadline)
.await
.map(|route| route.handle)
}
pub async fn open_route_scoped(
&self,
target: RouteTarget,
identity: BindIdentity,
scope: ScopeSelector,
opts: CallOptions,
) -> Result<RouteHandle, CallError> {
if opts
.scope
.as_ref()
.is_some_and(|selected| selected != &scope)
{
return Err(CallError::not_sent(
"CallOptions.scope differs from open_route_scoped's explicit scope",
));
}
let deadline = Instant::now() + opts.timeout;
let consumer_identity = self.shared.route_open_consumer_identity(&opts)?;
let consumer_capabilities = route_open_consumer_capabilities(&opts);
let role_versions = route_open_role_versions(opts.role_versions.as_ref());
let key = RouteKey::new(
&target,
&identity,
consumer_identity.as_ref(),
consumer_capabilities.as_deref(),
)
.with_role_versions(role_versions.as_ref())
.with_scope(Some(&scope));
let params = RouteOpenParams {
target: &target,
identity: &identity,
consumer_identity: &consumer_identity,
consumer_capabilities: &consumer_capabilities,
role_versions: &role_versions,
reverse_requests: &opts.reverse_requests,
scope: Some(&scope),
};
self.shared
.ensure_route(&key, ¶ms, &opts, deadline)
.await
.map(|route| route.handle)
}
pub async fn open_route_with_admission_facts(
&self,
target: RouteTarget,
identity: BindIdentity,
facts: serde_json::Value,
) -> Result<RouteHandle, CallError> {
let deadline = Instant::now() + self.shared.opts.call_timeout;
self.open_admitted_route(
target,
identity,
facts,
&CallOptions::default(),
None,
deadline,
)
.await
}
pub async fn open_route_with_admission_facts_and_options(
&self,
target: RouteTarget,
identity: BindIdentity,
facts: serde_json::Value,
opts: CallOptions,
) -> Result<RouteHandle, CallError> {
opts.reject_scope()?;
let deadline = Instant::now() + opts.timeout;
let reverse_requests = opts.reverse_requests.clone();
self.open_admitted_route(
target,
identity,
facts,
&opts,
Some(reverse_requests),
deadline,
)
.await
}
async fn open_admitted_route(
&self,
target: RouteTarget,
identity: BindIdentity,
facts: serde_json::Value,
opts: &CallOptions,
reverse_requests: Option<ReverseRequestRegistry>,
deadline: Instant,
) -> Result<RouteHandle, CallError> {
let consumer_capabilities = route_open_consumer_capabilities(opts);
let target_label = route_target_label(&target);
let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
busy_refusals: Some(true),
target,
identity,
consumer_identity: self.shared.route_open_consumer_identity(opts)?,
consumer_capabilities,
role_versions: route_open_role_versions(opts.role_versions.as_ref()),
admission_facts: Some(facts),
scope: None,
})
.map_err(|err| CallError::not_sent(format!("failed to encode route.open: {err}")))?;
let terminal = self
.shared
.paced_route_open(body, deadline, reverse_requests.clone())
.await?;
let (generation, body) = match terminal {
TerminalFrame::Response {
generation, body, ..
} => (generation, body),
TerminalFrame::Error { body, .. } => {
return Err(CallError::route_open_refused(target_label, body));
}
_ => {
return Err(CallError::not_sent_with_cause(
"route.open returned a non-response frame",
NotSentCause::RouteOpenAbandoned,
));
}
};
let ClientControlResponse::RouteOpen {
route_channel,
route_epoch,
request_window,
watch_window,
..
} = decode_route_open_response(&body)
.map_err(|err| {
CallError::not_sent_with_cause(
format!("failed to decode route.open response: {err}"),
NotSentCause::RouteOpenAbandoned,
)
})?
.0
else {
return Err(CallError::not_sent_with_cause(
"route.open returned an unexpected control response",
NotSentCause::RouteOpenAbandoned,
));
};
let handle = self
.shared
.ingress_handle(generation, route_channel, route_epoch)
.unwrap_or_else(|| match reverse_requests {
Some(reverse_requests) => RouteHandle::new_consumer(
route_channel,
route_epoch,
generation,
reverse_requests,
),
None => RouteHandle::new(route_channel, route_epoch, generation),
});
let route = RouteState::windows(handle, request_window, watch_window);
self.shared.install_one_shot_route(route.clone())?;
Ok(route.handle)
}
pub async fn describe_connection(&self) -> Result<Option<ConnectionFacts>, CallError> {
let body =
serde_json::to_vec(&subc_control::ConnectionDescribeRequest::new()).map_err(|err| {
CallError::not_sent(format!("failed to encode connection.describe: {err}"))
})?;
let deadline = Instant::now() + self.shared.opts.call_timeout;
match self
.shared
.control_call(body, deadline, false, None)
.await?
{
TerminalFrame::Response { body, .. } => {
let reply: subc_control::ConnectionDescribeReply = serde_json::from_slice(&body)
.map_err(|err| {
CallError::invalid_response(format!(
"failed to decode connection.describe: {err}"
))
})?;
Ok(Some(ConnectionFacts {
local: reply.local,
operator_app: reply.operator_app,
}))
}
TerminalFrame::Error { body, .. } if body.code == "unknown_control_op" => Ok(None),
TerminalFrame::Error { body, .. } => Err(CallError::Module(body)),
TerminalFrame::StreamEnd => Err(CallError::invalid_response(
"connection.describe returned StreamEnd",
)),
}
}
pub async fn app_open(&self, app: &str) -> Result<subc_control::AppOpenReply, AppOpenError> {
let body = serde_json::to_vec(&subc_control::AppOpenRequest::new(app))
.map_err(|err| CallError::not_sent(format!("failed to encode app.open: {err}")))?;
let deadline = Instant::now() + self.shared.opts.call_timeout;
match self
.shared
.control_call(body, deadline, false, None)
.await?
{
TerminalFrame::Response { body, .. } => serde_json::from_slice(&body).map_err(|err| {
CallError::invalid_response(format!("failed to decode app.open: {err}")).into()
}),
TerminalFrame::Error { body, .. } => Err(AppOpenError::Refused {
code: body.code,
message: body.message,
}),
TerminalFrame::StreamEnd => {
Err(CallError::invalid_response("app.open returned StreamEnd").into())
}
}
}
pub async fn catalog_list(&self) -> Result<CatalogList, CallError> {
let deadline = Instant::now() + self.shared.opts.call_timeout;
let body = serde_json::to_vec(&serde_json::json!({
"op": subc_control::ops::CATALOG_LIST,
}))
.map_err(|err| CallError::not_sent(format!("failed to encode catalog.list: {err}")))?;
let mut attempt = 0usize;
loop {
attempt = attempt.saturating_add(1);
match self
.shared
.control_call(body.clone(), deadline, false, None)
.await
{
Ok(TerminalFrame::Response { body, .. }) => {
let response =
serde_json::from_slice::<ClientControlResponse>(&body).map_err(|err| {
CallError::invalid_response(format!(
"failed to decode catalog.list response: {err}"
))
})?;
let ClientControlResponse::CatalogList {
generation,
modules,
subc_ops,
} = response
else {
return Err(CallError::invalid_response(
"catalog.list returned an unexpected control response",
));
};
return Ok(CatalogList {
generation,
modules,
subc_ops,
});
}
Ok(TerminalFrame::Error { body, .. }) => return Err(CallError::Module(body)),
Ok(TerminalFrame::StreamEnd) => {
return Err(CallError::invalid_response(
"catalog.list returned StreamEnd",
));
}
Err(err)
if is_retryable_catalog_transport_error(&err) && Instant::now() < deadline =>
{
let delay = self.shared.jittered(
self.shared
.opts
.reconnect_backoff
.delay_after_attempt(attempt),
);
self.shared.sleep_until_retry(deadline, delay).await?;
continue;
}
Err(err) => return Err(err),
}
}
}
pub async fn supervisor_list(&self) -> Result<SupervisorListResponse, CallError> {
let deadline = Instant::now() + self.shared.opts.call_timeout;
let body = serde_json::to_vec(&ClientControlRequest::SupervisorList {}).map_err(|err| {
CallError::not_sent(format!("failed to encode supervisor.list: {err}"))
})?;
let mut attempt = 0usize;
loop {
attempt = attempt.saturating_add(1);
match self
.shared
.control_call(body.clone(), deadline, false, None)
.await
{
Ok(TerminalFrame::Response { body, .. }) => {
let response =
serde_json::from_slice::<ClientControlResponse>(&body).map_err(|err| {
CallError::invalid_response(format!(
"failed to decode supervisor.list response: {err}"
))
})?;
let ClientControlResponse::SupervisorList {
generation,
modules,
daemon,
agent_commands,
} = response
else {
return Err(CallError::invalid_response(
"supervisor.list returned an unexpected control response",
));
};
return Ok(SupervisorListResponse {
generation,
modules,
daemon,
agent_commands,
});
}
Ok(TerminalFrame::Error { body, .. }) => return Err(CallError::Module(body)),
Ok(TerminalFrame::StreamEnd) => {
return Err(CallError::invalid_response(
"supervisor.list returned StreamEnd",
));
}
Err(err)
if is_retryable_catalog_transport_error(&err) && Instant::now() < deadline =>
{
let delay = self.shared.jittered(
self.shared
.opts
.reconnect_backoff
.delay_after_attempt(attempt),
);
self.shared.sleep_until_retry(deadline, delay).await?;
continue;
}
Err(err) => return Err(err),
}
}
}
pub async fn spawn_snapshot(&self) -> Result<SpawnSnapshot, CallError> {
let deadline = Instant::now() + self.shared.opts.call_timeout;
let body = serde_json::to_vec(&ClientControlRequest::SupervisorSpawnSnapshot {}).map_err(
|err| CallError::not_sent(format!("failed to encode supervisor.spawn_snapshot: {err}")),
)?;
let mut attempt = 0usize;
loop {
attempt = attempt.saturating_add(1);
match self
.shared
.control_call(body.clone(), deadline, false, None)
.await
{
Ok(TerminalFrame::Response { body, .. }) => {
let response =
serde_json::from_slice::<ClientControlResponse>(&body).map_err(|err| {
CallError::invalid_response(format!(
"failed to decode supervisor.spawn_snapshot response: {err}"
))
})?;
let ClientControlResponse::SupervisorSpawnSnapshot { snapshot } = response
else {
return Err(CallError::invalid_response(
"supervisor.spawn_snapshot returned an unexpected control response",
));
};
return Ok(snapshot);
}
Ok(TerminalFrame::Error { body, .. }) => return Err(CallError::Module(body)),
Ok(TerminalFrame::StreamEnd) => {
return Err(CallError::invalid_response(
"supervisor.spawn_snapshot returned StreamEnd",
));
}
Err(err)
if is_retryable_catalog_transport_error(&err) && Instant::now() < deadline =>
{
let delay = self.shared.jittered(
self.shared
.opts
.reconnect_backoff
.delay_after_attempt(attempt),
);
self.shared.sleep_until_retry(deadline, delay).await?;
continue;
}
Err(err) => return Err(err),
}
}
}
pub async fn query_activity(
&self,
request: ActivityQueryRequest,
) -> Result<ActivityQueryResponse, CallError> {
let deadline = Instant::now() + self.shared.opts.call_timeout;
let body = self.shared.activity_read_body(request)?;
let mut attempt = 0usize;
loop {
attempt = attempt.saturating_add(1);
match self
.shared
.control_call(body.clone(), deadline, false, None)
.await
{
Ok(TerminalFrame::Response { body, .. }) => {
return serde_json::from_slice(&body).map_err(|err| {
CallError::invalid_response(format!(
"invalid activity.query response: {err}"
))
});
}
Ok(TerminalFrame::Error { body, .. }) => return Err(CallError::Module(body)),
Ok(TerminalFrame::StreamEnd) => {
return Err(CallError::invalid_response(
"activity.query returned StreamEnd",
));
}
Err(err)
if is_retryable_catalog_transport_error(&err) && Instant::now() < deadline =>
{
let delay = self.shared.jittered(
self.shared
.opts
.reconnect_backoff
.delay_after_attempt(attempt),
);
self.shared.sleep_until_retry(deadline, delay).await?;
}
Err(err) => return Err(err),
}
}
}
pub async fn subscribe_activity(
&self,
since: Option<ActivityCursor>,
) -> Result<ActivitySubscription, CallError> {
let deadline = Instant::now() + self.shared.opts.call_timeout;
let body = self
.shared
.activity_read_body(ActivitySubscribeRequest { since })?;
loop {
self.shared.ensure_connected_for_call(deadline).await?;
let permit = Arc::new(Semaphore::new(1))
.acquire_owned()
.await
.map_err(|_| CallError::not_sent("activity subscription permit closed"))?;
match self
.shared
.send_subscription(SubscriptionSend {
expected_handle: None,
channel: 0,
epoch: 0,
body: body.clone(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
event_buffer: ACTIVITY_EVENT_BUFFER,
deadline,
permit: RoutePermit::legacy(permit),
})
.await
{
Ok(inner) => {
return Ok(ActivitySubscription {
inner,
finished: false,
})
}
Err(err) if err.is_not_sent() && Instant::now() < deadline => continue,
Err(err) => return Err(err),
}
}
}
pub async fn spawn_subscribe(
&self,
since: Option<SpawnCursor>,
) -> Result<SpawnSubscription, CallError> {
let deadline = Instant::now() + self.shared.opts.call_timeout;
let body = serde_json::to_vec(&ClientControlRequest::SupervisorSpawnSubscribe { since })
.map_err(|err| {
CallError::not_sent(format!(
"failed to encode supervisor.spawn_subscribe: {err}"
))
})?;
loop {
self.shared.ensure_connected_for_call(deadline).await?;
let permit = Arc::new(Semaphore::new(1))
.acquire_owned()
.await
.map_err(|_| CallError::not_sent("spawn subscription permit closed"))?;
match self
.shared
.send_subscription(SubscriptionSend {
expected_handle: None,
channel: 0,
epoch: 0,
body: body.clone(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
event_buffer: SPAWN_EVENT_BUFFER,
deadline,
permit: RoutePermit::legacy(permit),
})
.await
{
Ok(inner) => {
return Ok(SpawnSubscription {
inner,
finished: false,
})
}
Err(err) if err.is_not_sent() && Instant::now() < deadline => continue,
Err(err) => return Err(err),
}
}
}
pub async fn resolve_provider(&self, capability: &str) -> Result<String, CallError> {
let claimants = self.resolve_providers(capability).await?;
match claimants.as_slice() {
[] => Err(CallError::CapabilityUnprovided {
capability: capability.to_string(),
}),
[claimant] => Ok(claimant.clone()),
_ => Err(CallError::CapabilityAmbiguous {
capability: capability.to_string(),
claimants,
}),
}
}
pub async fn resolve_providers(&self, capability: &str) -> Result<Vec<String>, CallError> {
validate_capability_for_resolution(capability)?;
let catalog = self.catalog_list().await?;
Ok(capability_claimants(&catalog, capability))
}
pub async fn request(
&self,
handle: &RouteHandle,
body: Vec<u8>,
opts: CallOptions,
) -> Result<Vec<u8>, CallError> {
self.request_inner(handle, body, opts, None).await
}
pub fn request_cancellable(
&self,
handle: &RouteHandle,
body: Vec<u8>,
opts: CallOptions,
) -> (
CancelHandle,
impl Future<Output = Result<Vec<u8>, CallError>> + Send + '_,
) {
let consumer = self;
let handle = *handle;
let (cancel, registration) = CancelRegistration::new(Arc::clone(&self.shared));
let future = async move {
let guard = registration;
consumer
.request_inner(&handle, body, opts, Some(Arc::clone(&guard.state)))
.await
};
(cancel, future)
}
async fn request_inner(
&self,
handle: &RouteHandle,
body: Vec<u8>,
opts: CallOptions,
cancel: Option<Arc<UnaryCancel>>,
) -> Result<Vec<u8>, CallError> {
opts.reject_scope()?;
let deadline = Instant::now() + opts.timeout;
let route = self.shared.route_state(*handle)?;
let permit = route.acquire(false, deadline, cancel.as_ref()).await?;
let mut busy_delay = Duration::from_millis(25);
loop {
let result = self
.shared
.send_request_with_slot(
RequestSend {
binary: false,
expected_handle: Some(*handle),
channel: handle.channel,
epoch: handle.epoch,
body: body.clone(),
priority: opts.priority,
admission_class: opts.admission_class,
deadline,
retain_late_route_open: false,
route_open_reverse_requests: None,
},
cancel.clone(),
Some(Arc::clone(&permit)),
)
.await;
match result? {
TerminalFrame::Response { body, .. } => return Ok(body),
TerminalFrame::StreamEnd => return Ok(Vec::new()),
TerminalFrame::Error { body, flags }
if error_codes::is_route_busy_retryable(flags, &body.code, route.opted_in) =>
{
if !self
.shared
.busy_backoff(deadline, busy_delay, cancel.as_ref())
.await
{
return Err(busy_error(&body));
}
busy_delay = (busy_delay * 2).min(Duration::from_secs(1));
self.shared.validate_current_handle(*handle)?;
}
TerminalFrame::Error { body, .. } => return Err(CallError::Module(body)),
}
}
}
pub async fn subscribe_route(
&self,
handle: &RouteHandle,
body: Vec<u8>,
opts: SubscribeOptions,
) -> Result<Subscription, CallError> {
if opts.scope.is_some() {
return Err(CallError::not_sent(
"SubscribeOptions.scope is not supported on subscribe_route; select the scope when opening the route",
));
}
let deadline = Instant::now() + opts.route_open_timeout;
let route = self.shared.route_state(*handle)?;
let permit = route.acquire(true, deadline, None).await?;
self.shared
.send_subscription(SubscriptionSend {
expected_handle: Some(*handle),
channel: handle.channel,
epoch: handle.epoch,
body,
priority: opts.priority,
admission_class: opts.admission_class,
event_buffer: opts.event_buffer,
deadline,
permit,
})
.await
}
pub async fn poll_route(
&self,
handle: &RouteHandle,
kind: PollKind,
timeout: Duration,
) -> Result<RoutePollResult, CallError> {
let deadline = Instant::now() + timeout;
let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
route_channel: handle.channel,
route_epoch: handle.epoch,
kind,
})
.map_err(|err| CallError::not_sent(format!("failed to encode route.poll: {err}")))?;
let terminal = self
.shared
.send_request(RequestSend {
binary: false,
expected_handle: Some(*handle),
channel: 0,
epoch: 0,
body,
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline,
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await?;
let body = match terminal {
TerminalFrame::Response { body, .. } => body,
TerminalFrame::Error { body, .. } => return Err(CallError::Module(body)),
TerminalFrame::StreamEnd => {
return Err(CallError::invalid_response(
"route.poll returned a non-response frame",
));
}
};
let ClientControlResponse::RoutePoll {
route_channel,
route_epoch,
status,
live,
} = serde_json::from_slice(&body).map_err(|err| {
CallError::invalid_response(format!("failed to decode route.poll: {err}"))
})?
else {
return Err(CallError::invalid_response(
"route.poll returned an unexpected control response",
));
};
if route_channel != handle.channel || route_epoch != handle.epoch {
return Err(CallError::invalid_response(
"route.poll response echoed a different route handle",
));
}
self.shared.validate_current_handle(*handle)?;
Ok(RoutePollResult {
handle: *handle,
status,
live,
})
}
pub fn dropped_route_frames(&self) -> u64 {
self.shared.lock_inner().dropped_route_frames
}
pub fn push_events(
&self,
handle: &RouteHandle,
) -> Result<mpsc::Receiver<PushEvent>, CallError> {
self.shared.register_push_events(*handle)
}
pub fn pushes_dropped_no_receiver(&self) -> u64 {
self.shared
.pushes_dropped_no_receiver
.load(Ordering::Relaxed)
}
pub fn pushes_dropped_receiver_full(&self) -> u64 {
self.shared
.pushes_dropped_receiver_full
.load(Ordering::Relaxed)
}
pub fn control_pushes(&self, capacity: usize) -> mpsc::Receiver<ControlPush> {
let (sender, receiver) = mpsc::channel(capacity.max(1));
self.shared.lock_inner().control_push_receiver = Some(sender);
receiver
}
pub fn control_pushes_dropped(&self) -> u64 {
self.shared.control_pushes_dropped.load(Ordering::Relaxed)
}
pub async fn call(
&self,
target: RouteTarget,
identity: BindIdentity,
body: Vec<u8>,
opts: CallOptions,
) -> Result<Vec<u8>, CallError> {
self.call_with_reply(target, identity, body, opts)
.await
.map(CallReply::into_body)
}
pub async fn call_with_reply(
&self,
target: RouteTarget,
identity: BindIdentity,
body: Vec<u8>,
opts: CallOptions,
) -> Result<CallReply, CallError> {
self.call_inner(target, identity, (body, false), opts, None)
.await
}
pub fn call_cancellable(
&self,
target: RouteTarget,
identity: BindIdentity,
body: Vec<u8>,
opts: CallOptions,
) -> (
CancelHandle,
impl Future<Output = Result<Vec<u8>, CallError>> + Send + '_,
) {
let (cancel, reply) = self.call_cancellable_with_reply(target, identity, body, opts);
(cancel, async move { reply.await.map(CallReply::into_body) })
}
pub fn call_cancellable_with_reply(
&self,
target: RouteTarget,
identity: BindIdentity,
body: Vec<u8>,
opts: CallOptions,
) -> (
CancelHandle,
impl Future<Output = Result<CallReply, CallError>> + Send + '_,
) {
let consumer = self;
let (cancel, registration) = CancelRegistration::new(Arc::clone(&self.shared));
let future = async move {
let guard = registration;
consumer
.call_inner(
target,
identity,
(body, false),
opts,
Some(Arc::clone(&guard.state)),
)
.await
};
(cancel, future)
}
pub async fn call_framed_with_reply(
&self,
target: RouteTarget,
identity: BindIdentity,
head: serde_json::Value,
payload: Vec<u8>,
opts: CallOptions,
) -> Result<CallReply, CallError> {
let body = subc_protocol::framed_body::encode(&head, &payload)
.map_err(|error| CallError::not_sent(format!("invalid framed request: {error}")))?;
self.call_inner(target, identity, (body, true), opts, None)
.await
}
pub fn call_framed_cancellable_with_reply(
&self,
target: RouteTarget,
identity: BindIdentity,
head: serde_json::Value,
payload: Vec<u8>,
opts: CallOptions,
) -> (
CancelHandle,
impl Future<Output = Result<CallReply, CallError>> + Send + '_,
) {
let (cancel, registration) = CancelRegistration::new(Arc::clone(&self.shared));
(cancel, async move {
let guard = registration;
let body = subc_protocol::framed_body::encode(&head, &payload)
.map_err(|error| CallError::not_sent(format!("invalid framed request: {error}")))?;
self.call_inner(
target,
identity,
(body, true),
opts,
Some(Arc::clone(&guard.state)),
)
.await
})
}
async fn call_inner(
&self,
target: RouteTarget,
identity: BindIdentity,
body: (Vec<u8>, bool),
opts: CallOptions,
cancel: Option<Arc<UnaryCancel>>,
) -> Result<CallReply, CallError> {
let mut history = CallAttemptHistory {
never_forwarded: true,
sent: false,
};
self.call_attempts(target, identity, body, opts, cancel, &mut history)
.await
.map_err(|err| {
let err = if history.sent && matches!(err, CallError::StaleRouteHandle(_)) {
CallError::not_sent(err.to_string())
} else {
err
};
err.with_never_forwarded(history.never_forwarded)
})
}
async fn call_attempts(
&self,
target: RouteTarget,
identity: BindIdentity,
body: (Vec<u8>, bool),
opts: CallOptions,
cancel: Option<Arc<UnaryCancel>>,
history: &mut CallAttemptHistory,
) -> Result<CallReply, CallError> {
let (body, binary) = body;
let call_deadline = Instant::now() + opts.timeout;
let mut resent_after_unsent_refusal = false;
let consumer_identity = self.shared.route_open_consumer_identity(&opts)?;
let consumer_capabilities = route_open_consumer_capabilities(&opts);
let role_versions = route_open_role_versions(opts.role_versions.as_ref());
let route_key = RouteKey::new(
&target,
&identity,
consumer_identity.as_ref(),
consumer_capabilities.as_deref(),
)
.with_role_versions(role_versions.as_ref())
.with_scope(opts.scope.as_ref());
let route_open = RouteOpenParams {
target: &target,
identity: &identity,
consumer_identity: &consumer_identity,
consumer_capabilities: &consumer_capabilities,
role_versions: &role_versions,
reverse_requests: &opts.reverse_requests,
scope: opts.scope.as_ref(),
};
let mut busy_permit: Option<Arc<RoutePermit>> = None;
let mut busy_delay = Duration::from_millis(25);
loop {
let route = self
.shared
.ensure_route(&route_key, &route_open, &opts, call_deadline)
.await
.map_err(request_not_sent_after_route_open_failure)?;
if busy_permit
.as_ref()
.is_some_and(|permit| !permit.belongs_to(&route))
{
busy_permit.take();
}
let permit = match busy_permit.take() {
Some(permit) => permit,
None => route
.acquire(false, call_deadline, cancel.as_ref())
.await
.map_err(|err| {
if matches!(err, CallError::StaleRouteHandle(_)) {
self.shared.route_not_sent_error(route.handle)
} else {
err
}
})?,
};
if !self.shared.route_is_current(&route_key, &route) {
drop(permit);
self.shared
.sleep_until_retry(call_deadline, opts.route_retry.base)
.await?;
continue;
}
let daemon_origin_trusted = {
let inner = self.shared.lock_inner();
inner.generation == route.handle.connection_token() && inner.daemon_origin_trusted
};
let response = self
.shared
.send_request_with_slot(
RequestSend {
expected_handle: Some(route.handle),
channel: route.handle.channel,
epoch: route.handle.epoch,
body: body.clone(),
binary,
priority: opts.priority,
admission_class: opts.admission_class,
deadline: call_deadline,
retain_late_route_open: false,
route_open_reverse_requests: None,
},
cancel.clone(),
Some(Arc::clone(&permit)),
)
.await;
history.sent |= response.is_ok();
let proven_undelivered = match &response {
Ok(TerminalFrame::Error { body, flags }) => {
is_refusal_on_route(daemon_origin_trusted, *flags, &body.code, route.opted_in)
}
Err(err) => {
err.never_forwarded() == Some(true)
|| matches!(err, CallError::StaleRouteHandle(_))
}
_ => false,
};
history.never_forwarded &= proven_undelivered;
match response {
Ok(TerminalFrame::Response { body, flags, .. }) => {
return Ok(CallReply {
body,
binary: flags.is_binary(),
});
}
Ok(TerminalFrame::StreamEnd) => {
return Ok(CallReply {
body: Vec::new(),
binary: false,
});
}
Ok(TerminalFrame::Error { body, flags })
if error_codes::is_route_busy_retryable(flags, &body.code, route.opted_in) =>
{
if !self
.shared
.busy_backoff(call_deadline, busy_delay, cancel.as_ref())
.await
{
return Err(busy_error(&body));
}
busy_delay = (busy_delay * 2).min(Duration::from_secs(1));
busy_permit = Some(permit);
continue;
}
Ok(TerminalFrame::Error { body, flags })
if (error_codes::is_established_route_dead(flags, &body.code)
|| is_data_plane_drain_refusal(&body.code))
&& !resent_after_unsent_refusal
&& !cancel.as_ref().is_some_and(|cancel| cancel.requested())
&& Instant::now() < call_deadline =>
{
resent_after_unsent_refusal = true;
self.shared.invalidate_route(&route_key, Some(route.handle));
continue;
}
Ok(TerminalFrame::Error { body, flags })
if is_daemon_refusal_before_forward(
daemon_origin_trusted,
flags,
&body.code,
) =>
{
return Err(CallError::not_sent_with_cause(
format!(
"daemon refused before forwarding: {} ({})",
body.code, body.message
),
NotSentCause::DaemonRefusedBeforeForward { code: body.code },
));
}
Ok(TerminalFrame::Error { body, .. }) => return Err(CallError::Module(body)),
Err(err)
if err.is_not_sent()
&& Instant::now() < call_deadline
&& !cancel.as_ref().is_some_and(|cancel| cancel.requested()) =>
{
self.shared.invalidate_route(&route_key, Some(route.handle));
self.shared.ensure_connected_for_call(call_deadline).await?;
continue;
}
Err(err)
if err.outcome_cause() == Some(OutcomeUnknownCause::Deadline)
&& !cancel.as_ref().is_some_and(|cancel| cancel.requested()) =>
{
self.shared.invalidate_route_with_deadline(
&route_key,
Some(route.handle),
true,
);
return Err(err);
}
Err(err) => return Err(err),
}
}
}
pub async fn subscribe(
&self,
target: RouteTarget,
identity: BindIdentity,
body: Vec<u8>,
opts: SubscribeOptions,
) -> Result<Subscription, CallError> {
let open_deadline = Instant::now() + opts.route_open_timeout;
let route_opts = CallOptions {
scope: opts.scope.clone(),
timeout: opts.route_open_timeout,
priority: opts.priority,
admission_class: opts.admission_class,
route_retry: opts.route_retry,
route_retry_deadline: opts.route_retry_deadline,
consumer_identity: opts.consumer_identity.clone(),
reverse_requests: opts.reverse_requests.clone(),
role_versions: opts.role_versions.clone(),
};
let consumer_identity = self.shared.route_open_consumer_identity(&route_opts)?;
let consumer_capabilities = route_open_consumer_capabilities(&route_opts);
let role_versions = route_open_role_versions(route_opts.role_versions.as_ref());
let route_key = RouteKey::new(
&target,
&identity,
consumer_identity.as_ref(),
consumer_capabilities.as_deref(),
)
.with_role_versions(role_versions.as_ref())
.with_scope(opts.scope.as_ref());
let route_open = RouteOpenParams {
target: &target,
identity: &identity,
consumer_identity: &consumer_identity,
consumer_capabilities: &consumer_capabilities,
role_versions: &role_versions,
reverse_requests: &opts.reverse_requests,
scope: opts.scope.as_ref(),
};
loop {
let route = self
.shared
.ensure_route(&route_key, &route_open, &route_opts, open_deadline)
.await
.map_err(request_not_sent_after_route_open_failure)?;
let permit = route
.acquire(true, open_deadline, None)
.await
.map_err(|err| {
if matches!(err, CallError::StaleRouteHandle(_)) {
self.shared.route_not_sent_error(route.handle)
} else {
err
}
})?;
if !self.shared.route_is_current(&route_key, &route) {
drop(permit);
self.shared
.sleep_until_retry(open_deadline, opts.route_retry.base)
.await?;
continue;
}
match self
.shared
.send_subscription(SubscriptionSend {
expected_handle: Some(route.handle),
channel: route.handle.channel,
epoch: route.handle.epoch,
body: body.clone(),
priority: opts.priority,
admission_class: opts.admission_class,
event_buffer: opts.event_buffer,
deadline: open_deadline,
permit,
})
.await
{
Ok(subscription) => return Ok(subscription),
Err(err) if err.is_not_sent() && Instant::now() < open_deadline => {
self.shared.invalidate_route(&route_key, Some(route.handle));
self.shared.ensure_connected_for_call(open_deadline).await?;
continue;
}
Err(err) => return Err(err),
}
}
}
pub async fn close_route(
&self,
target: RouteTarget,
identity: BindIdentity,
opts: CloseRouteOptions,
) {
let Ok(consumer_identity) = self.shared.close_route_consumer_identity(&opts) else {
return;
};
let consumer_capabilities = close_route_consumer_capabilities(&opts);
let key = RouteKey::new(
&target,
&identity,
consumer_identity.as_ref(),
consumer_capabilities.as_deref(),
)
.with_role_versions(route_open_role_versions(opts.role_versions.as_ref()).as_ref())
.with_scope(opts.scope.as_ref());
self.shared.close_route(&key, &opts).await;
}
pub async fn close_handle(
&self,
handle: &RouteHandle,
opts: CloseRouteOptions,
) -> Result<(), CallError> {
if opts.scope.is_some() {
return Err(CallError::not_sent(
"CloseRouteOptions.scope is not supported on close_handle; the handle already selects the route",
));
}
self.shared.close_handle(*handle, &opts).await
}
pub fn current_epoch(&self) -> u64 {
self.shared.lock_inner().epoch
}
pub(crate) fn pool_reusable(&self) -> bool {
!self.shared.close_token.is_cancelled()
}
pub fn on_connection_state(&self, cb: impl Fn(ConnectionState) + Send + 'static) {
self.shared
.lock_inner()
.callbacks
.push(Arc::new(Mutex::new(Box::new(cb))));
}
pub async fn close(&self) {
self.shared.close_sync("consumer closed");
tokio::task::yield_now().await;
}
}
impl Drop for SubcConsumer {
fn drop(&mut self) {
self.shared.close_sync("consumer dropped");
}
}
#[derive(Debug)]
pub enum ConsumerError {
Discovery {
source: connection_file::DiscoveryError,
},
ConnectionFile {
path: PathBuf,
source: ConnectionFileError,
},
NoEndpoint {
path: PathBuf,
},
Connect {
path: PathBuf,
endpoint: String,
source: io::Error,
},
Auth {
path: PathBuf,
endpoint: String,
source: AuthError,
},
Closed,
}
impl ConsumerError {
pub(crate) fn claim(error: crate::ClaimError) -> Self {
Self::Connect {
path: PathBuf::new(),
endpoint: "claim binding".into(),
source: io::Error::other(error),
}
}
}
impl fmt::Display for ConsumerError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Discovery { source } => source.fmt(f),
Self::ConnectionFile { path, source } => write!(
f,
"failed to read subc connection file '{}': {source}",
path.display()
),
Self::NoEndpoint { path } => {
write!(
f,
"subc connection file '{}' has no endpoint for the requested transport",
path.display()
)
}
Self::Connect {
path,
endpoint,
source,
} => write!(
f,
"failed to connect to subc endpoint {endpoint} from '{}': {source}",
path.display()
),
Self::Auth {
path,
endpoint,
source,
} => write!(
f,
"failed to authenticate to subc endpoint {endpoint} from '{}': {source}",
path.display()
),
Self::Closed => write!(f, "consumer closed"),
}
}
}
impl Error for ConsumerError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Discovery { source } => Some(source),
Self::ConnectionFile { source, .. } => Some(source),
Self::Connect { source, .. } => Some(source),
Self::Auth { source, .. } => Some(source),
Self::NoEndpoint { .. } | Self::Closed => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum OutcomeUnknownCause {
Deadline,
WriterClosed,
ConsumerClosed,
ConnectionFailed,
RouteEnded,
CompletionFailed,
}
#[derive(Debug, Clone)]
struct OutcomeUnknownSource {
message: String,
cause: OutcomeUnknownCause,
route_reason: Option<RouteEnd>,
}
impl fmt::Display for OutcomeUnknownSource {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.message)
}
}
impl Error for OutcomeUnknownSource {}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum NotSentCause {
RouteOpenRefused { code: String },
RouteOpenAbandoned,
BeforeQueue,
WriterCapacity,
WriterRejected,
InvalidResponse,
CompletionFailed,
DaemonRefusedBeforeForward { code: String },
}
#[derive(Debug, Clone)]
struct NotSentSource {
message: String,
cause: NotSentCause,
never_forwarded: bool,
route_reason: Option<RouteEnd>,
refusal: Option<Box<RouteOpenRefused>>,
}
impl fmt::Display for NotSentSource {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.message)
}
}
impl Error for NotSentSource {}
#[derive(Debug)]
pub enum CallError {
DaemonTooOld { daemon_ver: String },
NotSent(Box<dyn Error + Send + Sync>),
OutcomeUnknown(Box<dyn Error + Send + Sync>),
Module(ErrorBody),
SubscriptionBackpressure(Box<dyn Error + Send + Sync>),
StaleRouteHandle(RouteHandle),
CapabilityUnprovided { capability: String },
CapabilityAmbiguous {
capability: String,
claimants: Vec<String>,
},
InvalidCapabilityIdentifier { capability: String },
}
impl CallError {
pub fn scope_bind_proof_refusal(&self) -> Option<ScopeBindProofRefusal> {
match self.route_open_refusal()?.code.as_str() {
error_codes::SCOPE_BIND_PROOF_REQUIRED => Some(ScopeBindProofRefusal::Required),
error_codes::SCOPE_BIND_PROOF_MISMATCH => Some(ScopeBindProofRefusal::Mismatch),
_ => None,
}
}
pub fn close_reason(&self) -> Option<&RouteEndReason> {
self.route_end().map(|end| &end.reason)
}
fn route_end(&self) -> Option<&RouteEnd> {
match self {
Self::NotSent(source) => source
.downcast_ref::<NotSentSource>()?
.route_reason
.as_ref(),
Self::OutcomeUnknown(source) => source
.downcast_ref::<OutcomeUnknownSource>()?
.route_reason
.as_ref(),
_ => None,
}
}
pub fn terminal(&self) -> Option<bool> {
self.route_end()?.terminal
}
pub fn outcome_cause(&self) -> Option<OutcomeUnknownCause> {
match self {
Self::OutcomeUnknown(source) => source
.downcast_ref::<OutcomeUnknownSource>()
.map(|source| source.cause),
_ => None,
}
}
pub fn not_sent_cause(&self) -> Option<NotSentCause> {
match self {
Self::NotSent(source) => source
.downcast_ref::<NotSentSource>()
.map(|source| source.cause.clone()),
_ => None,
}
}
pub fn never_forwarded(&self) -> Option<bool> {
match self {
Self::NotSent(source) => source
.downcast_ref::<NotSentSource>()
.map(|source| source.never_forwarded),
_ => None,
}
}
pub fn code(&self) -> Option<&str> {
match self {
Self::Module(body) => Some(&body.code),
Self::DaemonTooOld { .. } => Some("daemon_too_old"),
Self::CapabilityUnprovided { .. } => Some("capability_unprovided"),
Self::CapabilityAmbiguous { .. } => Some("capability_ambiguous"),
Self::InvalidCapabilityIdentifier { .. } => Some("invalid_capability_identifier"),
Self::NotSent(source) => match &source.downcast_ref::<NotSentSource>()?.cause {
NotSentCause::DaemonRefusedBeforeForward { code }
if code == error_codes::ROUTE_BUSY =>
{
Some(code)
}
_ => None,
},
Self::OutcomeUnknown(_)
| Self::SubscriptionBackpressure(_)
| Self::StaleRouteHandle(_) => None,
}
}
fn not_sent(reason: impl Into<String>) -> Self {
Self::not_sent_with_cause(reason, NotSentCause::BeforeQueue)
}
fn not_sent_with_cause(reason: impl Into<String>, cause: NotSentCause) -> Self {
Self::NotSent(Box::new(NotSentSource {
message: reason.into(),
cause,
never_forwarded: true,
route_reason: None,
refusal: None,
}))
}
fn with_never_forwarded(mut self, never_forwarded: bool) -> Self {
if let Self::NotSent(source) = &mut self {
if let Some(source) = source.downcast_mut::<NotSentSource>() {
source.never_forwarded &= never_forwarded;
}
}
self
}
fn invalid_response(reason: impl Into<String>) -> Self {
Self::not_sent_with_cause(reason, NotSentCause::InvalidResponse).with_never_forwarded(false)
}
fn outcome_unknown(reason: impl Into<String>, cause: OutcomeUnknownCause) -> Self {
Self::outcome_unknown_source(OutcomeUnknownSource {
message: reason.into(),
cause,
route_reason: None,
})
}
fn outcome_unknown_source(source: OutcomeUnknownSource) -> Self {
Self::OutcomeUnknown(Box::new(source))
}
fn is_not_sent(&self) -> bool {
matches!(self, Self::NotSent(_))
}
pub fn route_open_refusal(&self) -> Option<&ErrorBody> {
match self {
Self::NotSent(err) => err
.downcast_ref::<NotSentSource>()
.and_then(|source| source.refusal.as_ref())
.map(|refused| &refused.body),
_ => None,
}
}
fn subscription_backpressure(reason: impl Into<String>) -> Self {
Self::SubscriptionBackpressure(Box::new(SimpleError(reason.into())))
}
}
impl fmt::Display for CallError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::DaemonTooOld { daemon_ver } => write!(
f,
"daemon_too_old: route.open requires daemon 0.39.0 or newer (connected version: {daemon_ver})"
),
Self::NotSent(err) => write!(f, "request not sent: {err}"),
Self::OutcomeUnknown(err) => write!(f, "request outcome unknown: {err}"),
Self::Module(body) => write!(f, "module error {}: {}", body.code, body.message),
Self::SubscriptionBackpressure(err) => {
write!(f, "subscription event channel backpressure: {err}")
}
Self::StaleRouteHandle(handle) => write!(f, "stale route handle: {handle:?}"),
Self::CapabilityUnprovided { capability } => {
write!(
f,
"capability_unprovided: no catalog claimant for {capability}"
)
}
Self::CapabilityAmbiguous {
capability,
claimants,
} => write!(
f,
"capability_ambiguous: multiple catalog claimants for {capability}: {claimants:?}"
),
Self::InvalidCapabilityIdentifier { capability } => write!(
f,
"invalid_capability_identifier: malformed capability identifier {capability:?}"
),
}
}
}
impl Error for CallError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::NotSent(err)
| Self::OutcomeUnknown(err)
| Self::SubscriptionBackpressure(err) => Some(err.as_ref()),
Self::Module(_)
| Self::DaemonTooOld { .. }
| Self::StaleRouteHandle(_)
| Self::CapabilityUnprovided { .. }
| Self::CapabilityAmbiguous { .. }
| Self::InvalidCapabilityIdentifier { .. } => None,
}
}
}
#[derive(Debug, Clone)]
struct RouteOpenRefused {
target: String,
body: ErrorBody,
most_recent: Option<ErrorBody>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum ScopeBindProofRefusal {
Required,
Mismatch,
}
impl fmt::Display for RouteOpenRefused {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"route.open failed for target {}: {}",
self.target,
DescribeRefusal(&self.body)
)?;
if let Some(most_recent) = &self.most_recent {
write!(
f,
"; the most recent refusal was {}",
DescribeRefusal(most_recent)
)?;
}
Ok(())
}
}
impl Error for RouteOpenRefused {}
struct DescribeRefusal<'a>(&'a ErrorBody);
impl fmt::Display for DescribeRefusal<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.0.code)?;
if let Some(reason) = refusal_reason(self.0) {
write!(f, " (reason {reason})")?;
}
write!(f, " ({})", self.0.message)
}
}
fn refusal_reason(body: &ErrorBody) -> Option<&str> {
body.detail.as_ref()?.get("reason")?.as_str()
}
const ADMISSION_PRESSURE_REASONS: [&str; 2] = ["open_admission_full", "target_binds_full"];
const ADMISSION_PRESSURE_MESSAGES: [&str; 2] =
["route.open binds in flight", "route.bind relays in flight"];
fn route_open_refusal_rank(body: &ErrorBody) -> u8 {
if body.code == error_codes::MODULE_RELOADING || body.code == error_codes::MODULE_WARMING {
return 3;
}
let reason = refusal_reason(body);
if body.code == "delegation_not_registered" || reason == Some("delegation_not_registered") {
return 2;
}
if body.code == error_codes::TARGET_UNAVAILABLE {
let admission_pressure = match reason {
Some(reason) => ADMISSION_PRESSURE_REASONS.contains(&reason),
None => ADMISSION_PRESSURE_MESSAGES
.iter()
.any(|marker| body.message.contains(marker)),
};
if admission_pressure {
return 1;
}
}
0
}
#[derive(Default)]
struct RouteOpenRefusals {
best: Option<ErrorBody>,
most_recent: Option<ErrorBody>,
}
impl RouteOpenRefusals {
fn record(&mut self, body: ErrorBody) {
let outranks = self
.best
.as_ref()
.is_none_or(|best| route_open_refusal_rank(&body) >= route_open_refusal_rank(best));
if outranks {
self.best = Some(body);
self.most_recent = None;
} else {
self.most_recent = Some(body);
}
}
fn into_error(self, target: String) -> Option<CallError> {
let body = self.best?;
Some(CallError::from_route_open_refusal(RouteOpenRefused {
target,
body,
most_recent: self.most_recent,
}))
}
}
fn route_target_label(target: &RouteTarget) -> String {
match target {
RouteTarget::ToolProvider { module_id } => format!("tool_provider:{module_id}"),
RouteTarget::ManagementSurface { module_id } => format!("management_surface:{module_id}"),
RouteTarget::InternalService {
module_id,
service_id,
} => format!("internal_service:{module_id}:{service_id}"),
}
}
impl CallError {
fn route_open_refused(target: String, body: ErrorBody) -> Self {
Self::from_route_open_refusal(RouteOpenRefused {
target,
body,
most_recent: None,
})
}
fn from_route_open_refusal(refusal: RouteOpenRefused) -> Self {
Self::NotSent(Box::new(NotSentSource {
message: refusal.to_string(),
cause: NotSentCause::RouteOpenRefused {
code: refusal.body.code.clone(),
},
never_forwarded: true,
route_reason: None,
refusal: Some(Box::new(refusal)),
}))
}
}
#[derive(Debug)]
struct SimpleError(String);
impl fmt::Display for SimpleError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.0)
}
}
impl Error for SimpleError {}
type Callback = Arc<Mutex<Box<dyn Fn(ConnectionState) + Send + 'static>>>;
type OpeningWaiter = oneshot::Sender<Result<RouteState, SharedCallFailure>>;
struct Opening {
waiters: Vec<OpeningWaiter>,
closed: bool,
}
struct Shared {
claim_bound: bool,
connection_file: PathBuf,
operator_app_candidates: Option<Vec<PathBuf>>,
opts: ConsumerOptions,
transport: ConsumerTransport,
identity_from_environment: bool,
inner: Mutex<Inner>,
notify: Notify,
#[cfg(test)]
before_wait: Mutex<Option<Box<dyn FnOnce() + Send>>>,
close_token: CancellationToken,
last_inbound_ms: AtomicU64,
liveness_probe_running: AtomicBool,
pushes_dropped_no_receiver: AtomicU64,
control_pushes_dropped: AtomicU64,
pushes_dropped_receiver_full: AtomicU64,
random_unit: Box<dyn Fn() -> f64 + Send + Sync>,
route_open_slots: Semaphore,
route_description: tokio::sync::Mutex<()>,
}
struct Inner {
generation: u64,
daemon_origin_trusted: bool,
daemon_ver: String,
accepts_close_reasons: Option<bool>,
epoch: u64,
next_corr: Option<u64>,
writer: Option<mpsc::Sender<WriteCommand>>,
pending: HashMap<PendingKey, PendingEntry>,
wire_slots: HashMap<PendingKey, Arc<RoutePermit>>,
routes: HashMap<RouteKey, RouteState>,
route_by_channel: HashMap<u16, RouteKey>,
one_shot_routes: HashMap<u16, RouteState>,
route_epochs: HashMap<u16, RouteHandle>,
route_targets: HashMap<RouteHandle, RouteOpenTarget>,
retired_routes: HashSet<RouteHandle>,
retirement_goodbyes: HashSet<RouteHandle>,
retirement_queue: VecDeque<RouteHandle>,
retirement_notify: Arc<Notify>,
retirement_task: Option<JoinHandle<()>>,
route_modules: HashMap<u16, String>,
route_end_reasons: HashMap<u16, (RouteEnd, bool)>,
legacy_channel_reasons: HashMap<u16, RouteEnd>,
push_event_receivers: HashMap<RouteHandle, mpsc::Sender<PushEvent>>,
control_push_receiver: Option<mpsc::Sender<ControlPush>>,
dropped_route_frames: u64,
openings: HashMap<RouteKey, Opening>,
callbacks: Vec<Callback>,
closed: bool,
reconnect: ReconnectState,
restored_token: u64,
reader_task: Option<JoinHandle<()>>,
writer_task: Option<JoinHandle<()>>,
}
impl Inner {
fn record_route_end(&mut self, push: &ControlPush) {
let Some(reason) = push.route_close_reason() else {
return;
};
let end = RouteEnd {
reason: RouteEndReason::Daemon(reason),
terminal: (push.op == "route.closed")
.then(|| {
push.body
.get("terminal")
.and_then(serde_json::Value::as_bool)
})
.flatten(),
};
if let Some(channels) = push.body.get("channels") {
if let Some(channels) = channels.as_array() {
for channel in channels
.iter()
.filter_map(|v| v.as_u64().and_then(|v| u16::try_from(v).ok()))
{
let final_reason = push.op == "route.closed";
if final_reason
|| !self
.route_end_reasons
.get(&channel)
.is_some_and(|(_, final_reason)| *final_reason)
{
self.route_end_reasons
.insert(channel, (end.clone(), final_reason));
}
}
}
} else if let Some(module) = push.body.get("module_id").and_then(|v| v.as_str()) {
for (&channel, target) in &self.route_modules {
if target == module {
self.legacy_channel_reasons.insert(channel, end.clone());
}
}
}
}
fn route_end(&self, channel: u16) -> RouteEnd {
self.route_end_reasons
.get(&channel)
.map(|(reason, _)| reason.clone())
.or_else(|| self.legacy_channel_reasons.get(&channel).cloned())
.unwrap_or_else(|| RouteEndReason::Unknown.into())
}
#[cfg(test)]
fn route_end_reason(&self, channel: u16) -> RouteEndReason {
self.route_end(channel).reason
}
fn cache_route(&mut self, key: RouteKey, route: RouteState) -> RouteState {
let cached = self.routes.entry(key.clone()).or_insert(route).clone();
let previous = self
.route_by_channel
.insert(cached.handle.channel, key.clone());
debug_assert!(previous.as_ref().is_none_or(|previous| previous == &key));
self.route_epochs
.insert(cached.handle.channel, cached.handle);
cached
}
fn remove_route(&mut self, key: &RouteKey) -> Option<RouteState> {
let route = self.evict_route(key)?;
release_reverse_request_registry(route.handle);
Some(route)
}
fn evict_route(&mut self, key: &RouteKey) -> Option<RouteState> {
let route = self.routes.remove(key)?;
let indexed = self.route_by_channel.remove(&route.handle.channel);
debug_assert_eq!(indexed.as_ref(), Some(key));
Some(route)
}
fn reap_retired_route(&mut self, handle: RouteHandle) {
if !self.retired_routes.contains(&handle)
|| self
.pending
.keys()
.chain(self.wire_slots.keys())
.any(|key| {
key.generation == handle.connection_token()
&& key.channel == handle.channel
&& key.epoch == handle.epoch
})
{
return;
}
let handle = *self
.retired_routes
.get(&handle)
.expect("retired route exists");
if self.retirement_goodbyes.contains(&handle) {
if !self.queue_retirement_goodbye(handle) {
return;
}
self.retirement_goodbyes.remove(&handle);
}
self.retired_routes.remove(&handle);
if self.route_epochs.get(&handle.channel) == Some(&handle) {
self.route_epochs.remove(&handle.channel);
self.route_targets.remove(&handle);
self.push_event_receivers.remove(&handle);
}
release_reverse_request_registry(handle);
}
fn queue_retirement_goodbye(&mut self, handle: RouteHandle) -> bool {
if self.generation != handle.connection_token()
|| self.route_epochs.get(&handle.channel) != Some(&handle)
|| self.writer.is_none()
{
return true;
}
if self.retirement_queue.is_empty() {
let frame = Frame::build(
FrameType::Goodbye,
Flags::new(false, Priority::Interactive, false),
handle.channel,
handle.epoch,
0,
Vec::new(),
)
.expect("header-only route GOODBYE is valid");
if self
.writer
.as_ref()
.unwrap()
.try_send(WriteCommand {
frame,
pending: None,
})
.is_ok()
{
return true;
}
}
if !self.retirement_queue.contains(&handle) {
self.retirement_queue.push_back(handle);
}
self.retirement_notify.notify_one();
false
}
fn remove_pending(&mut self, key: &PendingKey) -> Option<PendingEntry> {
let entry = self.pending.remove(key);
if entry.as_ref().is_some_and(|entry| !entry.accepted) {
if let Some(slot) = self.wire_slots.remove(key) {
slot.release();
}
}
self.reap_retired_route(RouteHandle::new(key.channel, key.epoch, key.generation));
entry
}
fn release_wire_slots(&mut self, handle: RouteHandle) {
self.wire_slots.retain(|key, slot| {
let keep = key.generation != handle.connection_token()
|| key.channel != handle.channel
|| key.epoch != handle.epoch;
if !keep {
slot.release();
}
keep
});
}
fn remove_route_by_handle(&mut self, handle: RouteHandle) -> Option<RouteState> {
if let Some(key) = self.route_by_channel.get(&handle.channel).cloned() {
let matches = self
.routes
.get(&key)
.is_some_and(|route| route.handle == handle);
debug_assert!(matches);
if matches {
return self.remove_route(&key);
}
}
let route = self
.one_shot_routes
.get(&handle.channel)
.is_some_and(|route| route.handle == handle)
.then(|| self.one_shot_routes.remove(&handle.channel))
.flatten();
if let Some(route) = &route {
release_reverse_request_registry(route.handle);
}
route
}
fn drain_routes(&mut self) -> Vec<RouteState> {
self.route_targets.clear();
for (_, slot) in self.wire_slots.drain() {
slot.release();
}
self.retirement_goodbyes.clear();
self.retirement_queue.clear();
if let Some(task) = self.retirement_task.take() {
task.abort();
}
for handle in self.retired_routes.drain() {
release_reverse_request_registry(handle);
}
self.route_by_channel.clear();
let routes = self
.routes
.drain()
.map(|(_, route)| route)
.chain(self.one_shot_routes.drain().map(|(_, route)| route))
.collect::<Vec<_>>();
for route in &routes {
release_reverse_request_registry(route.handle);
}
routes
}
fn close_routes(&mut self) {
self.push_event_receivers.clear();
for route in self.drain_routes() {
route.close();
}
}
}
impl Shared {
fn new(connection_file: PathBuf, opts: ConsumerOptions) -> Self {
Self::with_transport(connection_file, opts, ConsumerTransport::Tcp)
}
fn with_transport(
connection_file: PathBuf,
opts: ConsumerOptions,
transport: ConsumerTransport,
) -> Self {
Self {
claim_bound: crate::registration::bound(),
connection_file,
operator_app_candidates: None,
opts,
transport,
identity_from_environment: true,
inner: Mutex::new(Inner {
generation: 1,
daemon_origin_trusted: false,
daemon_ver: String::new(),
accepts_close_reasons: None,
epoch: 1,
next_corr: Some(1),
writer: None,
pending: HashMap::new(),
wire_slots: HashMap::new(),
routes: HashMap::new(),
route_by_channel: HashMap::new(),
one_shot_routes: HashMap::new(),
route_epochs: HashMap::new(),
route_targets: HashMap::new(),
retired_routes: HashSet::new(),
retirement_goodbyes: HashSet::new(),
retirement_queue: VecDeque::new(),
retirement_notify: Arc::new(Notify::new()),
retirement_task: None,
route_modules: HashMap::new(),
route_end_reasons: HashMap::new(),
legacy_channel_reasons: HashMap::new(),
push_event_receivers: HashMap::new(),
control_push_receiver: None,
dropped_route_frames: 0,
openings: HashMap::new(),
callbacks: Vec::new(),
closed: false,
reconnect: ReconnectState::Idle,
restored_token: 0,
reader_task: None,
writer_task: None,
}),
notify: Notify::new(),
#[cfg(test)]
before_wait: Mutex::new(None),
close_token: CancellationToken::new(),
last_inbound_ms: AtomicU64::new(0),
liveness_probe_running: AtomicBool::new(false),
pushes_dropped_no_receiver: AtomicU64::new(0),
control_pushes_dropped: AtomicU64::new(0),
pushes_dropped_receiver_full: AtomicU64::new(0),
random_unit: Box::new(default_random_unit),
route_open_slots: Semaphore::new(MAX_ROUTE_OPENS_IN_FLIGHT),
route_description: tokio::sync::Mutex::new(()),
}
}
fn jittered(&self, delay: Duration) -> Duration {
let unit = (self.random_unit)();
let unit = if unit.is_finite() {
unit.clamp(0.0, 1.0)
} else {
0.0
};
let half = delay / 2;
half + half.mul_f64(unit)
}
async fn paced_route_open(
self: &Arc<Self>,
body: Vec<u8>,
deadline: Instant,
reverse_requests: Option<ReverseRequestRegistry>,
) -> Result<TerminalFrame, CallError> {
let _slot = match timeout_at(deadline, self.route_open_slots.acquire()).await {
Ok(Ok(permit)) => permit,
Ok(Err(_)) => return Err(CallError::not_sent("consumer closed")),
Err(_) => {
return Err(CallError::not_sent(format!(
"deadline elapsed waiting for one of {MAX_ROUTE_OPENS_IN_FLIGHT} route.open slots"
)));
}
};
self.control_call(body, deadline, true, reverse_requests)
.await
}
fn lock_inner(&self) -> MutexGuard<'_, Inner> {
self.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
fn install_initial(self: &Arc<Self>, opened: OpenedConnection) -> Result<(), ConsumerError> {
self.install_connection(opened, InstallKind::Initial)
.map(|_| ())
}
fn install_reconnected(
self: &Arc<Self>,
opened: OpenedConnection,
) -> Result<(u64, u64), ConsumerError> {
let generation_epoch = self.install_connection(opened, InstallKind::Reconnect)?;
self.notify.notify_waiters();
Ok(generation_epoch)
}
fn install_connection(
self: &Arc<Self>,
opened: OpenedConnection,
kind: InstallKind,
) -> Result<(u64, u64), ConsumerError> {
if self.close_token.is_cancelled() {
return Err(ConsumerError::Closed);
}
let (reader, writer) = tokio::io::split(opened.stream);
let (tx, rx) = mpsc::channel(EGRESS_BUFFER);
let (generation, epoch, old_reader, old_writer) = {
let mut inner = self.lock_inner();
if inner.closed {
return Err(ConsumerError::Closed);
}
let (generation, epoch) = match kind {
InstallKind::Initial => (inner.generation, inner.epoch),
InstallKind::Reconnect => {
inner.generation = inner
.generation
.checked_add(1)
.ok_or(ConsumerError::Closed)?;
inner.epoch = inner.epoch.checked_add(1).ok_or(ConsumerError::Closed)?;
(inner.generation, inner.epoch)
}
};
inner.close_routes();
inner.route_epochs.clear();
inner.route_targets.clear();
inner.route_modules.clear();
inner.route_end_reasons.clear();
inner.legacy_channel_reasons.clear();
inner.next_corr = Some(1);
inner.writer = Some(tx);
inner.daemon_origin_trusted = opened.daemon_origin_trusted;
inner.daemon_ver = opened.daemon_ver;
inner.accepts_close_reasons = None;
self.last_inbound_ms.store(0, Ordering::Release);
(
generation,
epoch,
inner.reader_task.take(),
inner.writer_task.take(),
)
};
if let Some(handle) = old_reader {
handle.abort();
}
if let Some(handle) = old_writer {
handle.abort();
}
let reader_shared = Arc::clone(self);
let reader_task = tokio::spawn(async move {
reader_loop(reader_shared, reader, generation).await;
});
let writer_shared = Arc::clone(self);
let writer_task = tokio::spawn(async move {
writer_loop(writer_shared, writer, rx, generation).await;
});
{
let mut inner = self.lock_inner();
if inner.closed || inner.generation != generation {
reader_task.abort();
writer_task.abort();
return Err(ConsumerError::Closed);
}
inner.reader_task = Some(reader_task);
inner.writer_task = Some(writer_task);
let retirement_shared = Arc::clone(self);
let retirement_notify = Arc::clone(&inner.retirement_notify);
inner.retirement_task = Some(tokio::spawn(async move {
retirement_goodbye_loop(retirement_shared, generation, retirement_notify).await;
}));
}
Ok((generation, epoch))
}
async fn ensure_connected_for_call(
self: &Arc<Self>,
deadline: Instant,
) -> Result<(), CallError> {
loop {
if Instant::now() >= deadline {
return Err(CallError::not_sent(
"call deadline elapsed waiting for reconnection",
));
}
let notified = self.notify.notified();
tokio::pin!(notified);
let action = {
let mut inner = self.lock_inner();
notified.as_mut().enable();
if inner.closed {
return Err(CallError::not_sent("consumer closed"));
}
if inner.writer.is_some() {
return Ok(());
}
let generation = inner.generation;
let reconnect_is_live = match &inner.reconnect {
ReconnectState::Idle => false,
ReconnectState::Inline { generation: active } => *active == generation,
ReconnectState::Background {
generation: active,
task,
} => *active == generation && !task.is_finished(),
};
if reconnect_is_live {
EnsureAction::Wait
} else {
let stale_task =
match std::mem::replace(&mut inner.reconnect, ReconnectState::Idle) {
ReconnectState::Background { task, .. } => Some(task),
ReconnectState::Idle | ReconnectState::Inline { .. } => None,
};
inner.reconnect = ReconnectState::Inline { generation };
EnsureAction::Lead {
generation,
stale_task,
}
}
};
match action {
EnsureAction::Wait => {
#[cfg(test)]
{
let hook = self.before_wait.lock().unwrap().take();
if let Some(hook) = hook {
hook();
}
}
timeout_at(deadline, notified).await.map_err(|_| {
CallError::not_sent("call deadline elapsed waiting for reconnection")
})?;
}
EnsureAction::Lead {
generation,
stale_task,
} => {
if let Some(handle) = stale_task {
handle.abort();
}
let mut guard = InlineReconnectGuard::new(Arc::clone(self), generation);
let result = timeout_at(deadline, self.reconnect_with_retry(generation)).await;
guard.finish();
return match result {
Ok(Ok(())) => Ok(()),
Ok(Err(err)) => Err(CallError::not_sent(err.to_string())),
Err(_) => Err(CallError::not_sent(
"call deadline elapsed waiting for reconnection",
)),
};
}
}
}
}
fn spawn_reconnect(self: &Arc<Self>, generation: u64) -> bool {
let stale_task = {
let mut inner = self.lock_inner();
if inner.closed || inner.writer.is_some() || inner.generation != generation {
return false;
}
let should_spawn = match &inner.reconnect {
ReconnectState::Idle => true,
ReconnectState::Inline { generation: active } => *active < generation,
ReconnectState::Background {
generation: active,
task,
} => *active < generation || (*active == generation && task.is_finished()),
};
if !should_spawn {
return false;
}
let stale_task = match std::mem::replace(&mut inner.reconnect, ReconnectState::Idle) {
ReconnectState::Background { task, .. } => Some(task),
ReconnectState::Idle | ReconnectState::Inline { .. } => None,
};
let shared = Arc::clone(self);
let handle = tokio::spawn(async move {
let _result = shared.reconnect_with_retry(generation).await;
shared.finish_background_reconnect(generation);
});
inner.reconnect = ReconnectState::Background {
generation,
task: handle,
};
stale_task
};
if let Some(handle) = stale_task {
handle.abort();
}
true
}
async fn reconnect_with_retry(
self: &Arc<Self>,
reconnect_generation: u64,
) -> Result<(), ConsumerError> {
let mut last_error: Option<ConsumerError> = None;
for attempt in 1..=self.opts.reconnect_backoff.max_attempts {
if self.close_token.is_cancelled() {
return Err(ConsumerError::Closed);
}
if !self.reconnect_attempt_is_current(reconnect_generation) {
return Ok(());
}
match open_connection(
&self.connection_file,
self.opts.handshake_timeout,
self.transport,
self.operator_app_candidates.as_deref(),
)
.await
{
Ok(opened) => {
if !self.reconnect_attempt_is_current(reconnect_generation) {
return Ok(());
}
let (generation, epoch) = self.install_reconnected(opened)?;
self.schedule_restored(generation, epoch);
return Ok(());
}
Err(err) => {
let transient = is_reconnect_transient(&err);
last_error = Some(err);
if !transient || attempt >= self.opts.reconnect_backoff.max_attempts {
break;
}
let delay =
self.jittered(self.opts.reconnect_backoff.delay_after_attempt(attempt));
tokio::select! {
() = self.close_token.cancelled() => return Err(ConsumerError::Closed),
() = sleep(delay) => {}
}
}
}
}
Err(last_error.unwrap_or(ConsumerError::Closed))
}
fn reconnect_attempt_is_current(&self, generation: u64) -> bool {
let inner = self.lock_inner();
!inner.closed
&& inner.writer.is_none()
&& inner.generation == generation
&& matches!(
&inner.reconnect,
ReconnectState::Inline { generation: active }
| ReconnectState::Background {
generation: active,
..
} if *active == generation
)
}
fn finish_inline_reconnect(&self, generation: u64) {
let finished = {
let mut inner = self.lock_inner();
if matches!(
&inner.reconnect,
ReconnectState::Inline { generation: active } if *active == generation
) {
inner.reconnect = ReconnectState::Idle;
true
} else {
false
}
};
if finished {
self.notify.notify_waiters();
}
}
fn finish_background_reconnect(&self, generation: u64) {
let completed_task = {
let mut inner = self.lock_inner();
if !matches!(
&inner.reconnect,
ReconnectState::Background { generation: active, .. } if *active == generation
) {
None
} else {
match std::mem::replace(&mut inner.reconnect, ReconnectState::Idle) {
ReconnectState::Background { task, .. } => Some(task),
ReconnectState::Idle | ReconnectState::Inline { .. } => unreachable!(),
}
}
};
if completed_task.is_some() {
drop(completed_task);
self.notify.notify_waiters();
}
}
fn schedule_restored(self: &Arc<Self>, generation: u64, epoch: u64) {
let token = {
let mut inner = self.lock_inner();
inner.restored_token = inner.restored_token.saturating_add(1);
inner.restored_token
};
let shared = Arc::clone(self);
tokio::spawn(async move {
tokio::select! {
() = shared.close_token.cancelled() => {}
() = sleep(shared.opts.restored_debounce) => {
let should_emit = {
let inner = shared.lock_inner();
!inner.closed
&& inner.generation == generation
&& inner.epoch == epoch
&& inner.restored_token == token
&& inner.writer.is_some()
};
if should_emit {
shared.emit_connection_state(ConnectionState::Restored { epoch });
}
}
}
});
}
fn install_one_shot_route(&self, route: RouteState) -> Result<(), CallError> {
let handle = route.handle;
let mut inner = self.lock_inner();
if inner.closed || inner.generation != handle.connection_token() || inner.writer.is_none() {
return Err(CallError::StaleRouteHandle(handle));
}
if inner
.route_epochs
.get(&handle.channel)
.is_some_and(|existing| *existing != handle)
{
return Err(CallError::not_sent(
"daemon returned a route channel already in use",
));
}
inner.one_shot_routes.insert(handle.channel, route);
inner.route_epochs.insert(handle.channel, handle);
Ok(())
}
async fn ensure_route(
self: &Arc<Self>,
key: &RouteKey,
route_open: &RouteOpenParams<'_>,
opts: &CallOptions,
call_deadline: Instant,
) -> Result<RouteState, CallError> {
loop {
let action = {
let mut inner = self.lock_inner();
if inner.closed {
return Err(CallError::not_sent("consumer closed"));
}
if let Some(route) = inner.routes.get(key) {
if route.handle.connection_token() == inner.generation && inner.writer.is_some()
{
return Ok(route.clone());
}
}
if let Some(opening) = inner.openings.get_mut(key) {
let (tx, rx) = oneshot::channel();
opening.waiters.push(tx);
RouteOpenAction::Wait(rx)
} else {
inner.openings.insert(
key.clone(),
Opening {
waiters: Vec::new(),
closed: false,
},
);
RouteOpenAction::Lead
}
};
match action {
RouteOpenAction::Wait(rx) => match timeout_at(call_deadline, rx).await {
Ok(Ok(Ok(route))) => return Ok(route),
Ok(Ok(Err(err))) => return Err(err.into_call_error()),
Ok(Err(_)) => continue,
Err(_) => {
return Err(CallError::not_sent_with_cause(
"call deadline elapsed waiting for route.open",
NotSentCause::RouteOpenAbandoned,
));
}
},
RouteOpenAction::Lead => {
let mut guard = OpeningGuard::new(Arc::clone(self), key.clone());
let result = self
.open_route_with_retry(key, route_open, opts, call_deadline)
.await
.map_err(SharedCallFailure::from);
guard.finish(result.clone());
return result.map_err(SharedCallFailure::into_call_error);
}
}
}
}
async fn open_route_with_retry(
self: &Arc<Self>,
key: &RouteKey,
route_open: &RouteOpenParams<'_>,
opts: &CallOptions,
call_deadline: Instant,
) -> Result<RouteState, CallError> {
let route_deadline = (Instant::now() + opts.route_retry_deadline).min(call_deadline);
let mut attempt = 0usize;
let mut refusals = RouteOpenRefusals::default();
let expired = |err: CallError, refusals: &mut RouteOpenRefusals| {
let ended_by_deadline =
err.is_not_sent() || err.outcome_cause() == Some(OutcomeUnknownCause::Deadline);
if ended_by_deadline && Instant::now() >= route_deadline {
if let Some(refused) = std::mem::take(refusals).into_error(key.target_label()) {
return refused;
}
}
err
};
loop {
attempt = attempt.saturating_add(1);
let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
busy_refusals: Some(true),
target: route_open.target.clone(),
identity: route_open.identity.clone(),
consumer_identity: route_open.consumer_identity.clone(),
consumer_capabilities: route_open.consumer_capabilities.clone(),
role_versions: route_open.role_versions.clone(),
admission_facts: None,
scope: route_open.scope.cloned(),
})
.map_err(|err| CallError::not_sent(format!("failed to encode route.open: {err}")))?;
match self
.paced_route_open(
body,
route_deadline,
Some(route_open.reverse_requests.clone()),
)
.await
{
Ok(TerminalFrame::Response {
generation, body, ..
}) => {
let (response, _) = decode_route_open_response(&body).map_err(|err| {
CallError::not_sent_with_cause(
format!("failed to decode route.open response: {err}"),
NotSentCause::RouteOpenAbandoned,
)
})?;
let ClientControlResponse::RouteOpen {
route_channel,
route_epoch,
request_window,
watch_window,
..
} = response
else {
return Err(CallError::not_sent_with_cause(
"route.open returned an unexpected control response",
NotSentCause::RouteOpenAbandoned,
));
};
let handle = self
.ingress_handle(generation, route_channel, route_epoch)
.unwrap_or_else(|| {
RouteHandle::new_consumer(
route_channel,
route_epoch,
generation,
route_open.reverse_requests.clone(),
)
});
let route = RouteState::windows(handle, request_window, watch_window);
let install = {
let mut inner = self.lock_inner();
if inner.closed {
return Err(CallError::not_sent("consumer closed"));
}
let closed_during_open = inner.openings.get(key).is_some_and(|o| o.closed);
if closed_during_open
|| inner.generation != generation
|| inner.writer.is_none()
{
RouteInstall::Discard {
closed: closed_during_open,
}
} else {
let cached = inner.cache_route(key.clone(), route.clone());
RouteInstall::Cached(cached)
}
};
match install {
RouteInstall::Cached(cached) => return Ok(cached),
RouteInstall::Discard { closed } => {
if closed {
self.send_route_goodbye(route.handle, true);
self.uninstall_route_handle(route.handle);
return Err(CallError::not_sent_with_cause(
"route was closed before route.open completed",
NotSentCause::RouteOpenAbandoned,
));
}
}
}
self.sleep_until_retry(route_deadline, self.jittered(opts.route_retry.base))
.await?;
}
Ok(TerminalFrame::Error { body, .. }) => {
if !is_retryable_route_open_code(&body.code) {
return Err(CallError::route_open_refused(key.target_label(), body));
}
refusals.record(body);
if Instant::now() >= route_deadline {
return Err(std::mem::take(&mut refusals)
.into_error(key.target_label())
.unwrap_or_else(|| {
CallError::not_sent("route.open retry deadline elapsed")
}));
}
let delay = self.jittered(opts.route_retry.delay_after_attempt(attempt));
if let Err(err) = self.sleep_until_retry(route_deadline, delay).await {
return Err(expired(err, &mut refusals));
}
continue;
}
Ok(TerminalFrame::StreamEnd) => {
return Err(CallError::not_sent_with_cause(
"route.open returned StreamEnd",
NotSentCause::RouteOpenAbandoned,
));
}
Err(err) if err.is_not_sent() && Instant::now() < route_deadline => {
let delay = self.jittered(opts.route_retry.delay_after_attempt(attempt));
if let Err(err) = self.sleep_until_retry(route_deadline, delay).await {
return Err(expired(err, &mut refusals));
}
}
Err(err) => return Err(expired(err, &mut refusals)),
}
}
#[allow(unreachable_code)]
Err(CallError::not_sent(format!(
"route.open retry deadline elapsed for target {}",
key.target_label()
)))
}
async fn sleep_until_retry(&self, deadline: Instant, delay: Duration) -> Result<(), CallError> {
if Instant::now() >= deadline {
return Err(CallError::not_sent("retry deadline elapsed"));
}
let bounded = delay.min(deadline.saturating_duration_since(Instant::now()));
tokio::select! {
() = self.close_token.cancelled() => Err(CallError::not_sent("consumer closed")),
() = sleep(bounded) => Ok(()),
}
}
async fn control_call(
self: &Arc<Self>,
body: Vec<u8>,
deadline: Instant,
retain_late_route_open: bool,
route_open_reverse_requests: Option<ReverseRequestRegistry>,
) -> Result<TerminalFrame, CallError> {
self.ensure_connected_for_call(deadline).await?;
if retain_late_route_open {
{
let inner = self.lock_inner();
if !daemon_version_at_least(&inner.daemon_ver, (0, 39, 0)) {
return Err(CallError::DaemonTooOld {
daemon_ver: inner.daemon_ver.clone(),
});
}
}
self.describe_route_close_reasons(deadline).await?;
}
self.send_request(RequestSend {
binary: false,
expected_handle: None,
channel: 0,
epoch: 0,
body,
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline,
retain_late_route_open,
route_open_reverse_requests,
})
.await
}
async fn describe_route_close_reasons(
self: &Arc<Self>,
deadline: Instant,
) -> Result<(), CallError> {
let _lookup = timeout_at(deadline, self.route_description.lock())
.await
.map_err(|_| CallError::not_sent("deadline waiting for server.describe"))?;
loop {
self.ensure_connected_for_call(deadline).await?;
let generation = {
let inner = self.lock_inner();
if inner.accepts_close_reasons.is_some() {
return Ok(());
}
inner.generation
};
let reply = self
.send_request(RequestSend {
binary: false,
expected_handle: None,
channel: 0,
epoch: 0,
body: br#"{"op":"server.describe"}"#.to_vec(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline: deadline.min(Instant::now() + CAPABILITY_PROBE_TIMEOUT),
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await;
let supported = match reply {
Ok(TerminalFrame::Response { body, .. }) => {
matches!(serde_json::from_slice::<ClientControlResponse>(&body),
Ok(ClientControlResponse::ServerDescribe { capabilities, .. })
if capabilities.iter().any(|cap| cap == ROUTE_CLOSE_REASONS_CAPABILITY))
}
_ => false,
};
let mut inner = self.lock_inner();
if inner.generation == generation {
inner.accepts_close_reasons = Some(supported);
return Ok(());
}
}
}
async fn send_request(
self: &Arc<Self>,
request: RequestSend,
) -> Result<TerminalFrame, CallError> {
self.send_request_cancellable(request, None).await
}
fn allocate_request(
self: &Arc<Self>,
expected_handle: Option<RouteHandle>,
channel: u16,
epoch: u32,
) -> Result<(PendingKey, mpsc::Sender<WriteCommand>), CallError> {
let mut inner = self.lock_inner();
if inner.closed {
return Err(CallError::not_sent("consumer closed"));
}
let generation = inner.generation;
if let Some(expected) = expected_handle {
let route_pair_matches =
channel == 0 || (expected.channel == channel && expected.epoch == epoch);
if expected.connection_token() != generation
|| !route_pair_matches
|| inner.route_epochs.get(&expected.channel) != Some(&expected)
|| inner.retired_routes.contains(&expected)
{
return Err(CallError::StaleRouteHandle(expected));
}
}
let Some(writer) = inner.writer.clone() else {
return Err(CallError::not_sent("subc connection is down before send"));
};
let Some(corr) = inner.next_corr else {
drop(inner);
self.handle_generation_drop(
generation,
"channel-0 correlation allocator exhausted".to_string(),
);
return Err(CallError::not_sent(
"correlation allocator exhausted; connection closed",
));
};
inner.next_corr = corr.checked_add(1);
Ok((
PendingKey {
generation,
channel,
epoch,
corr,
},
writer,
))
}
async fn send_request_cancellable(
self: &Arc<Self>,
request: RequestSend,
cancel: Option<Arc<UnaryCancel>>,
) -> Result<TerminalFrame, CallError> {
self.send_request_with_slot(request, cancel, None).await
}
async fn send_request_with_slot(
self: &Arc<Self>,
request: RequestSend,
cancel: Option<Arc<UnaryCancel>>,
slot: Option<Arc<RoutePermit>>,
) -> Result<TerminalFrame, CallError> {
let RequestSend {
expected_handle,
channel,
epoch,
body,
binary,
priority,
admission_class,
deadline,
retain_late_route_open,
route_open_reverse_requests,
} = request;
if Instant::now() >= deadline {
return Err(CallError::not_sent(
"call deadline elapsed before request was sent",
));
}
let slot = match (slot, expected_handle) {
(Some(slot), _) => Some(slot),
(None, Some(handle)) if channel != 0 => match self.route_state(handle) {
Ok(route) if route.opted_in => {
Some(route.acquire(false, deadline, cancel.as_ref()).await?)
}
_ => None,
},
(None, _) => None,
};
let (key, writer) = self.allocate_request(expected_handle, channel, epoch)?;
let generation = key.generation;
let mut frame = Frame::build(
FrameType::Request,
Flags::new(binary, priority, false).with_admission_class(admission_class),
channel,
epoch,
key.corr,
body,
)
.map_err(|err| CallError::not_sent(format!("failed to build request frame: {err}")))?;
let (tx, rx) = oneshot::channel();
{
let mut inner = self.lock_inner();
if inner.closed || inner.generation != generation || inner.writer.is_none() {
return Err(CallError::not_sent(
"connection changed before request registration",
));
}
if retain_late_route_open && !daemon_version_at_least(&inner.daemon_ver, (0, 39, 0)) {
return Err(CallError::DaemonTooOld {
daemon_ver: inner.daemon_ver.clone(),
});
}
if retain_late_route_open {
let supported = inner.accepts_close_reasons.ok_or_else(|| {
CallError::not_sent("connection changed before route.open capability discovery")
})?;
let mut body: serde_json::Value = serde_json::from_slice(&frame.body)
.map_err(|err| CallError::not_sent(format!("invalid route.open: {err}")))?;
RouteOpenRequestExt::new()
.with_accepts_close_reasons(supported.then(|| {
ACCEPTED_CLOSE_REASONS
.iter()
.map(|reason| (*reason).to_string())
.collect()
}))
.write_to(&mut body)
.map_err(|err| CallError::not_sent(format!("invalid route.open: {err}")))?;
frame = Frame::build(
FrameType::Request,
frame.header.flags,
channel,
epoch,
key.corr,
serde_json::to_vec(&body)
.map_err(|err| CallError::not_sent(err.to_string()))?,
)
.map_err(|err| CallError::not_sent(err.to_string()))?;
}
if let Some(expected) = expected_handle {
if inner.route_epochs.get(&expected.channel) != Some(&expected)
|| inner.retired_routes.contains(&expected)
{
return Err(CallError::StaleRouteHandle(expected));
}
}
let expected_control_handle = (channel == 0 && !retain_late_route_open)
.then_some(expected_handle)
.flatten();
inner.pending.insert(
key,
PendingEntry::unary(
tx,
retain_late_route_open,
expected_control_handle,
route_open_reverse_requests,
),
);
if let Some(slot) = slot.filter(|slot| slot.opted_in()) {
inner.wire_slots.insert(key, slot);
}
if retain_late_route_open {
inner.pending.get_mut(&key).unwrap().route_open_module =
serde_json::from_slice::<serde_json::Value>(&frame.body)
.ok()
.and_then(|body| {
body.get("target")?
.get("module_id")?
.as_str()
.map(str::to_string)
});
}
}
let mut registration =
PendingRegistration::new(Arc::clone(self), key, retain_late_route_open);
registration.unary_cancel = cancel.clone();
match timeout_at(
deadline,
writer.send(WriteCommand {
frame,
pending: Some(key),
}),
)
.await
{
Ok(Ok(())) => {}
Ok(Err(_)) => {
let accepted = registration.remove_pending().unwrap_or(false);
return Err(classify_failure(
accepted,
"writer task closed before accepting request",
OutcomeUnknownCause::WriterClosed,
));
}
Err(_) => {
let accepted = registration.remove_pending().unwrap_or(false);
return Err(classify_failure_with_not_sent_cause(
accepted,
"call deadline elapsed waiting for writer capacity",
OutcomeUnknownCause::Deadline,
NotSentCause::WriterCapacity,
));
}
}
if let Some(cancel) = cancel {
cancel.activate(key, priority, deadline);
if cancel.requested() {
cancel.cancel().await;
}
}
self.wait_for_reply(registration, rx, deadline).await
}
async fn wait_for_reply(
self: &Arc<Self>,
mut registration: PendingRegistration,
rx: oneshot::Receiver<PendingResult>,
deadline: Instant,
) -> Result<TerminalFrame, CallError> {
let key = registration.key;
let channel = key.channel;
tokio::select! {
result = timeout_at(deadline, rx) => match result {
Ok(Ok(result)) => {
registration.disarm();
result.into_call_result()
}
Ok(Err(_)) => {
registration.disarm();
Err(CallError::outcome_unknown(
"pending response channel closed",
OutcomeUnknownCause::CompletionFailed,
))
}
Err(_) => {
let accepted = if registration.retain_on_drop {
registration.disarm();
self.pending_accepted(key).unwrap_or(true)
} else {
registration.remove_pending().unwrap_or(true)
};
if accepted {
let cancel_sent = registration.unary_cancel.as_ref().is_some_and(|cancel| cancel.sent());
if !cancel_sent && self.lock_inner().wire_slots.contains_key(&key) {
self.send_cancel(RouteHandle::new(key.channel, key.epoch, key.generation), key.corr, Priority::Interactive);
}
self.spawn_liveness_probe();
}
Err(classify_failure(
accepted,
format!("request on channel {channel} timed out at its deadline"),
OutcomeUnknownCause::Deadline,
))
}
},
() = self.close_token.cancelled() => {
let accepted = registration.remove_pending().unwrap_or(true);
Err(classify_failure(accepted, "consumer closed while request was pending", OutcomeUnknownCause::ConsumerClosed))
}
}
}
fn spawn_liveness_probe(self: &Arc<Self>) {
if self.liveness_probe_running.swap(true, Ordering::AcqRel) {
return;
}
let allocation = {
let mut inner = self.lock_inner();
let control_pending = inner
.pending
.keys()
.any(|key| key.generation == inner.generation && key.channel == 0);
if inner.closed || control_pending || !matches!(&inner.reconnect, ReconnectState::Idle)
{
None
} else {
match (inner.next_corr, inner.writer.clone()) {
(Some(corr), Some(writer)) => {
inner.next_corr = corr.checked_add(1);
Some((inner.generation, corr, writer))
}
(None, _) | (_, None) => None,
}
}
};
let Some((generation, corr, writer)) = allocation else {
self.liveness_probe_running.store(false, Ordering::Release);
return;
};
let t0 = epoch_millis();
let ping = match Frame::build(
FrameType::Ping,
Flags::new(false, Priority::Interactive, false),
0,
0,
corr,
Vec::new(),
) {
Ok(ping) => ping,
Err(_) => {
self.liveness_probe_running.store(false, Ordering::Release);
return;
}
};
let window = self.opts.liveness_probe_window;
let shared = Arc::clone(self);
tokio::spawn(async move {
let window_end = Instant::now() + window;
let _ = timeout_at(
window_end,
writer.send(WriteCommand {
frame: ping,
pending: None,
}),
)
.await;
sleep_until(window_end).await;
let control_pending_now = {
let inner = shared.lock_inner();
inner
.pending
.keys()
.any(|key| key.generation == generation && key.channel == 0)
};
if !shared.close_token.is_cancelled()
&& !control_pending_now && shared.generation_is_current(generation)
&& shared.last_inbound_ms.load(Ordering::Acquire) < t0
{
shared.handle_generation_drop(
generation,
format!(
"liveness probe convicted a half-open socket: no inbound frame for {}ms after a channel-0 Ping",
window.as_millis()
),
);
}
shared
.liveness_probe_running
.store(false, Ordering::Release);
});
}
async fn send_subscription(
self: &Arc<Self>,
subscription: SubscriptionSend,
) -> Result<Subscription, CallError> {
let SubscriptionSend {
expected_handle,
channel,
epoch,
body,
priority,
admission_class,
event_buffer,
deadline,
permit,
} = subscription;
if Instant::now() >= deadline {
return Err(CallError::not_sent(
"subscription deadline elapsed before request was sent",
));
}
let (generation, corr, writer) = {
let mut inner = self.lock_inner();
if inner.closed {
return Err(CallError::not_sent("consumer closed"));
}
let generation = inner.generation;
if let Some(expected) = expected_handle {
if expected.connection_token() != generation
|| expected.channel != channel
|| expected.epoch != epoch
|| inner.route_epochs.get(&channel) != Some(&expected)
|| inner.retired_routes.contains(&expected)
{
return Err(CallError::StaleRouteHandle(expected));
}
}
let Some(writer) = inner.writer.clone() else {
return Err(CallError::not_sent("subc connection is down before send"));
};
let Some(corr) = inner.next_corr else {
drop(inner);
self.handle_generation_drop(
generation,
"correlation allocator exhausted".to_string(),
);
return Err(CallError::not_sent(
"correlation allocator exhausted; connection closed",
));
};
inner.next_corr = corr.checked_add(1);
(generation, corr, writer)
};
let frame = Frame::build(
FrameType::Request,
Flags(
Flags::new(false, priority, false)
.with_admission_class(admission_class)
.0
| FLAG_SUBSCRIPTION,
),
channel,
epoch,
corr,
body,
)
.map_err(|err| CallError::not_sent(format!("failed to build request frame: {err}")))?;
let key = PendingKey {
generation,
channel,
epoch,
corr,
};
let position = Arc::new(SubscriptionPosition {
key: Mutex::new(key),
stop: CancellationToken::new(),
});
let retry = permit.opted_in().then(|| SubscriptionRetry {
frame: frame.clone(),
deadline,
delay: Duration::from_millis(25),
position: Arc::clone(&position),
});
let (events_tx, events_rx) = mpsc::channel(event_buffer.max(1));
let (closed_tx, closed_rx) = oneshot::channel();
let (ended_tx, ended_rx) = watch::channel(None);
{
let mut inner = self.lock_inner();
if inner.closed || inner.generation != generation || inner.writer.is_none() {
return Err(CallError::not_sent(
"connection changed before subscription registration",
));
}
if let Some(expected) = expected_handle {
if inner.route_epochs.get(&expected.channel) != Some(&expected)
|| inner.retired_routes.contains(&expected)
{
return Err(CallError::StaleRouteHandle(expected));
}
}
if permit.opted_in() {
inner.wire_slots.insert(key, Arc::clone(&permit));
}
let mut entry =
PendingEntry::subscription(events_tx, closed_tx, ended_tx, permit, priority);
entry.retry = retry;
inner.pending.insert(key, entry);
}
let mut registration = PendingRegistration::new(Arc::clone(self), key, false);
match timeout_at(
deadline,
writer.send(WriteCommand {
frame,
pending: Some(key),
}),
)
.await
{
Ok(Ok(())) => {}
Ok(Err(_)) => {
let accepted = registration.remove_pending().unwrap_or(false);
return Err(classify_failure(
accepted,
"writer task closed before accepting subscription request",
OutcomeUnknownCause::WriterClosed,
));
}
Err(_) => {
let accepted = registration.remove_pending().unwrap_or(false);
return Err(classify_failure_with_not_sent_cause(
accepted,
"subscription deadline elapsed waiting for writer capacity",
OutcomeUnknownCause::Deadline,
NotSentCause::WriterCapacity,
));
}
}
registration.disarm();
Ok(Subscription {
events: events_rx,
closed: SubscriptionClosed { rx: closed_rx },
ended: ended_rx,
cancel: SubscriptionCancel::new(Arc::clone(self), position, priority),
})
}
fn unsubscribe_subscription(
&self,
position: &SubscriptionPosition,
priority: Priority,
) -> Result<(), CallError> {
let key = *position.key.lock().unwrap_or_else(|err| err.into_inner());
let handle = RouteHandle::new(key.channel, key.epoch, key.generation);
self.validate_current_handle(handle)?;
let (key, entry) = {
let mut inner = self.lock_inner();
let key = *position.key.lock().unwrap_or_else(|err| err.into_inner());
position.stop.cancel();
let entry = inner.pending.remove(&key);
if entry.as_ref().is_some_and(|entry| !entry.accepted) {
if let Some(slot) = inner.wire_slots.remove(&key) {
slot.release();
}
}
(key, entry)
};
if let Some(entry) = entry {
let should_cancel = entry.accepted || entry.retry.is_none();
entry.settle_subscription_result(Ok(()));
if should_cancel {
self.send_cancel(handle, key.corr, priority);
}
self.lock_inner().reap_retired_route(handle);
}
Ok(())
}
fn route_stream_data(&self, key: PendingKey, body: Vec<u8>) {
let overflow = {
let mut inner = self.lock_inner();
let Some(entry) = inner.pending.get(&key) else {
return;
};
match entry.try_send_stream_data(body) {
Ok(()) | Err(StreamDataDelivery::NotSubscription) => return,
Err(StreamDataDelivery::Full) => {
let priority = entry
.subscription_priority()
.unwrap_or(Priority::Interactive);
let entry = inner.pending.remove(&key);
entry.map(|entry| {
(
entry,
priority,
"subscription event channel filled; reader dropped the stream instead of blocking",
)
})
}
Err(StreamDataDelivery::Closed) => {
let priority = entry
.subscription_priority()
.unwrap_or(Priority::Interactive);
let entry = inner.pending.remove(&key);
entry.map(|entry| {
(
entry,
priority,
"subscription event receiver closed before the stream ended",
)
})
}
}
};
if let Some((entry, priority, reason)) = overflow {
entry.settle_call_error(CallError::subscription_backpressure(reason));
self.send_cancel(
RouteHandle::new(key.channel, key.epoch, key.generation),
key.corr,
priority,
);
self.lock_inner().reap_retired_route(RouteHandle::new(
key.channel,
key.epoch,
key.generation,
));
}
}
fn register_push_events(
&self,
handle: RouteHandle,
) -> Result<mpsc::Receiver<PushEvent>, CallError> {
let (events_tx, events_rx) = mpsc::channel(DEFAULT_PUSH_EVENT_BUFFER);
let mut inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| inner.writer.is_none()
|| inner.route_epochs.get(&handle.channel) != Some(&handle)
{
return Err(CallError::StaleRouteHandle(handle));
}
inner.push_event_receivers.insert(handle, events_tx);
Ok(events_rx)
}
fn route_push(&self, handle: RouteHandle, body: Vec<u8>) {
let should_count_drop = {
let mut inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| inner.route_epochs.get(&handle.channel) != Some(&handle)
{
return;
}
match inner.push_event_receivers.get(&handle) {
None => DroppedPush::NoReceiver,
Some(events) => match events.try_send(PushEvent { handle, body }) {
Ok(()) => DroppedPush::Delivered,
Err(mpsc::error::TrySendError::Closed(_)) => {
inner.push_event_receivers.remove(&handle);
DroppedPush::NoReceiver
}
Err(mpsc::error::TrySendError::Full(_)) => DroppedPush::ReceiverFull,
},
}
};
match should_count_drop {
DroppedPush::Delivered => {}
DroppedPush::NoReceiver => {
self.pushes_dropped_no_receiver
.fetch_add(1, Ordering::Relaxed);
}
DroppedPush::ReceiverFull => {
self.pushes_dropped_receiver_full
.fetch_add(1, Ordering::Relaxed);
}
}
}
fn control_push(&self, body: &[u8]) {
let parsed = serde_json::from_slice::<serde_json::Value>(body)
.ok()
.and_then(|value| {
let op = value.get("op")?.as_str()?.to_string();
Some(ControlPush { op, body: value })
});
let delivered = match parsed {
None => false,
Some(push) => {
let mut inner = self.lock_inner();
inner.record_route_end(&push);
match inner.control_push_receiver.as_ref() {
None => false,
Some(sender) => match sender.try_send(push) {
Ok(()) => true,
Err(mpsc::error::TrySendError::Closed(_)) => {
inner.control_push_receiver = None;
false
}
Err(mpsc::error::TrySendError::Full(_)) => false,
},
}
}
};
if !delivered {
self.control_pushes_dropped.fetch_add(1, Ordering::Relaxed);
}
}
fn send_cancel(&self, handle: RouteHandle, corr: u64, priority: Priority) {
let writer = {
let inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| !route_is_installed(&inner, handle)
{
return;
}
inner.writer.clone()
};
let Some(writer) = writer else {
return;
};
let Ok(frame) = Frame::build(
FrameType::Cancel,
Flags::new(false, priority, false),
handle.channel,
handle.epoch,
corr,
Vec::new(),
) else {
return;
};
let _ = writer.try_send(WriteCommand {
frame,
pending: None,
});
}
fn mark_pending_accepted(&self, key: PendingKey) -> bool {
let mut inner = self.lock_inner();
if inner.closed || inner.generation != key.generation {
return false;
}
let Some(entry) = inner.pending.get_mut(&key) else {
return false;
};
if entry
.send_deadline
.is_some_and(|deadline| Instant::now() >= deadline)
{
let entry = inner.remove_pending(&key).unwrap();
drop(inner);
entry.settle_call_error(CallError::not_sent_with_cause(
"call deadline elapsed before request was sent",
NotSentCause::WriterRejected,
));
return false;
}
entry.accepted = true;
true
}
fn settle_pending(self: &Arc<Self>, key: PendingKey, terminal: PendingTerminal) {
let (entry, retry) = {
let mut inner = self.lock_inner();
let entry = inner.remove_pending(&key);
let slot = inner.wire_slots.remove(&key);
let busy = matches!(&terminal, PendingTerminal::Error { body, flags }
if error_codes::is_route_busy_retryable(*flags, &body.code, slot.as_ref().is_some_and(|slot| slot.opted_in())))
&& entry
.as_ref()
.is_some_and(|entry| !entry.completion_is_closed());
if let Some(slot) = slot {
if !busy {
slot.release();
}
}
if busy && entry.as_ref().is_some_and(|entry| entry.retry.is_some()) {
let mut entry = entry.unwrap();
entry.accepted = false;
let retry = entry.retry.clone();
inner.pending.insert(key, entry);
(None, retry)
} else {
inner.reap_retired_route(RouteHandle::new(key.channel, key.epoch, key.generation));
(entry, None)
}
};
if let Some(retry) = retry {
let shared = Arc::clone(self);
tokio::spawn(async move {
shared.retry_subscription(key, retry, terminal).await;
});
return;
}
let Some(entry) = entry else {
return;
};
if entry.retain_late_route_open && entry.completion_is_closed() {
if let PendingTerminal::Response {
generation, body, ..
} = &terminal
{
if let Ok(ClientControlResponse::RouteOpen {
route_channel,
route_epoch,
..
}) = serde_json::from_slice::<ClientControlResponse>(body)
{
let handle = RouteHandle::new(route_channel, route_epoch, *generation);
self.send_route_goodbye(handle, true);
self.uninstall_route_handle(handle);
}
}
return;
}
entry.settle_terminal(terminal);
}
fn pending_accepted(&self, key: PendingKey) -> Option<bool> {
self.lock_inner()
.pending
.get(&key)
.map(|entry| entry.accepted)
}
fn handle_generation_drop(self: &Arc<Self>, generation: u64, reason: String) {
self.handle_generation_drop_with_cause(
generation,
reason,
OutcomeUnknownCause::ConnectionFailed,
);
}
fn handle_generation_drop_with_cause(
self: &Arc<Self>,
generation: u64,
reason: String,
cause: OutcomeUnknownCause,
) {
let (should_emit, pending, callbacks) = {
let mut inner = self.lock_inner();
if inner.closed || inner.generation != generation || inner.writer.is_none() {
return;
}
inner.writer = None;
inner.restored_token = inner.restored_token.saturating_add(1);
inner.close_routes();
inner.route_epochs.clear();
inner.route_modules.clear();
inner.route_end_reasons.clear();
inner.legacy_channel_reasons.clear();
let pending = drain_pending_generation(&mut inner.pending, generation);
let callbacks = inner.callbacks.clone();
(true, pending, callbacks)
};
if should_emit {
settle_route_pending_entries_with_cause(
pending,
reason.clone(),
RouteEndReason::ConnectionLost,
cause,
);
emit_callbacks(callbacks, ConnectionState::Dropped);
self.notify.notify_waiters();
let _ = self.spawn_reconnect(generation);
}
}
fn close_sync(&self, reason: &str) {
let (pending, openings, routes, reader, writer, reconnect) = {
let mut inner = self.lock_inner();
if inner.closed {
return;
}
inner.closed = true;
let retirements = inner
.retirement_goodbyes
.iter()
.copied()
.collect::<Vec<_>>();
for handle in retirements {
inner.queue_retirement_goodbye(handle);
}
inner.writer = None;
inner.route_epochs.clear();
inner.route_modules.clear();
inner.route_end_reasons.clear();
inner.legacy_channel_reasons.clear();
inner.push_event_receivers.clear();
self.close_token.cancel();
let reconnect = match std::mem::replace(&mut inner.reconnect, ReconnectState::Idle) {
ReconnectState::Background { task, .. } => Some(task),
ReconnectState::Idle | ReconnectState::Inline { .. } => None,
};
(
inner
.pending
.drain()
.map(|(_, entry)| entry)
.collect::<Vec<_>>(),
inner
.openings
.drain()
.map(|(_, opening)| opening.waiters)
.collect::<Vec<_>>(),
inner.drain_routes(),
inner.reader_task.take(),
inner.writer_task.take(),
reconnect,
)
};
for route in routes {
route.close();
}
if let Some(handle) = reader {
handle.abort();
}
if let Some(mut handle) = writer {
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
runtime.spawn(async move {
if tokio::time::timeout(Duration::from_millis(250), &mut handle)
.await
.is_err()
{
handle.abort();
}
});
} else {
handle.abort();
}
}
if let Some(handle) = reconnect {
handle.abort();
}
settle_pending_entries(pending, reason.to_string());
fail_openings(openings, SharedCallFailure::not_sent(reason.to_string()));
self.notify.notify_waiters();
}
fn validate_current_handle(&self, handle: RouteHandle) -> Result<(), CallError> {
let inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| inner.writer.is_none()
|| !route_is_installed(&inner, handle)
{
Err(CallError::StaleRouteHandle(handle))
} else {
Ok(())
}
}
fn route_state(&self, handle: RouteHandle) -> Result<RouteState, CallError> {
let inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| inner.writer.is_none()
|| inner.route_epochs.get(&handle.channel) != Some(&handle)
{
return Err(CallError::StaleRouteHandle(handle));
}
let route = inner
.route_by_channel
.get(&handle.channel)
.and_then(|key| inner.routes.get(key))
.or_else(|| inner.one_shot_routes.get(&handle.channel));
debug_assert!(route.is_none_or(|route| route.handle == handle));
route
.filter(|route| route.handle == handle)
.cloned()
.ok_or(CallError::StaleRouteHandle(handle))
}
fn route_is_current(&self, key: &RouteKey, route: &RouteState) -> bool {
let inner = self.lock_inner();
if inner.closed
|| inner.generation != route.handle.connection_token()
|| inner.writer.is_none()
{
return false;
}
inner.routes.get(key).is_some_and(|cached| {
cached.handle == route.handle && Arc::ptr_eq(&cached.sem, &route.sem)
})
}
fn invalidate_route(&self, key: &RouteKey, expected_handle: Option<RouteHandle>) {
self.invalidate_route_with_deadline(key, expected_handle, false);
}
fn invalidate_route_with_deadline(
&self,
key: &RouteKey,
expected_handle: Option<RouteHandle>,
response_deadline_expired: bool,
) {
let removed = {
let mut inner = self.lock_inner();
match inner.routes.get(key) {
Some(route) if expected_handle.is_none_or(|expected| expected == route.handle) => {
let removed = inner.evict_route(key);
if let Some(route) = &removed {
inner.retired_routes.insert(route.handle);
if response_deadline_expired {
inner.retirement_goodbyes.insert(route.handle);
inner.release_wire_slots(route.handle);
let pending_calls = inner
.pending
.keys()
.filter(|pending| {
pending.generation == route.handle.connection_token()
&& pending.channel == route.handle.channel
&& pending.epoch == route.handle.epoch
})
.count();
tracing::warn!(
target_module = %key.target_label(),
channel = route.handle.channel,
epoch = route.handle.epoch,
pending_calls,
"retiring route after call response deadline"
);
}
inner.reap_retired_route(route.handle);
}
removed
}
_ => None,
}
};
if let Some(route) = removed {
route.close();
}
}
fn finish_opening(&self, key: &RouteKey, result: Result<RouteState, SharedCallFailure>) {
let opening = self.lock_inner().openings.remove(key);
for waiter in opening.map(|o| o.waiters).unwrap_or_default() {
let _ = waiter.send(result.clone());
}
}
async fn close_handle(
self: &Arc<Self>,
handle: RouteHandle,
opts: &CloseRouteOptions,
) -> Result<(), CallError> {
self.validate_current_handle(handle)?;
let route = {
let mut inner = self.lock_inner();
inner.route_end_reasons.insert(
handle.channel,
(RouteEndReason::ClosedByCaller.into(), true),
);
inner.remove_route_by_handle(handle)
};
let mut closing = RouteCloseGuard::new(
Arc::clone(self),
handle,
route,
"route closed by close_handle",
);
if opts.drain {
self.drain_channel(handle, opts.drain_timeout).await;
}
closing.close_pending();
self.queue_route_goodbye(handle).await;
closing.finish();
Ok(())
}
async fn close_route(self: &Arc<Self>, key: &RouteKey, opts: &CloseRouteOptions) {
let route = {
let mut inner = self.lock_inner();
if let Some(opening) = inner.openings.get_mut(key) {
opening.closed = true;
}
let route = inner.remove_route(key);
if let Some(route) = &route {
inner.route_end_reasons.insert(
route.handle.channel,
(RouteEndReason::ClosedByCaller.into(), true),
);
}
route
};
let Some(route) = route else {
return;
};
let handle = route.handle;
let mut closing = RouteCloseGuard::new(
Arc::clone(self),
handle,
Some(route),
"route closed by close_route",
);
if opts.drain {
self.drain_channel(handle, opts.drain_timeout).await;
}
closing.close_pending();
self.queue_route_goodbye(handle).await;
closing.finish();
}
fn uninstall_route_handle(&self, handle: RouteHandle) {
let mut inner = self.lock_inner();
inner.release_wire_slots(handle);
inner.retired_routes.remove(&handle);
inner.retirement_goodbyes.remove(&handle);
if inner.route_epochs.get(&handle.channel) == Some(&handle) {
let installed = inner.route_epochs.remove(&handle.channel).unwrap();
inner.route_targets.remove(&handle);
inner.push_event_receivers.remove(&handle);
release_reverse_request_registry(installed);
}
}
fn route_not_sent_error(&self, handle: RouteHandle) -> CallError {
let inner = self.lock_inner();
let reason = if inner.generation != handle.connection_token() || inner.writer.is_none() {
RouteEndReason::ConnectionLost.into()
} else {
inner.route_end(handle.channel)
};
CallError::NotSent(Box::new(NotSentSource {
message: "route flow-control semaphore closed".into(),
cause: NotSentCause::BeforeQueue,
never_forwarded: true,
route_reason: Some(reason),
refusal: None,
}))
}
fn fail_channel_pending(
&self,
handle: RouteHandle,
reason: &str,
close_reason: RouteEndReason,
) {
let entries = {
let mut inner = self.lock_inner();
inner.release_wire_slots(handle);
inner
.route_end_reasons
.insert(handle.channel, (close_reason.clone().into(), true));
drain_pending_handle(&mut inner.pending, handle, true)
};
settle_route_pending_entries(entries, reason.to_string(), close_reason);
}
async fn drain_channel(&self, handle: RouteHandle, timeout: Duration) {
let deadline = Instant::now() + timeout;
loop {
let has_inflight = {
let inner = self.lock_inner();
inner.pending.iter().any(|(key, entry)| {
key.generation == handle.connection_token()
&& key.channel == handle.channel
&& key.epoch == handle.epoch
&& !entry.is_subscription()
})
};
if !has_inflight || Instant::now() >= deadline {
return;
}
sleep(Duration::from_millis(5)).await;
}
}
async fn queue_route_goodbye(self: &Arc<Self>, handle: RouteHandle) {
let frame = Frame::build(
FrameType::Goodbye,
Flags::new(false, Priority::Interactive, false),
handle.channel,
handle.epoch,
0,
Vec::new(),
)
.expect("header-only route GOODBYE is valid");
loop {
let changed = self.notify.notified();
tokio::pin!(changed);
changed.as_mut().enable();
let writer = {
let inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| inner.route_epochs.get(&handle.channel) != Some(&handle)
{
return;
}
let Some(writer) = inner.writer.clone() else {
return;
};
writer
};
tokio::select! {
result = writer.reserve() => {
let Ok(permit) = result else {
self.handle_generation_drop(
handle.connection_token(),
"writer closed before route GOODBYE".into(),
);
return;
};
let inner = self.lock_inner();
if !inner.closed
&& inner.generation == handle.connection_token()
&& inner.writer.as_ref().is_some_and(|current| current.same_channel(&writer))
&& inner.route_epochs.get(&handle.channel) == Some(&handle)
{
permit.send(WriteCommand { frame, pending: None });
}
return;
}
_ = &mut changed => {}
_ = self.close_token.cancelled() => return,
}
}
}
fn send_route_goodbye(self: &Arc<Self>, handle: RouteHandle, close_on_failure: bool) -> bool {
let writer = {
let inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| inner.route_epochs.get(&handle.channel) != Some(&handle)
{
return false;
}
inner.writer.clone()
};
let Some(writer) = writer else {
return false;
};
let Ok(frame) = Frame::build(
FrameType::Goodbye,
Flags::new(false, Priority::Interactive, false),
handle.channel,
handle.epoch,
0,
Vec::new(),
) else {
return false;
};
if writer
.try_send(WriteCommand {
frame,
pending: None,
})
.is_ok()
{
return true;
}
if close_on_failure {
self.handle_generation_drop(
handle.connection_token(),
"failed to queue late route.open cleanup GOODBYE".to_string(),
);
}
false
}
fn emit_connection_state(&self, state: ConnectionState) {
let callbacks = self.lock_inner().callbacks.clone();
emit_callbacks(callbacks, state);
}
}
#[derive(Clone, Copy)]
enum InstallKind {
Initial,
Reconnect,
}
enum EnsureAction {
Wait,
Lead {
generation: u64,
stale_task: Option<JoinHandle<()>>,
},
}
enum ReconnectState {
Idle,
Inline {
generation: u64,
},
Background {
generation: u64,
task: JoinHandle<()>,
},
}
enum RouteInstall {
Cached(RouteState),
Discard { closed: bool },
}
enum RouteOpenAction {
Wait(oneshot::Receiver<Result<RouteState, SharedCallFailure>>),
Lead,
}
struct RouteOpenParams<'a> {
target: &'a RouteTarget,
identity: &'a BindIdentity,
consumer_identity: &'a Option<ConsumerIdentity>,
consumer_capabilities: &'a Option<Vec<String>>,
role_versions: &'a Option<BTreeMap<String, String>>,
reverse_requests: &'a ReverseRequestRegistry,
scope: Option<&'a ScopeSelector>,
}
struct CallAttemptHistory {
never_forwarded: bool,
sent: bool,
}
struct RequestSend {
expected_handle: Option<RouteHandle>,
channel: u16,
epoch: u32,
body: Vec<u8>,
binary: bool,
priority: Priority,
admission_class: AdmissionClass,
deadline: Instant,
retain_late_route_open: bool,
route_open_reverse_requests: Option<ReverseRequestRegistry>,
}
struct SubscriptionSend {
expected_handle: Option<RouteHandle>,
channel: u16,
epoch: u32,
body: Vec<u8>,
priority: Priority,
admission_class: AdmissionClass,
event_buffer: usize,
deadline: Instant,
permit: Arc<RoutePermit>,
}
struct SubscriptionPosition {
key: Mutex<PendingKey>,
stop: CancellationToken,
}
#[derive(Clone)]
struct SubscriptionRetry {
frame: Frame,
deadline: Instant,
delay: Duration,
position: Arc<SubscriptionPosition>,
}
struct RouteCloseGuard {
shared: Arc<Shared>,
handle: RouteHandle,
route: Option<RouteState>,
reason: &'static str,
finished: bool,
}
impl RouteCloseGuard {
fn new(
shared: Arc<Shared>,
handle: RouteHandle,
route: Option<RouteState>,
reason: &'static str,
) -> Self {
Self {
shared,
handle,
route,
reason,
finished: false,
}
}
fn close_pending(&self) {
if let Some(route) = &self.route {
route.close();
}
self.shared
.fail_channel_pending(self.handle, self.reason, RouteEndReason::ClosedByCaller);
}
fn finish(&mut self) {
self.shared.uninstall_route_handle(self.handle);
self.finished = true;
}
}
impl Drop for RouteCloseGuard {
fn drop(&mut self) {
if self.finished {
return;
}
self.close_pending();
let mut inner = self.shared.lock_inner();
if inner.closed
|| inner.generation != self.handle.connection_token()
|| inner.writer.is_none()
|| inner.route_epochs.get(&self.handle.channel) != Some(&self.handle)
{
return;
}
inner.retired_routes.insert(self.handle);
inner.retirement_goodbyes.insert(self.handle);
inner.reap_retired_route(self.handle);
}
}
struct OpeningGuard {
shared: Arc<Shared>,
key: RouteKey,
finished: bool,
}
impl OpeningGuard {
fn new(shared: Arc<Shared>, key: RouteKey) -> Self {
Self {
shared,
key,
finished: false,
}
}
fn finish(&mut self, result: Result<RouteState, SharedCallFailure>) {
self.shared.finish_opening(&self.key, result);
self.finished = true;
}
}
impl Drop for OpeningGuard {
fn drop(&mut self) {
if !self.finished {
self.shared.finish_opening(
&self.key,
Err(SharedCallFailure::not_sent(
"route.open future was cancelled",
)),
);
}
}
}
struct InlineReconnectGuard {
shared: Arc<Shared>,
generation: u64,
finished: bool,
}
impl InlineReconnectGuard {
fn new(shared: Arc<Shared>, generation: u64) -> Self {
Self {
shared,
generation,
finished: false,
}
}
fn finish(&mut self) {
self.shared.finish_inline_reconnect(self.generation);
self.finished = true;
}
}
impl Drop for InlineReconnectGuard {
fn drop(&mut self) {
if !self.finished {
self.shared.finish_inline_reconnect(self.generation);
}
}
}
#[derive(Clone)]
struct RouteState {
handle: RouteHandle,
sem: Arc<Semaphore>,
watch_sem: Option<Arc<Semaphore>>,
opted_in: bool,
}
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
struct RouteKey {
target: RouteTargetKey,
harness: String,
session: String,
consumer_identity: Option<ConsumerIdentityKey>,
consumer_capabilities: Option<ConsumerCapabilitiesKey>,
role_versions: Option<BTreeMap<String, String>>,
scope: Option<ScopeKey>,
}
impl RouteKey {
fn new(
target: &RouteTarget,
identity: &BindIdentity,
consumer_identity: Option<&ConsumerIdentity>,
consumer_capabilities: Option<&[String]>,
) -> Self {
Self {
target: RouteTargetKey::from(target),
harness: identity.harness.clone(),
session: identity.session.clone(),
consumer_identity: consumer_identity.map(ConsumerIdentityKey::from),
consumer_capabilities: consumer_capabilities.map(ConsumerCapabilitiesKey::from_slice),
role_versions: None,
scope: None,
}
}
fn with_role_versions(mut self, role_versions: Option<&BTreeMap<String, String>>) -> Self {
self.role_versions = role_versions.cloned();
self
}
fn with_scope(mut self, scope: Option<&ScopeSelector>) -> Self {
self.scope = scope.map(ScopeKey::from);
self
}
fn target_label(&self) -> String {
match &self.target {
RouteTargetKey::ToolProvider { module_id } => format!("tool_provider:{module_id}"),
RouteTargetKey::ManagementSurface { module_id } => {
format!("management_surface:{module_id}")
}
RouteTargetKey::InternalService {
module_id,
service_id,
} => format!("internal_service:{module_id}:{service_id}"),
}
}
}
#[derive(Clone, Hash, PartialEq, Eq)]
struct ConsumerIdentityKey {
module_id: String,
launch_nonce: String,
}
impl fmt::Debug for ConsumerIdentityKey {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ConsumerIdentityKey")
.field("module_id", &self.module_id)
.field(
"launch_nonce",
&format_args!("<{} bytes redacted>", self.launch_nonce.len()),
)
.finish()
}
}
impl From<&ConsumerIdentity> for ConsumerIdentityKey {
fn from(value: &ConsumerIdentity) -> Self {
Self {
module_id: value.module_id.clone(),
launch_nonce: value.launch_nonce.clone(),
}
}
}
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
struct ScopeKey {
owner: String,
scope_ref: String,
scope_epoch: Option<u64>,
bind_proof_digest: Option<[u8; 32]>,
}
impl From<&ScopeSelector> for ScopeKey {
fn from(value: &ScopeSelector) -> Self {
Self {
owner: serde_json::to_string(&value.owner)
.unwrap_or_else(|_| format!("{:?}", value.owner)),
scope_ref: value.scope_ref.clone(),
scope_epoch: value.scope_epoch,
bind_proof_digest: value
.bind_proof
.as_ref()
.map(|proof| Sha256::digest(proof.as_bytes()).into()),
}
}
}
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
struct ConsumerCapabilitiesKey {
values: Vec<String>,
}
impl ConsumerCapabilitiesKey {
fn from_slice(values: &[String]) -> Self {
let mut values = values.to_vec();
values.sort();
values.dedup();
Self { values }
}
}
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
enum RouteTargetKey {
ToolProvider {
module_id: String,
},
ManagementSurface {
module_id: String,
},
InternalService {
module_id: String,
service_id: String,
},
}
impl From<&RouteTarget> for RouteTargetKey {
fn from(value: &RouteTarget) -> Self {
match value {
RouteTarget::ToolProvider { module_id } => Self::ToolProvider {
module_id: module_id.clone(),
},
RouteTarget::ManagementSurface { module_id } => Self::ManagementSurface {
module_id: module_id.clone(),
},
RouteTarget::InternalService {
module_id,
service_id,
} => Self::InternalService {
module_id: module_id.clone(),
service_id: service_id.clone(),
},
}
}
}
#[derive(Debug, Clone)]
struct SharedCallFailure {
kind: FailureKind,
}
impl SharedCallFailure {
fn not_sent(message: impl Into<String>) -> Self {
CallError::not_sent_with_cause(message, NotSentCause::RouteOpenAbandoned).into()
}
fn into_call_error(self) -> CallError {
match self.kind {
FailureKind::NotSent(source) => CallError::NotSent(Box::new(source)),
FailureKind::OutcomeUnknown(source) => CallError::outcome_unknown_source(source),
FailureKind::DaemonTooOld(daemon_ver) => CallError::DaemonTooOld { daemon_ver },
}
}
}
impl From<CallError> for SharedCallFailure {
fn from(value: CallError) -> Self {
match value {
CallError::DaemonTooOld { daemon_ver } => Self {
kind: FailureKind::DaemonTooOld(daemon_ver),
},
CallError::NotSent(err) => Self {
kind: FailureKind::NotSent(
err.downcast_ref::<NotSentSource>()
.expect("crate-built route.open failure has a typed source")
.clone(),
),
},
CallError::OutcomeUnknown(err) => Self {
kind: FailureKind::OutcomeUnknown(
err.downcast_ref::<OutcomeUnknownSource>()
.cloned()
.unwrap_or_else(|| OutcomeUnknownSource {
message: err.to_string(),
cause: OutcomeUnknownCause::CompletionFailed,
route_reason: None,
}),
),
},
CallError::Module(body) => Self {
kind: FailureKind::OutcomeUnknown(OutcomeUnknownSource {
message: format!(
"unexpected module error during route.open: {} ({})",
body.code, body.message
),
cause: OutcomeUnknownCause::CompletionFailed,
route_reason: None,
}),
},
CallError::SubscriptionBackpressure(err) => Self {
kind: FailureKind::OutcomeUnknown(OutcomeUnknownSource {
message: err.to_string(),
cause: OutcomeUnknownCause::CompletionFailed,
route_reason: None,
}),
},
CallError::StaleRouteHandle(handle) => {
CallError::not_sent(format!("stale route handle: {handle:?}")).into()
}
error @ (CallError::CapabilityUnprovided { .. }
| CallError::CapabilityAmbiguous { .. }
| CallError::InvalidCapabilityIdentifier { .. }) => {
CallError::not_sent(error.to_string()).into()
}
}
}
}
#[derive(Debug, Clone)]
enum FailureKind {
NotSent(NotSentSource),
OutcomeUnknown(OutcomeUnknownSource),
DaemonTooOld(String),
}
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
struct PendingKey {
generation: u64,
channel: u16,
epoch: u32,
corr: u64,
}
struct PendingEntry {
accepted: bool,
send_deadline: Option<Instant>,
retain_late_route_open: bool,
expected_control_handle: Option<RouteHandle>,
route_open_reverse_requests: Option<ReverseRequestRegistry>,
route_open_module: Option<String>,
retry: Option<SubscriptionRetry>,
completion: PendingCompletion,
}
enum PendingCompletion {
Unary(oneshot::Sender<PendingResult>),
Subscription {
events: mpsc::Sender<Vec<u8>>,
closed: oneshot::Sender<Result<(), CallError>>,
ended: watch::Sender<Option<SubscriptionEnd>>,
_permit: Arc<RoutePermit>,
priority: Priority,
},
}
enum StreamDataDelivery {
NotSubscription,
Full,
Closed,
}
impl PendingEntry {
fn unary(
tx: oneshot::Sender<PendingResult>,
retain_late_route_open: bool,
expected_control_handle: Option<RouteHandle>,
route_open_reverse_requests: Option<ReverseRequestRegistry>,
) -> Self {
Self {
accepted: false,
send_deadline: None,
retain_late_route_open,
expected_control_handle,
route_open_reverse_requests,
route_open_module: None,
retry: None,
completion: PendingCompletion::Unary(tx),
}
}
fn subscription(
events: mpsc::Sender<Vec<u8>>,
closed: oneshot::Sender<Result<(), CallError>>,
ended: watch::Sender<Option<SubscriptionEnd>>,
permit: Arc<RoutePermit>,
priority: Priority,
) -> Self {
Self {
accepted: false,
send_deadline: None,
retain_late_route_open: false,
expected_control_handle: None,
route_open_reverse_requests: None,
route_open_module: None,
retry: None,
completion: PendingCompletion::Subscription {
events,
closed,
ended,
_permit: permit,
priority,
},
}
}
fn completion_is_closed(&self) -> bool {
match &self.completion {
PendingCompletion::Unary(tx) => tx.is_closed(),
PendingCompletion::Subscription { closed, .. } => closed.is_closed(),
}
}
fn is_subscription(&self) -> bool {
matches!(&self.completion, PendingCompletion::Subscription { .. })
}
fn subscription_priority(&self) -> Option<Priority> {
match &self.completion {
PendingCompletion::Subscription { priority, .. } => Some(*priority),
PendingCompletion::Unary(_) => None,
}
}
fn try_send_stream_data(&self, body: Vec<u8>) -> Result<(), StreamDataDelivery> {
let PendingCompletion::Subscription { events, .. } = &self.completion else {
return Err(StreamDataDelivery::NotSubscription);
};
events.try_send(body).map_err(|err| match err {
mpsc::error::TrySendError::Full(_) => StreamDataDelivery::Full,
mpsc::error::TrySendError::Closed(_) => StreamDataDelivery::Closed,
})
}
fn settle_terminal(self, terminal: PendingTerminal) {
if let Some(retry) = &self.retry {
retry.position.stop.cancel();
}
match self.completion {
PendingCompletion::Unary(tx) => {
let _ = tx.send(PendingResult::Terminal(terminal));
}
PendingCompletion::Subscription { closed, ended, .. } => {
let (end, result) = match terminal {
PendingTerminal::Response { body, .. } => {
(SubscriptionEnd::Response(body), Ok(()))
}
PendingTerminal::StreamEnd => (SubscriptionEnd::StreamEnd, Ok(())),
PendingTerminal::Error { body, flags } => {
let err = if error_codes::is_route_busy_retryable(
flags,
&body.code,
self.retry.is_some(),
) {
busy_error(&body)
} else {
CallError::Module(body.clone())
};
(SubscriptionEnd::Error(body.clone()), Err(err))
}
};
let _ = ended.send(Some(end));
let _ = closed.send(result);
}
}
}
fn settle_failure(self, reason: String) {
if let Some(retry) = &self.retry {
retry.position.stop.cancel();
}
let err = classify_failure(self.accepted, reason, OutcomeUnknownCause::ConsumerClosed);
match self.completion {
PendingCompletion::Unary(tx) => {
let _ = tx.send(PendingResult::CallError(err));
}
PendingCompletion::Subscription { closed, ended, .. } => {
let _ = ended.send(Some(SubscriptionEnd::RouteClosed {
reason: RouteEndReason::ClosedByCaller,
}));
let _ = closed.send(Err(err));
}
}
}
fn settle_call_error(self, err: CallError) {
if let Some(retry) = &self.retry {
retry.position.stop.cancel();
}
match self.completion {
PendingCompletion::Unary(tx) => {
let _ = tx.send(PendingResult::CallError(err));
}
PendingCompletion::Subscription { closed, ended, .. } => {
let end = if let Some(reason) = err.close_reason() {
match reason {
RouteEndReason::ConnectionLost => SubscriptionEnd::ConnectionLost,
reason => SubscriptionEnd::RouteClosed {
reason: reason.clone(),
},
}
} else if matches!(err, CallError::SubscriptionBackpressure(_)) {
SubscriptionEnd::Backpressure
} else {
SubscriptionEnd::ConnectionLost
};
let _ = ended.send(Some(end));
let _ = closed.send(Err(err));
}
}
}
fn settle_subscription_result(self, result: Result<(), CallError>) {
if let Some(retry) = &self.retry {
retry.position.stop.cancel();
}
match self.completion {
PendingCompletion::Subscription { closed, ended, .. } => {
let _ = ended.send(Some(SubscriptionEnd::Cancelled));
let _ = closed.send(result);
}
PendingCompletion::Unary(tx) => {
let _ = tx.send(PendingResult::Failure {
accepted: self.accepted,
reason: "subscription cancel matched a unary request".to_string(),
});
}
}
}
}
struct PendingRegistration {
shared: Arc<Shared>,
key: PendingKey,
active: bool,
retain_on_drop: bool,
unary_cancel: Option<Arc<UnaryCancel>>,
}
impl PendingRegistration {
fn new(shared: Arc<Shared>, key: PendingKey, retain_on_drop: bool) -> Self {
Self {
shared,
key,
active: true,
retain_on_drop,
unary_cancel: None,
}
}
fn remove_pending(&mut self) -> Option<bool> {
if !self.active {
return None;
}
self.active = false;
self.shared
.lock_inner()
.remove_pending(&self.key)
.map(|entry| entry.accepted)
}
fn disarm(&mut self) {
self.active = false;
}
}
impl Drop for PendingRegistration {
fn drop(&mut self) {
if self.retain_on_drop {
self.disarm();
} else {
let _ = self.remove_pending();
}
}
}
enum PendingResult {
Terminal(PendingTerminal),
CallError(CallError),
Failure { accepted: bool, reason: String },
}
impl PendingResult {
fn into_call_result(self) -> Result<TerminalFrame, CallError> {
match self {
Self::Terminal(terminal) => Ok(terminal.into_terminal_frame()),
Self::CallError(err) => Err(err),
Self::Failure { accepted, reason } => Err(classify_failure(
accepted,
reason,
OutcomeUnknownCause::CompletionFailed,
)),
}
}
}
enum PendingTerminal {
Response {
generation: u64,
body: Vec<u8>,
flags: Flags,
},
Error {
body: ErrorBody,
flags: Flags,
},
StreamEnd,
}
impl PendingTerminal {
fn into_terminal_frame(self) -> TerminalFrame {
match self {
Self::Response {
generation,
body,
flags,
} => TerminalFrame::Response {
generation,
body,
flags,
},
Self::Error { body, flags } => TerminalFrame::Error { body, flags },
Self::StreamEnd => TerminalFrame::StreamEnd,
}
}
}
#[derive(Debug)]
enum TerminalFrame {
Response {
generation: u64,
body: Vec<u8>,
flags: Flags,
},
Error {
body: ErrorBody,
flags: Flags,
},
StreamEnd,
}
struct WriteCommand {
frame: Frame,
pending: Option<PendingKey>,
}
trait ConsumerStream: AsyncRead + AsyncWrite + Unpin + Send {}
impl<T: AsyncRead + AsyncWrite + Unpin + Send> ConsumerStream for T {}
struct OpenedConnection {
stream: Box<dyn ConsumerStream>,
daemon_origin_trusted: bool,
daemon_ver: String,
}
async fn open_connection(
path: &Path,
deadline: Duration,
transport: ConsumerTransport,
operator_candidates: Option<&[PathBuf]>,
) -> Result<OpenedConnection, ConsumerError> {
let path = operator_candidates
.map(crate::operator_app::owned_candidate)
.unwrap_or(path);
if transport == ConsumerTransport::Local {
let read =
connection_file::read_for_client_with_local_endpoint(path).map_err(|source| {
ConsumerError::ConnectionFile {
path: path.to_path_buf(),
source,
}
})?;
let endpoint = read
.local_endpoint
.ok_or_else(|| ConsumerError::NoEndpoint {
path: path.to_path_buf(),
})?;
let label = match &endpoint {
connection_file::LocalEndpoint::Unix { path } => path.display().to_string(),
connection_file::LocalEndpoint::Pipe { name } => name.clone(),
_ => "unsupported local endpoint".to_owned(),
};
let mut stream =
connect_local(&endpoint)
.await
.map_err(|source| ConsumerError::Connect {
path: path.to_path_buf(),
endpoint: label.clone(),
source,
})?;
authenticate_client(&mut stream, &read.info, deadline)
.await
.map_err(|source| ConsumerError::Auth {
path: path.to_path_buf(),
endpoint: label,
source,
})?;
crate::registration::bind_consumer(&mut stream, deadline)
.await
.map_err(ConsumerError::claim)?;
return Ok(OpenedConnection {
stream,
daemon_origin_trusted: trusts_daemon_origin(&read.info.daemon_ver),
daemon_ver: read.info.daemon_ver,
});
}
let conn =
connection_file::read_for_client(path).map_err(|source| ConsumerError::ConnectionFile {
path: path.to_path_buf(),
source,
})?;
open_connection_with_info(path, &conn, deadline).await
}
async fn connect_local(
endpoint: &connection_file::LocalEndpoint,
) -> io::Result<Box<dyn ConsumerStream>> {
match endpoint {
#[cfg(unix)]
connection_file::LocalEndpoint::Unix { path } => {
Ok(Box::new(tokio::net::UnixStream::connect(path).await?))
}
#[cfg(windows)]
connection_file::LocalEndpoint::Pipe { name } => Ok(Box::new(
tokio::net::windows::named_pipe::ClientOptions::new().open(name)?,
)),
_ => Err(io::Error::new(
io::ErrorKind::Unsupported,
"unsupported local endpoint kind",
)),
}
}
async fn open_connection_with_info(
path: &Path,
conn: &connection_file::ConnectionInfo,
deadline: Duration,
) -> Result<OpenedConnection, ConsumerError> {
let endpoint = conn
.endpoints
.first()
.ok_or_else(|| ConsumerError::NoEndpoint {
path: path.to_path_buf(),
})?;
let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
let mut stream = TcpStream::connect(&endpoint_label)
.await
.map_err(|source| ConsumerError::Connect {
path: path.to_path_buf(),
endpoint: endpoint_label.clone(),
source,
})?;
let _ = stream.set_nodelay(true);
authenticate_client(&mut stream, conn, deadline)
.await
.map_err(|source| ConsumerError::Auth {
path: path.to_path_buf(),
endpoint: endpoint_label,
source,
})?;
crate::registration::bind_consumer(&mut stream, deadline)
.await
.map_err(ConsumerError::claim)?;
Ok(OpenedConnection {
stream: Box::new(stream),
daemon_origin_trusted: trusts_daemon_origin(&conn.daemon_ver),
daemon_ver: conn.daemon_ver.clone(),
})
}
fn dispatch_reverse_request(shared: &Arc<Shared>, frame: Frame, handle: RouteHandle) {
let method = serde_json::from_slice::<serde_json::Value>(&frame.body)
.ok()
.and_then(|body| body.get("method")?.as_str().map(str::to_string));
let method_family = method
.as_deref()
.and_then(|method| method.split('/').next())
.filter(|family| !family.is_empty())
.map(str::to_string);
let handler = method_family.as_deref().and_then(|family| {
route_reverse_request_registry(handle.reverse_request_registry_id())
.ok()?
.handler(family)
});
let generation = handle.connection_token();
let shared = Arc::clone(shared);
tokio::spawn(async move {
let (frame_type, body) = if let (Some(handler), Some(method)) = (handler, method) {
let request_body = frame.body.clone();
let corr = frame.header.corr;
match tokio::spawn(async move {
handler(request_body, ReverseRequestContext { corr, method }).await
})
.await
{
Ok(Ok(body)) => (FrameType::Response, body),
Ok(Err(err)) => (
FrameType::Error,
serde_json::to_vec(&ErrorBody::new(REVERSE_REQUEST_UNHANDLED, err.to_string()))
.unwrap_or_default(),
),
Err(err) => (
FrameType::Error,
serde_json::to_vec(&ErrorBody::new(
REVERSE_REQUEST_UNHANDLED,
join_error_message(err),
))
.unwrap_or_default(),
),
}
} else {
let message = method_family.map_or_else(
|| "reverse request has no valid method".to_string(),
|family| format!("no reverse-request handler is registered for {family}"),
);
(
FrameType::Error,
serde_json::to_vec(&ErrorBody::new(REVERSE_REQUEST_UNHANDLED, message))
.unwrap_or_default(),
)
};
if shared.ingress_handle(generation, handle.channel, handle.epoch) != Some(handle) {
return;
}
let reply = match Frame::build_with_version(
frame.header.ver,
frame_type,
Flags::new(false, Priority::Interactive, false),
handle.channel,
handle.epoch,
frame.header.corr,
body,
) {
Ok(reply) => reply,
Err(_) => return,
};
let writer = shared.lock_inner().writer.clone();
if let Some(writer) = writer {
let _ = writer
.send(WriteCommand {
frame: reply,
pending: None,
})
.await;
}
});
}
fn join_error_message(error: tokio::task::JoinError) -> String {
if !error.is_panic() {
return error.to_string();
}
let payload = error.into_panic();
if let Some(message) = payload.downcast_ref::<String>() {
return message.clone();
}
if let Some(message) = payload.downcast_ref::<&str>() {
return (*message).to_string();
}
"reverse-request handler panicked".to_string()
}
async fn reader_loop<R: AsyncRead + Unpin>(shared: Arc<Shared>, mut reader: R, generation: u64) {
loop {
match read_frame(&mut reader).await {
Ok(Some(frame)) => {
if !dispatch_frame(&shared, generation, frame).await {
return;
}
}
Ok(None) => {
shared.handle_generation_drop(generation, "subc connection closed".to_string());
return;
}
Err(err) => {
shared.handle_generation_drop(generation, err.to_string());
return;
}
}
}
}
async fn dispatch_frame(shared: &Arc<Shared>, generation: u64, frame: Frame) -> bool {
if !shared.record_inbound_if_current(generation) {
return false;
}
if frame.header.channel != 0
&& !shared.validate_ingress_handle(generation, frame.header.channel, frame.header.epoch)
{
return true;
}
if frame.header.ty == FrameType::Request && frame.header.channel != 0 {
if let Some(handle) =
shared.ingress_handle(generation, frame.header.channel, frame.header.epoch)
{
dispatch_reverse_request(shared, frame, handle);
}
return true;
}
let key = PendingKey {
generation,
channel: frame.header.channel,
epoch: frame.header.epoch,
corr: frame.header.corr,
};
if frame.header.channel == 0 && frame.header.ty == FrameType::Response {
if let Some(expected) = shared.pending_expected_control_handle(key) {
let echoes_expected = matches!(
serde_json::from_slice::<ClientControlResponse>(&frame.body),
Ok(ClientControlResponse::RoutePoll {
route_channel,
route_epoch,
..
}) if route_channel == expected.channel && route_epoch == expected.epoch
);
if !echoes_expected {
shared.count_dropped_route_frame();
return true;
}
}
}
if frame.header.channel == 0
&& frame.header.ty == FrameType::Response
&& shared.pending_expects_route_open(key)
{
if let Ok((
ClientControlResponse::RouteOpen {
route_channel,
route_epoch,
..
},
authority,
)) = decode_route_open_response(&frame.body)
{
let reverse_requests = shared
.pending_route_open_reverse_requests(key)
.unwrap_or_default();
let module = shared
.lock_inner()
.pending
.get(&key)
.and_then(|entry| entry.route_open_module.clone());
let handle =
RouteHandle::new_consumer(route_channel, route_epoch, generation, reverse_requests);
shared.install_ingress_handle(handle);
if let Some(target) = authority.target {
let mut inner = shared.lock_inner();
if inner.route_epochs.get(&handle.channel) == Some(&handle) {
inner.route_targets.insert(handle, target);
}
}
if let Some(module) = module {
shared
.lock_inner()
.route_modules
.insert(route_channel, module);
}
}
}
match frame.header.ty {
FrameType::Response => shared.settle_pending(
key,
PendingTerminal::Response {
generation,
body: frame.body,
flags: frame.header.flags,
},
),
FrameType::Error => {
let body =
serde_json::from_slice::<ErrorBody>(&frame.body).unwrap_or_else(|err| ErrorBody {
code: "invalid_error_body".to_string(),
message: err.to_string(),
detail: None,
});
shared.settle_pending(
key,
PendingTerminal::Error {
body,
flags: frame.header.flags,
},
);
}
FrameType::StreamEnd => shared.settle_pending(key, PendingTerminal::StreamEnd),
FrameType::StreamData => shared.route_stream_data(key, frame.body),
FrameType::Push if frame.header.channel == 0 => {
shared.control_push(&frame.body);
}
FrameType::Push => shared.route_push(
RouteHandle::new(frame.header.channel, frame.header.epoch, generation),
frame.body,
),
FrameType::Goodbye if frame.header.channel == 0 => {
shared.handle_generation_drop(generation, "subc sent GOODBYE".to_string());
return false;
}
FrameType::Goodbye => {
let handle = RouteHandle::new(frame.header.channel, frame.header.epoch, generation);
let close_reason = shared.lock_inner().route_end(handle.channel);
shared.invalidate_routes_for_handle(handle);
let pending = {
let mut inner = shared.lock_inner();
drain_pending_handle(&mut inner.pending, handle, true)
};
settle_route_pending_entries(pending, "route closed by subc".to_string(), close_reason);
}
FrameType::Ping if frame.header.channel == 0 => {
if let Ok(pong) = Frame::build_with_version(
frame.header.ver,
FrameType::Pong,
frame.header.flags,
0,
0,
frame.header.corr,
Vec::new(),
) {
let writer = shared.lock_inner().writer.clone();
if let Some(writer) = writer {
let _ = writer
.send(WriteCommand {
frame: pong,
pending: None,
})
.await;
}
}
}
_ => {}
}
true
}
impl Shared {
fn record_inbound_if_current(&self, generation: u64) -> bool {
let inner = self.lock_inner();
if inner.closed || inner.generation != generation || inner.writer.is_none() {
return false;
}
self.last_inbound_ms
.store(epoch_millis(), Ordering::Release);
true
}
fn generation_is_current(&self, generation: u64) -> bool {
let inner = self.lock_inner();
!inner.closed && inner.generation == generation && inner.writer.is_some()
}
fn pending_expected_control_handle(&self, key: PendingKey) -> Option<RouteHandle> {
self.lock_inner()
.pending
.get(&key)
.and_then(|entry| entry.expected_control_handle)
}
fn count_dropped_route_frame(&self) {
let mut inner = self.lock_inner();
inner.dropped_route_frames = inner.dropped_route_frames.saturating_add(1);
}
fn pending_expects_route_open(&self, key: PendingKey) -> bool {
self.lock_inner()
.pending
.get(&key)
.is_some_and(|entry| entry.retain_late_route_open)
}
fn pending_route_open_reverse_requests(
&self,
key: PendingKey,
) -> Option<ReverseRequestRegistry> {
self.lock_inner()
.pending
.get(&key)
.and_then(|entry| entry.route_open_reverse_requests.clone())
}
fn ingress_handle(&self, generation: u64, channel: u16, epoch: u32) -> Option<RouteHandle> {
let inner = self.lock_inner();
let handle = inner.route_epochs.get(&channel).copied()?;
(handle.connection_token() == generation && handle.epoch == epoch).then_some(handle)
}
fn validate_ingress_handle(&self, generation: u64, channel: u16, epoch: u32) -> bool {
let mut inner = self.lock_inner();
let expected = RouteHandle::new(channel, epoch, generation);
if inner.route_epochs.get(&channel) == Some(&expected) {
true
} else {
inner.dropped_route_frames = inner.dropped_route_frames.saturating_add(1);
false
}
}
fn install_ingress_handle(&self, handle: RouteHandle) {
let mut inner = self.lock_inner();
if !inner.closed && inner.generation == handle.connection_token() && inner.writer.is_some()
{
inner.route_end_reasons.remove(&handle.channel);
inner.legacy_channel_reasons.remove(&handle.channel);
inner.route_modules.remove(&handle.channel);
if let Some(previous) = inner.route_epochs.insert(handle.channel, handle) {
inner.route_targets.remove(&previous);
if previous.reverse_request_registry_id() != handle.reverse_request_registry_id() {
release_reverse_request_registry(previous);
}
}
}
}
fn invalidate_routes_for_handle(&self, handle: RouteHandle) {
let removed = {
let mut inner = self.lock_inner();
if inner.route_epochs.get(&handle.channel) != Some(&handle) {
return;
}
inner.release_wire_slots(handle);
let installed = inner.route_epochs.remove(&handle.channel).unwrap();
inner.route_targets.remove(&handle);
inner.retired_routes.remove(&handle);
inner.retirement_goodbyes.remove(&handle);
inner.push_event_receivers.remove(&handle);
release_reverse_request_registry(installed);
inner
.remove_route_by_handle(handle)
.into_iter()
.collect::<Vec<_>>()
};
for route in removed {
route.close();
}
}
}
async fn retirement_goodbye_loop(shared: Arc<Shared>, generation: u64, notify: Arc<Notify>) {
loop {
let notified = notify.notified();
let next = {
let inner = shared.lock_inner();
if inner.closed || inner.generation != generation || inner.writer.is_none() {
return;
}
inner
.retirement_queue
.front()
.map(|handle| (*handle, inner.writer.as_ref().unwrap().clone()))
};
let Some((handle, writer)) = next else {
notified.await;
continue;
};
let Ok(permit) = writer.reserve().await else {
shared.handle_generation_drop(
generation,
"writer closed before retirement GOODBYE".into(),
);
return;
};
let mut inner = shared.lock_inner();
if inner.closed || inner.generation != generation || inner.writer.is_none() {
return;
}
if inner.retirement_queue.front() != Some(&handle) {
continue;
}
inner.retirement_queue.pop_front();
inner.retirement_goodbyes.remove(&handle);
if inner.route_epochs.get(&handle.channel) == Some(&handle) {
let frame = Frame::build(
FrameType::Goodbye,
Flags::new(false, Priority::Interactive, false),
handle.channel,
handle.epoch,
0,
Vec::new(),
)
.expect("header-only route GOODBYE is valid");
permit.send(WriteCommand {
frame,
pending: None,
});
}
inner.reap_retired_route(handle);
}
}
async fn writer_loop<W>(
shared: Arc<Shared>,
writer: W,
mut rx: mpsc::Receiver<WriteCommand>,
generation: u64,
) where
W: AsyncWrite + Unpin,
{
let mut writer = BufWriter::new(writer);
while let Some(command) = rx.recv().await {
if let Some(key) = command.pending {
if !shared.mark_pending_accepted(key) {
continue;
}
}
if let Err(err) = write_frame(&mut writer, &command.frame).await {
shared.handle_generation_drop_with_cause(
generation,
err.to_string(),
OutcomeUnknownCause::WriterClosed,
);
return;
}
while let Ok(command) = rx.try_recv() {
if let Some(key) = command.pending {
if !shared.mark_pending_accepted(key) {
continue;
}
}
if let Err(err) = write_frame(&mut writer, &command.frame).await {
shared.handle_generation_drop_with_cause(
generation,
err.to_string(),
OutcomeUnknownCause::WriterClosed,
);
return;
}
}
if let Err(err) = writer.flush().await.map_err(FrameIoError::Io) {
shared.handle_generation_drop_with_cause(
generation,
err.to_string(),
OutcomeUnknownCause::WriterClosed,
);
return;
}
}
}
impl Shared {
fn implicit_consumer_identity(&self) -> Option<ConsumerIdentity> {
if self.identity_from_environment && !self.claim_bound {
consumer_identity_from_env()
} else {
None
}
}
fn activity_read_body(&self, request: impl serde::Serialize) -> Result<Vec<u8>, CallError> {
#[derive(serde::Serialize)]
struct ReadRequest<T> {
#[serde(flatten)]
request: T,
#[serde(skip_serializing_if = "Option::is_none")]
consumer_identity: Option<ConsumerIdentity>,
}
serde_json::to_vec(&ReadRequest {
request,
consumer_identity: self.implicit_consumer_identity(),
})
.map_err(|err| CallError::not_sent(format!("failed to encode activity read: {err}")))
}
fn route_open_consumer_identity(
&self,
opts: &CallOptions,
) -> Result<Option<ConsumerIdentity>, CallError> {
if self.claim_bound && opts.consumer_identity.is_some() {
return Err(CallError::not_sent(
"consumer identity conflicts with connection binding",
));
}
Ok(opts
.consumer_identity
.clone()
.or_else(|| self.implicit_consumer_identity()))
}
fn close_route_consumer_identity(
&self,
opts: &CloseRouteOptions,
) -> Result<Option<ConsumerIdentity>, CallError> {
if self.claim_bound && opts.consumer_identity.is_some() {
return Err(CallError::not_sent(
"consumer identity conflicts with connection binding",
));
}
Ok(opts
.consumer_identity
.clone()
.or_else(|| self.implicit_consumer_identity()))
}
}
fn route_open_consumer_capabilities(opts: &CallOptions) -> Option<Vec<String>> {
opts.reverse_requests.seal();
let mut capabilities = opts.reverse_requests.capabilities();
capabilities.extend(opts.consumer_capabilities());
capabilities.sort();
capabilities.dedup();
(!capabilities.is_empty()).then_some(capabilities)
}
fn route_open_role_versions(
role_versions: Option<&BTreeMap<String, String>>,
) -> Option<BTreeMap<String, String>> {
role_versions
.filter(|role_versions| !role_versions.is_empty())
.cloned()
}
fn close_route_consumer_capabilities(opts: &CloseRouteOptions) -> Option<Vec<String>> {
opts.reverse_requests.seal();
let mut capabilities = opts.reverse_requests.capabilities();
capabilities.extend(opts.consumer_capabilities());
capabilities.sort();
capabilities.dedup();
(!capabilities.is_empty()).then_some(capabilities)
}
fn consumer_identity_from_env() -> Option<ConsumerIdentity> {
let module_id = std::env::var(SUBC_MODULE_ID_ENV)
.ok()
.filter(|value| !value.is_empty())?;
let launch_nonce = crate::launch_nonce()
.ok()
.flatten()
.map(|nonce| nonce.value().to_string())?;
Some(ConsumerIdentity {
module_id,
launch_nonce,
})
}
fn validate_capability_for_resolution(capability: &str) -> Result<(), CallError> {
if is_valid_capability_identifier(capability) {
Ok(())
} else {
Err(CallError::InvalidCapabilityIdentifier {
capability: capability.to_string(),
})
}
}
fn capability_claimants(catalog: &CatalogList, capability: &str) -> Vec<String> {
let mut claimants = catalog
.modules
.iter()
.filter(|module| {
module.capabilities.as_ref().is_some_and(|capabilities| {
capabilities
.provides
.iter()
.any(|claim| claim == capability)
})
})
.map(|module| module.module_id.clone())
.collect::<Vec<_>>();
claimants.sort();
claimants
}
fn epoch_millis() -> u64 {
u64::try_from(
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis(),
)
.unwrap_or(u64::MAX)
}
fn default_random_unit() -> f64 {
use std::hash::{BuildHasher, RandomState};
static COUNTER: AtomicU64 = AtomicU64::new(0);
let bits = RandomState::new().hash_one(COUNTER.fetch_add(1, Ordering::Relaxed));
(bits >> 11) as f64 / (1u64 << 53) as f64
}
fn request_not_sent_after_route_open_failure(err: CallError) -> CallError {
match err {
CallError::OutcomeUnknown(source) => CallError::not_sent_with_cause(
format!("request not sent: route.open did not complete ({source})"),
NotSentCause::RouteOpenAbandoned,
),
other => other,
}
}
fn classify_failure(
accepted: bool,
reason: impl Into<String>,
cause: OutcomeUnknownCause,
) -> CallError {
classify_failure_with_not_sent_cause(accepted, reason, cause, NotSentCause::WriterRejected)
}
fn classify_failure_with_not_sent_cause(
accepted: bool,
reason: impl Into<String>,
cause: OutcomeUnknownCause,
not_sent_cause: NotSentCause,
) -> CallError {
if accepted {
CallError::outcome_unknown(reason, cause)
} else {
CallError::not_sent_with_cause(reason, not_sent_cause)
}
}
fn settle_route_pending_entries(
entries: Vec<PendingEntry>,
message: String,
reason: impl Into<RouteEnd>,
) {
let reason = reason.into();
let cause = if reason.reason == RouteEndReason::ConnectionLost {
OutcomeUnknownCause::ConnectionFailed
} else {
OutcomeUnknownCause::RouteEnded
};
settle_route_pending_entries_with_cause(entries, message, reason, cause);
}
fn settle_route_pending_entries_with_cause(
entries: Vec<PendingEntry>,
message: String,
reason: impl Into<RouteEnd>,
cause: OutcomeUnknownCause,
) {
let reason = reason.into();
for entry in entries {
let err = if entry.accepted {
CallError::outcome_unknown_source(OutcomeUnknownSource {
message: message.clone(),
cause,
route_reason: Some(reason.clone()),
})
} else {
CallError::NotSent(Box::new(NotSentSource {
message: message.clone(),
cause: NotSentCause::WriterRejected,
never_forwarded: true,
route_reason: Some(reason.clone()),
refusal: None,
}))
};
entry.settle_call_error(err);
}
}
fn settle_pending_entries(entries: Vec<PendingEntry>, reason: String) {
for entry in entries {
entry.settle_failure(reason.clone());
}
}
fn drain_pending_generation(
pending: &mut HashMap<PendingKey, PendingEntry>,
generation: u64,
) -> Vec<PendingEntry> {
let keys = pending
.keys()
.copied()
.filter(|key| key.generation == generation)
.collect::<Vec<_>>();
keys.into_iter()
.filter_map(|key| pending.remove(&key))
.collect()
}
fn drain_pending_handle(
pending: &mut HashMap<PendingKey, PendingEntry>,
handle: RouteHandle,
include_subscriptions: bool,
) -> Vec<PendingEntry> {
let keys = pending
.iter()
.filter_map(|(key, entry)| {
(key.generation == handle.connection_token()
&& key.channel == handle.channel
&& key.epoch == handle.epoch
&& (include_subscriptions || !entry.is_subscription()))
.then_some(*key)
})
.collect::<Vec<_>>();
keys.into_iter()
.filter_map(|key| pending.remove(&key))
.collect()
}
fn fail_openings(openings: Vec<Vec<OpeningWaiter>>, failure: SharedCallFailure) {
for waiters in openings {
for waiter in waiters {
let _ = waiter.send(Err(failure.clone()));
}
}
}
fn emit_callbacks(callbacks: Vec<Callback>, state: ConnectionState) {
for callback in callbacks {
if let Ok(callback) = callback.lock() {
callback(state.clone());
}
}
}
pub fn is_retryable_route_open_code(code: &str) -> bool {
error_codes::is_retryable_route_open(code)
}
fn is_data_plane_drain_refusal(code: &str) -> bool {
code == error_codes::MODULE_RELOADING
}
const DAEMON_ORIGIN_MIN_VERSION: (u64, u64, u64) = (0, 37, 15);
pub fn trusts_daemon_origin(daemon_ver: &str) -> bool {
daemon_version_at_least(daemon_ver, DAEMON_ORIGIN_MIN_VERSION)
}
fn daemon_version_at_least(daemon_ver: &str, minimum: (u64, u64, u64)) -> bool {
let mut components = daemon_ver.split('.');
let mut next = || {
let component = components.next()?;
if component.is_empty() || !component.bytes().all(|byte| byte.is_ascii_digit()) {
return None;
}
component.parse::<u64>().ok()
};
let version = (next(), next(), next());
match version {
(Some(major), Some(minor), Some(patch)) if components.next().is_none() => {
(major, minor, patch) >= minimum
}
_ => false,
}
}
pub fn is_daemon_refusal_before_forward(
daemon_origin_trusted: bool,
flags: Flags,
code: &str,
) -> bool {
daemon_origin_trusted
&& flags.is_daemon_origin()
&& matches!(
code,
error_codes::MODULE_RELOADING
| error_codes::UNKNOWN_CHANNEL
| error_codes::STALE_ROUTE_EPOCH
)
}
fn route_is_installed(inner: &Inner, handle: RouteHandle) -> bool {
handle.channel == 0 || inner.route_epochs.get(&handle.channel) == Some(&handle)
}
fn is_retryable_catalog_transport_error(err: &CallError) -> bool {
matches!(err, CallError::NotSent(_) | CallError::OutcomeUnknown(_))
}
fn is_reconnect_transient(err: &ConsumerError) -> bool {
if let ConsumerError::Connect { source, .. } = err {
if let Some(claim) = source
.get_ref()
.and_then(|source| source.downcast_ref::<crate::ClaimError>())
{
return matches!(claim, crate::ClaimError::Refused { code } if error_codes::is_retryable_claim(code));
}
}
match err {
ConsumerError::Connect { source, .. } => matches!(
source.kind(),
io::ErrorKind::ConnectionRefused
| io::ErrorKind::ConnectionReset
| io::ErrorKind::ConnectionAborted
| io::ErrorKind::TimedOut
| io::ErrorKind::NotConnected
| io::ErrorKind::AddrNotAvailable
),
ConsumerError::ConnectionFile { source, .. } => match source {
ConnectionFileError::Io { source, .. } => source.kind() == io::ErrorKind::NotFound,
_ => false,
},
ConsumerError::Auth { .. } => true,
ConsumerError::Discovery { .. }
| ConsumerError::NoEndpoint { .. }
| ConsumerError::Closed => false,
}
}
impl From<FrameBuildError> for CallError {
fn from(err: FrameBuildError) -> Self {
Self::not_sent(err.to_string())
}
}
#[cfg(test)]
mod local_transport_tests;
#[cfg(test)]
mod tests {
use super::*;
mod busy_refusals_tests;
mod not_sent_tests;
mod route_close_tests;
mod route_deadline_tests;
#[derive(Clone)]
struct InstrumentedWriter {
state: Arc<InstrumentedWriterState>,
fail_flush: bool,
}
#[derive(Default)]
struct InstrumentedWriterState {
bytes: Mutex<Vec<u8>>,
flushes: std::sync::atomic::AtomicUsize,
}
impl InstrumentedWriter {
fn new(fail_flush: bool) -> Self {
Self {
state: Arc::new(InstrumentedWriterState::default()),
fail_flush,
}
}
fn bytes(&self) -> Vec<u8> {
self.state
.bytes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone()
}
fn flush_count(&self) -> usize {
self.state.flushes.load(Ordering::SeqCst)
}
}
impl AsyncWrite for InstrumentedWriter {
fn poll_write(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<io::Result<usize>> {
self.state
.bytes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.extend_from_slice(buf);
Poll::Ready(Ok(buf.len()))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
self.state.flushes.fetch_add(1, Ordering::SeqCst);
if self.fail_flush {
Poll::Ready(Err(io::Error::new(
io::ErrorKind::BrokenPipe,
"instrumented flush failure",
)))
} else {
Poll::Ready(Ok(()))
}
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
}
fn writer_test_shared() -> Arc<Shared> {
Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions {
reconnect_backoff: RetryBackoff {
max_attempts: 1,
..RetryBackoff::default()
},
..ConsumerOptions::default()
},
))
}
fn reverse_test_shared(
reverse_requests: ReverseRequestRegistry,
) -> (Arc<Shared>, mpsc::Receiver<WriteCommand>, RouteHandle) {
let shared = writer_test_shared();
let (writer, receiver) = mpsc::channel(16);
let handle = RouteHandle::new_consumer(17, 3, 1, reverse_requests);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.route_epochs.insert(handle.channel, handle);
}
(shared, receiver, handle)
}
fn reverse_request_frame(handle: RouteHandle, corr: u64, body: Vec<u8>) -> Frame {
Frame::build(
FrameType::Request,
Flags::new(false, Priority::Interactive, false),
handle.channel,
handle.epoch,
corr,
body,
)
.unwrap()
}
#[tokio::test]
async fn binary_reply_request_preserves_flagged_non_utf8_bytes() {
let (shared, mut receiver, handle) = reverse_test_shared(ReverseRequestRegistry::new());
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
shared
.install_one_shot_route(RouteState {
handle,
sem: Arc::new(Semaphore::new(1)),
watch_sem: None,
opted_in: false,
})
.unwrap();
let request = tokio::spawn(async move {
consumer
.request(&handle, b"bytes".to_vec(), CallOptions::default())
.await
});
let command = receiver.recv().await.unwrap();
let bytes = vec![0, 0xff, 0x80, b'{'];
let frame = Frame::build(
FrameType::Response,
Flags::new(true, Priority::Interactive, false),
handle.channel,
handle.epoch,
command.frame.header.corr,
bytes.clone(),
)
.unwrap();
assert!(dispatch_frame(&shared, 1, frame).await);
assert_eq!(request.await.unwrap().unwrap(), bytes);
}
async fn capture_route_open(
reverse_requests: ReverseRequestRegistry,
) -> (serde_json::Value, RouteHandle) {
capture_route_open_with_options(CallOptions {
reverse_requests,
..CallOptions::default()
})
.await
}
async fn capture_route_open_with_options(
opts: CallOptions,
) -> (serde_json::Value, RouteHandle) {
let shared = writer_test_shared();
let (writer, mut receiver) = mpsc::channel(4);
shared.lock_inner().writer = Some(writer);
shared.lock_inner().daemon_ver = "0.39.0".into();
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let task = tokio::spawn(async move {
consumer
.open_route(
RouteTarget::ToolProvider {
module_id: "reverse-provider".to_string(),
},
BindIdentity::new("test".to_string(), "reverse".to_string()),
opts,
)
.await
});
let command = receiver.recv().await.expect("route.open frame");
let request = serde_json::from_slice(&command.frame.body).unwrap();
let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
request_window: None,
watch_window: None,
route_channel: 17,
route_epoch: 3,
})
.unwrap();
assert!(
dispatch_frame(
&shared,
1,
response_frame(0, 0, command.frame.header.corr, response_body),
)
.await
);
let handle = task.await.unwrap().unwrap();
(request, handle)
}
#[tokio::test]
async fn held_subscriptions_set_the_flag_but_ordinary_requests_do_not() {
let (shared, mut receiver, handle) = reverse_test_shared(ReverseRequestRegistry::new());
let call_shared = Arc::clone(&shared);
let call = tokio::spawn(async move {
call_shared
.send_request(RequestSend {
binary: false,
expected_handle: Some(handle),
channel: handle.channel,
epoch: handle.epoch,
body: b"ordinary".to_vec(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline: Instant::now() + Duration::from_secs(1),
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await
});
let ordinary = receiver.recv().await.unwrap();
assert_eq!(ordinary.frame.header.encode()[6] & FLAG_SUBSCRIPTION, 0);
assert!(
dispatch_frame(
&shared,
1,
response_frame(
handle.channel,
handle.epoch,
ordinary.frame.header.corr,
Vec::new()
),
)
.await
);
assert!(call.await.unwrap().is_ok());
let permit = Arc::new(Semaphore::new(1)).acquire_owned().await.unwrap();
let subscription = Arc::clone(&shared)
.send_subscription(SubscriptionSend {
expected_handle: Some(handle),
channel: handle.channel,
epoch: handle.epoch,
body: b"held".to_vec(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
event_buffer: 1,
deadline: Instant::now() + Duration::from_secs(1),
permit: RoutePermit::legacy(permit),
})
.await
.unwrap();
let held = receiver.recv().await.unwrap();
assert_ne!(
held.frame.header.encode()[6] & FLAG_SUBSCRIPTION,
0,
"a held-open subscription must be marked for drain accounting"
);
drop(subscription);
release_reverse_request_registry(handle);
}
#[tokio::test]
async fn registered_handlers_derive_route_open_consumer_capabilities() {
let registry = ReverseRequestRegistry::new();
registry
.on_request("elicitation", |_body, _ctx| async { Vec::new() })
.unwrap();
let (request, handle) = capture_route_open(registry).await;
assert_eq!(
request.get("consumer_capabilities"),
Some(&serde_json::json!(["elicitation"]))
);
release_reverse_request_registry(handle);
}
#[tokio::test]
async fn declared_consumer_capabilities_merge_with_handler_families_on_route_open() {
let registry = ReverseRequestRegistry::new();
registry
.on_request("elicitation", |_body, _ctx| async { Vec::new() })
.unwrap();
let opts = CallOptions {
reverse_requests: registry,
..CallOptions::default()
}
.with_consumer_capability("tool-result/v1")
.unwrap()
.with_consumer_capability("elicitation")
.unwrap()
.with_consumer_capability("tool-result/v1")
.unwrap();
let (request, handle) = capture_route_open_with_options(opts).await;
assert_eq!(
request.get("consumer_capabilities"),
Some(&serde_json::json!(["elicitation", "tool-result/v1"]))
);
release_reverse_request_registry(handle);
}
#[test]
fn invalid_consumer_capability_declarations_are_rejected_before_route_open() {
let (shared, mut receiver) = {
let shared = writer_test_shared();
let (writer, receiver) = mpsc::channel(4);
shared.lock_inner().writer = Some(writer);
shared.lock_inner().daemon_ver = "0.39.0".into();
(shared, receiver)
};
let too_long = "x".repeat(65);
for invalid in [
String::new(),
"content v1".to_string(),
"content\tv1".to_string(),
"content\nv1".to_string(),
"cöntent/v1".to_string(),
"content/\u{7f}v1".to_string(),
too_long,
] {
let error = CallOptions::default()
.with_consumer_capability(invalid.clone())
.expect_err("invalid declarations must not produce route options");
assert_eq!(error.capability(), invalid);
assert!(
receiver.try_recv().is_err(),
"invalid declaration {invalid:?} must be rejected before any frame is queued"
);
}
let close_error = CloseRouteOptions::default()
.with_consumer_capability("bad value")
.expect_err("close-route declarations use the same validation");
assert_eq!(close_error.capability(), "bad value");
let call_options = CallOptions::default()
.with_consumer_capability("tool-result/v1")
.unwrap();
assert_eq!(
call_options.consumer_capabilities(),
vec!["tool-result/v1".to_string()]
);
let close_options = CloseRouteOptions::default()
.with_consumer_capability("tool-result/v1")
.unwrap();
assert_eq!(
close_options.consumer_capabilities(),
vec!["tool-result/v1".to_string()]
);
shared.close_sync("test complete");
}
#[tokio::test]
async fn no_handlers_omit_consumer_capabilities_from_route_open() {
let (request, handle) = capture_route_open(ReverseRequestRegistry::new()).await;
assert!(!request
.as_object()
.unwrap()
.contains_key("consumer_capabilities"));
release_reverse_request_registry(handle);
}
#[tokio::test]
async fn reverse_request_without_handler_returns_typed_error_instead_of_silence() {
let (shared, mut receiver, handle) = reverse_test_shared(ReverseRequestRegistry::new());
let body = serde_json::to_vec(&serde_json::json!({
"jsonrpc": "2.0",
"id": 1,
"method": "elicitation/create",
"params": {}
}))
.unwrap();
assert!(dispatch_frame(&shared, 1, reverse_request_frame(handle, 800, body)).await);
let reply = tokio::time::timeout(Duration::from_secs(1), receiver.recv())
.await
.unwrap()
.unwrap()
.frame;
assert_eq!(reply.header.ty, FrameType::Error);
assert_eq!(reply.header.corr, 800);
let error: ErrorBody = serde_json::from_slice(&reply.body).unwrap();
assert_eq!(error.code, REVERSE_REQUEST_UNHANDLED);
release_reverse_request_registry(handle);
}
#[tokio::test]
async fn reverse_request_round_trip_preserves_raw_body_corr_and_response() {
let registry = ReverseRequestRegistry::new();
let seen = Arc::new(Mutex::new(None));
let seen_by_handler = Arc::clone(&seen);
let response = br#"{"jsonrpc":"2.0","id":9,"result":{"accepted":true}}"#.to_vec();
let expected_response = response.clone();
registry
.on_request("elicitation", move |body, ctx| {
*seen_by_handler
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()) =
Some((body, ctx.corr, ctx.method));
let response = response.clone();
async move { response }
})
.unwrap();
let (shared, mut receiver, handle) = reverse_test_shared(registry);
let request_body =
br#"{"jsonrpc":"2.0","id":9,"method":"elicitation/create","params":{"z":1,"a":2}}"#
.to_vec();
assert!(
dispatch_frame(
&shared,
1,
reverse_request_frame(handle, 801, request_body.clone()),
)
.await
);
let reply = tokio::time::timeout(Duration::from_secs(1), receiver.recv())
.await
.unwrap()
.unwrap()
.frame;
assert_eq!(reply.header.ty, FrameType::Response);
assert_eq!(reply.header.corr, 801);
assert_eq!(reply.body, expected_response);
assert_eq!(
seen.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone(),
Some((request_body, 801, "elicitation/create".to_string()))
);
release_reverse_request_registry(handle);
}
#[tokio::test]
async fn panicking_reverse_handler_returns_typed_refusal_and_sibling_request_resolves() {
let registry = ReverseRequestRegistry::new();
registry
.on_request("sampling", |_body, _ctx| async move {
panic!("person prompt exploded");
})
.unwrap();
let (shared, mut receiver, handle) = reverse_test_shared(registry);
let sibling_shared = Arc::clone(&shared);
let sibling = tokio::spawn(async move {
sibling_shared
.send_request(RequestSend {
binary: false,
expected_handle: Some(handle),
channel: handle.channel,
epoch: handle.epoch,
body: b"sibling".to_vec(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline: Instant::now() + Duration::from_secs(1),
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await
});
let sibling_request = receiver.recv().await.unwrap();
let reverse_body = serde_json::to_vec(&serde_json::json!({
"method": "sampling/createMessage",
"params": {}
}))
.unwrap();
assert!(
dispatch_frame(&shared, 1, reverse_request_frame(handle, 802, reverse_body),).await
);
assert!(
dispatch_frame(
&shared,
1,
response_frame(
handle.channel,
handle.epoch,
sibling_request.frame.header.corr,
b"ok".to_vec(),
),
)
.await
);
assert!(matches!(
sibling.await.unwrap().unwrap(),
TerminalFrame::Response { body, .. } if body == b"ok"
));
let refusal = tokio::time::timeout(Duration::from_secs(1), receiver.recv())
.await
.unwrap()
.unwrap()
.frame;
assert_eq!(refusal.header.ty, FrameType::Error);
let error: ErrorBody = serde_json::from_slice(&refusal.body).unwrap();
assert_eq!(error.code, REVERSE_REQUEST_UNHANDLED);
assert!(error.message.contains("person prompt exploded"));
assert!(shared.generation_is_current(1));
release_reverse_request_registry(handle);
}
fn reservation_stand_in(
credit: usize,
capacity: usize,
) -> (
Arc<Shared>,
SubcConsumer,
mpsc::Receiver<WriteCommand>,
RouteHandle,
Arc<Semaphore>,
) {
let shared = writer_test_shared();
let (writer, rx) = mpsc::channel(capacity);
shared.lock_inner().writer = Some(writer);
shared.lock_inner().daemon_ver = "0.39.0".into();
let handle = RouteHandle::new(5, 3, 1);
let sem = Arc::new(Semaphore::new(credit));
shared
.install_one_shot_route(RouteState {
handle,
sem: Arc::clone(&sem),
watch_sem: None,
opted_in: false,
})
.unwrap();
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
(shared, consumer, rx, handle, sem)
}
#[tokio::test]
async fn reservation_send_is_synchronous_and_uses_prevalidated_body() {
let oversized = vec![0; subc_protocol::MAX_FRAME_BODY_LEN as usize + 1];
assert!(matches!(
RequestBody::new(oversized),
Err(CallError::NotSent(_))
));
let body = RequestBody::new(b"valid".to_vec()).unwrap();
let (shared, consumer, mut rx, handle, sem) = reservation_stand_in(1, 1);
let reserved = consumer
.reserve(&handle, Instant::now() + Duration::from_secs(1))
.await
.unwrap();
let corr = reserved.correlation_id();
assert_eq!(shared.lock_inner().writer.as_ref().unwrap().capacity(), 0);
assert!(rx.try_recv().is_err());
let (sent_corr, reply): (CorrelationId, PendingReply) =
reserved.send(body, CallOptions::default());
assert_eq!(sent_corr, corr);
let command = rx.try_recv().expect("send must enqueue without yielding");
assert_eq!(command.frame.body, b"valid");
assert_eq!(command.frame.header.corr, corr);
assert_eq!(sem.available_permits(), 0);
assert!(shared.mark_pending_accepted(command.pending.unwrap()));
dispatch_frame(
&shared,
1,
response_frame(handle.channel, handle.epoch, corr, b"reply".to_vec()),
)
.await;
assert_eq!(reply.await.unwrap(), b"reply");
assert_eq!(sem.available_permits(), 1);
consumer.close().await;
}
#[tokio::test]
async fn reservation_drop_releases_route_credit_and_writer_capacity_without_sending() {
let (shared, consumer, mut rx, handle, sem) = reservation_stand_in(1, 1);
let deadline = Instant::now() + Duration::from_secs(1);
let first = consumer.reserve(&handle, deadline).await.unwrap();
let second = consumer.reserve(&handle, deadline);
tokio::pin!(second);
assert_pending(second.as_mut()).await;
assert_eq!(sem.available_permits(), 0);
drop(first);
let second = timeout_at(deadline, second).await.unwrap().unwrap();
drop(second);
assert_eq!(sem.available_permits(), 1);
assert_eq!(shared.lock_inner().writer.as_ref().unwrap().capacity(), 1);
assert!(shared.lock_inner().pending.is_empty());
assert!(rx.try_recv().is_err());
consumer.close().await;
}
#[tokio::test]
async fn reservation_route_closed_before_send_matches_an_ordinary_unwritten_request() {
let (shared, consumer, mut rx, handle, _) = reservation_stand_in(2, 4);
let reserved = consumer
.reserve(&handle, Instant::now() + Duration::from_secs(1))
.await
.unwrap();
let ordinary = consumer.request(&handle, b"ordinary".to_vec(), CallOptions::default());
tokio::pin!(ordinary);
assert_pending(ordinary.as_mut()).await;
let ordinary_command = rx.try_recv().unwrap();
assert_eq!(ordinary_command.frame.body, b"ordinary");
consumer
.close_handle(&handle, CloseRouteOptions::default())
.await
.unwrap();
let (_, reply) = reserved.send(
RequestBody::new(b"reserved".to_vec()).unwrap(),
CallOptions::default(),
);
let ordinary_error = ordinary.await.unwrap_err();
let reserved_error = reply.await.unwrap_err();
assert!(matches!(reserved_error, CallError::NotSent(_)));
assert_eq!(
reserved_error.close_reason(),
Some(&RouteEndReason::ClosedByCaller)
);
assert_eq!(reserved_error.to_string(), ordinary_error.to_string());
assert_eq!(reserved_error.close_reason(), ordinary_error.close_reason());
rx.close();
let writer = InstrumentedWriter::new(false);
writer_loop(Arc::clone(&shared), writer.clone(), rx, 1).await;
let bytes = writer.bytes();
let mut bytes = bytes.as_slice();
let frame = read_frame(&mut bytes).await.unwrap().unwrap();
assert_eq!(frame.header.ty, FrameType::Goodbye);
assert!(
bytes.is_empty(),
"the closed route's reserved request must not be written"
);
consumer.close().await;
}
#[tokio::test]
async fn reservation_connection_loss_after_send_preserves_existing_errors() {
for written in [false, true] {
let (shared, consumer, rx, handle, sem) = reservation_stand_in(1, 1);
let reserved = consumer
.reserve(&handle, Instant::now() + Duration::from_secs(1))
.await
.unwrap();
let (_, reply) = reserved.send(
RequestBody::new(b"request".to_vec()).unwrap(),
CallOptions::default(),
);
let writer = InstrumentedWriter::new(written);
if !written {
shared.handle_generation_drop(1, "connection lost before dequeue".to_string());
}
writer_loop(Arc::clone(&shared), writer.clone(), rx, 1).await;
let err = reply.await.unwrap_err();
assert_eq!(err.close_reason(), Some(&RouteEndReason::ConnectionLost));
if written {
assert_eq!(err.outcome_cause(), Some(OutcomeUnknownCause::WriterClosed));
assert!(!writer.bytes().is_empty());
} else {
assert!(matches!(err, CallError::NotSent(_)));
assert!(writer.bytes().is_empty());
}
assert_eq!(sem.available_permits(), 1);
consumer.close().await;
}
}
#[tokio::test]
async fn reservation_correlations_are_unique_and_match_frames_written_out_of_order() {
let (shared, consumer, mut rx, handle, _) = reservation_stand_in(3, 3);
let deadline = Instant::now() + Duration::from_secs(1);
let first = consumer.reserve(&handle, deadline).await.unwrap();
let second = consumer.reserve(&handle, deadline).await.unwrap();
let third = consumer.reserve(&handle, deadline).await.unwrap();
let ids = [
first.correlation_id(),
second.correlation_id(),
third.correlation_id(),
];
assert_eq!(ids.into_iter().collect::<HashSet<_>>().len(), 3);
let opts = CallOptions {
priority: Priority::Background,
admission_class: AdmissionClass::Expedite,
..CallOptions::default()
};
let mut replies = Vec::new();
for (reserved, expected_id, body) in [
(third, ids[2], "third"),
(first, ids[0], "first"),
(second, ids[1], "second"),
] {
let (corr, reply) = reserved.send(
RequestBody::new(body.as_bytes().to_vec()).unwrap(),
opts.clone(),
);
assert_eq!(corr, expected_id);
replies.push((corr, body, reply));
}
rx.close();
let writer = InstrumentedWriter::new(false);
writer_loop(Arc::clone(&shared), writer.clone(), rx, 1).await;
let bytes = writer.bytes();
let mut bytes = bytes.as_slice();
for (corr, body, reply) in replies {
let frame = read_frame(&mut bytes).await.unwrap().unwrap();
assert_eq!(frame.header.ty, FrameType::Request);
assert_eq!(frame.header.corr, corr);
assert_eq!(frame.header.channel, handle.channel);
assert_eq!(frame.header.epoch, handle.epoch);
assert_eq!(
frame.header.flags,
Flags::new(false, Priority::Background, false)
.with_admission_class(AdmissionClass::Expedite)
);
assert_eq!(frame.body, body.as_bytes());
dispatch_frame(
&shared,
1,
response_frame(handle.channel, handle.epoch, corr, body.as_bytes().to_vec()),
)
.await;
assert_eq!(reply.await.unwrap(), body.as_bytes());
}
assert!(bytes.is_empty());
consumer.close().await;
}
#[tokio::test(start_paused = true)]
async fn reservation_deadline_before_send_prevents_writing_without_polling_reply() {
let (shared, consumer, mut rx, handle, sem) = reservation_stand_in(1, 1);
let reserved = consumer
.reserve(&handle, Instant::now() + Duration::from_secs(1))
.await
.unwrap();
tokio::time::advance(Duration::from_secs(2)).await;
let (_, reply) = reserved.send(
RequestBody::new(b"expired".to_vec()).unwrap(),
CallOptions::default(),
);
rx.close();
let writer = InstrumentedWriter::new(false);
writer_loop(Arc::clone(&shared), writer.clone(), rx, 1).await;
assert!(writer.bytes().is_empty());
assert!(matches!(reply.await, Err(CallError::NotSent(_))));
assert_eq!(sem.available_permits(), 1);
consumer.close().await;
}
#[tokio::test(start_paused = true)]
async fn reservation_deadline_bounds_writer_capacity_and_releases_credit() {
let (shared, consumer, mut rx, handle, sem) = reservation_stand_in(2, 1);
let first = consumer
.reserve(&handle, Instant::now() + Duration::from_secs(1))
.await
.unwrap();
let second = consumer
.reserve(&handle, Instant::now() + Duration::from_millis(25))
.await;
assert!(matches!(second, Err(CallError::NotSent(_))));
assert_eq!(sem.available_permits(), 1);
drop(first);
assert_eq!(sem.available_permits(), 2);
assert!(shared.lock_inner().pending.is_empty());
assert!(rx.try_recv().is_err());
consumer.close().await;
}
#[tokio::test]
async fn reservation_reconnect_during_writer_capacity_wait_fails_promptly_without_writing() {
let (shared, consumer, mut old_rx, handle, sem) = reservation_stand_in(1, 1);
let old_writer = shared.lock_inner().writer.clone().unwrap();
let held_capacity = old_writer.reserve_owned().await.unwrap();
let reserve = consumer.reserve(&handle, Instant::now() + Duration::from_secs(60));
tokio::pin!(reserve);
assert_pending(reserve.as_mut()).await;
assert_eq!(sem.available_permits(), 0);
let mut replacement_rx = drop_and_replace_test_transport(&shared);
assert_ne!(shared.lock_inner().generation, handle.connection_token());
drop(held_capacity);
let result = std::future::poll_fn(|cx| Poll::Ready(reserve.as_mut().poll(cx))).await;
assert!(matches!(
result,
Poll::Ready(Err(CallError::StaleRouteHandle(stale))) if stale == handle
));
assert!(shared.lock_inner().pending.is_empty());
assert_eq!(sem.available_permits(), 1);
assert!(old_rx.try_recv().is_err());
assert!(replacement_rx.try_recv().is_err());
old_rx.close();
replacement_rx.close();
let writer = InstrumentedWriter::new(false);
writer_loop(
Arc::clone(&shared),
writer.clone(),
old_rx,
handle.connection_token(),
)
.await;
let generation = shared.lock_inner().generation;
writer_loop(
Arc::clone(&shared),
writer.clone(),
replacement_rx,
generation,
)
.await;
assert!(writer.bytes().is_empty());
consumer.close().await;
}
#[tokio::test]
async fn reservation_reply_drop_cleans_up_without_polling() {
let (shared, consumer, mut rx, handle, sem) = reservation_stand_in(1, 1);
let reserved = consumer
.reserve(&handle, Instant::now() + Duration::from_secs(1))
.await
.unwrap();
let (_, reply) = reserved.send(
RequestBody::new(b"request".to_vec()).unwrap(),
CallOptions::default(),
);
drop(reply);
assert!(shared.lock_inner().pending.is_empty());
assert_eq!(sem.available_permits(), 1);
rx.close();
let writer = InstrumentedWriter::new(false);
writer_loop(Arc::clone(&shared), writer.clone(), rx, 1).await;
assert!(writer.bytes().is_empty());
consumer.close().await;
}
#[tokio::test]
async fn writer_batches_ready_frames_into_one_flush() {
const FRAME_COUNT: usize = 8;
let shared = writer_test_shared();
let (live_writer, _live_rx) = mpsc::channel(1);
let (tx, rx) = mpsc::channel(FRAME_COUNT + 1);
let instrumented = InstrumentedWriter::new(false);
let observer = instrumented.clone();
let mut expected = Vec::with_capacity(FRAME_COUNT);
let mut keys = Vec::with_capacity(FRAME_COUNT);
{
let mut inner = shared.lock_inner();
inner.writer = Some(live_writer);
inner.daemon_ver = "0.39.0".into();
for index in 0..FRAME_COUNT {
let corr = index as u64 + 1;
let frame = response_frame(7, 1, corr, vec![index as u8; index + 1]);
let key = PendingKey {
generation: 1,
channel: 7,
epoch: 1,
corr,
};
let (pending_tx, _pending_rx) = oneshot::channel();
inner
.pending
.insert(key, PendingEntry::unary(pending_tx, false, None, None));
expected.push(frame.clone());
keys.push(key);
tx.try_send(WriteCommand {
frame,
pending: Some(key),
})
.expect("the burst should fit in the writer queue");
if index == 0 {
tx.try_send(WriteCommand {
frame: response_frame(7, 1, 999, b"skip".to_vec()),
pending: Some(PendingKey {
generation: 1,
channel: 7,
epoch: 1,
corr: 999,
}),
})
.expect("the skipped command should fit in the writer queue");
}
}
}
drop(tx);
writer_loop(Arc::clone(&shared), instrumented, rx, 1).await;
{
let inner = shared.lock_inner();
for key in keys {
assert!(
inner.pending.get(&key).is_some_and(|entry| entry.accepted),
"every written command must be marked accepted"
);
}
}
let mut wire = std::io::Cursor::new(observer.bytes());
for expected_frame in expected {
let actual = read_frame(&mut wire)
.await
.expect("the emitted frame should decode")
.expect("the emitted frame should be present");
assert_eq!(actual, expected_frame);
}
assert!(
read_frame(&mut wire)
.await
.expect("the end of the emitted burst should be clean")
.is_none(),
"the writer must not emit extra frames"
);
let flush_count = observer.flush_count();
assert_eq!(
flush_count, 1,
"a ready burst must be coalesced into one flush"
);
shared.close_sync("test complete");
}
#[tokio::test]
async fn writer_flush_failure_drops_generation_and_preserves_acceptance_classification() {
let shared = writer_test_shared();
let (live_writer, _live_rx) = mpsc::channel(1);
let accepted_key = PendingKey {
generation: 1,
channel: 3,
epoch: 1,
corr: 1,
};
let not_sent_key = PendingKey {
corr: 2,
..accepted_key
};
let (accepted_tx, accepted_rx) = oneshot::channel();
let (not_sent_tx, not_sent_rx) = oneshot::channel();
{
let mut inner = shared.lock_inner();
inner.writer = Some(live_writer);
inner.daemon_ver = "0.39.0".into();
inner.pending.insert(
accepted_key,
PendingEntry::unary(accepted_tx, false, None, None),
);
inner.pending.insert(
not_sent_key,
PendingEntry::unary(not_sent_tx, false, None, None),
);
}
let (tx, rx) = mpsc::channel(1);
tx.send(WriteCommand {
frame: response_frame(3, 1, accepted_key.corr, b"accepted".to_vec()),
pending: Some(accepted_key),
})
.await
.unwrap();
drop(tx);
writer_loop(Arc::clone(&shared), InstrumentedWriter::new(true), rx, 1).await;
assert!(
shared.lock_inner().writer.is_none(),
"a flush failure must drop the active generation"
);
let accepted_error = accepted_rx
.await
.expect("the accepted request should be settled")
.into_call_result()
.unwrap_err();
assert!(matches!(accepted_error, CallError::OutcomeUnknown(_)));
assert_eq!(
accepted_error.outcome_cause(),
Some(OutcomeUnknownCause::WriterClosed)
);
let not_sent_error = not_sent_rx
.await
.expect("the unwritten request should be settled")
.into_call_result()
.unwrap_err();
assert!(matches!(not_sent_error, CallError::NotSent(_)));
assert_eq!(not_sent_error.outcome_cause(), None);
shared.close_sync("test complete");
}
#[tokio::test]
async fn consumer_close_attaches_outcome_cause_to_pending_call() {
let shared = writer_test_shared();
let (writer, _writer_rx) = mpsc::channel(1);
shared.lock_inner().writer = Some(writer);
shared.lock_inner().daemon_ver = "0.39.0".into();
let (tx, rx) = oneshot::channel();
let mut entry = PendingEntry::unary(tx, false, None, None);
entry.accepted = true;
shared.lock_inner().pending.insert(
PendingKey {
generation: 1,
channel: 3,
epoch: 1,
corr: 1,
},
entry,
);
let consumer = SubcConsumer { shared };
consumer.close().await;
let err = rx.await.unwrap().into_call_result().unwrap_err();
assert_eq!(
err.outcome_cause(),
Some(OutcomeUnknownCause::ConsumerClosed)
);
assert_eq!(err.to_string(), "request outcome unknown: consumer closed");
assert_eq!(err.close_reason(), None);
}
#[tokio::test]
async fn subscription_completion_drop_attaches_outcome_cause() {
let (tx, rx) = oneshot::channel();
let closed = SubscriptionClosed { rx };
drop(tx);
let err = closed.await.unwrap_err();
assert_eq!(
err.outcome_cause(),
Some(OutcomeUnknownCause::CompletionFailed)
);
assert_eq!(
err.to_string(),
"request outcome unknown: subscription closed result channel dropped"
);
}
#[test]
fn reconnect_classifier_treats_auth_failure_as_transient() {
let auth = ConsumerError::Auth {
path: PathBuf::from("/tmp/subc-connection.json"),
endpoint: "127.0.0.1:8757".to_string(),
source: subc_transport::AuthError::InvalidServerProof,
};
assert!(is_reconnect_transient(&auth), "rotation race must retry");
let absent = ConsumerError::ConnectionFile {
path: PathBuf::from("/tmp/subc-connection.json"),
source: ConnectionFileError::Io {
op: "read",
path: PathBuf::from("/tmp/subc-connection.json"),
source: io::Error::new(io::ErrorKind::NotFound, "gone"),
},
};
assert!(is_reconnect_transient(&absent));
}
#[tokio::test]
async fn newer_drop_supersedes_reconnect_and_ignores_stale_completion() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let stale_task = tokio::spawn(std::future::pending::<()>());
{
let mut inner = shared.lock_inner();
inner.generation = 2;
inner.reconnect = ReconnectState::Background {
generation: 1,
task: stale_task,
};
}
assert!(shared.spawn_reconnect(2));
assert!(matches!(
&shared.lock_inner().reconnect,
ReconnectState::Background { generation, .. } if *generation == 2
));
shared.finish_background_reconnect(1);
assert!(matches!(
&shared.lock_inner().reconnect,
ReconnectState::Background { generation, .. } if *generation == 2
));
shared.close_sync("test complete");
}
#[test]
fn retryable_route_open_codes_are_code_specific() {
for code in [
"module_reloading",
"module_warming",
"target_unavailable",
"module_timeout",
] {
assert!(is_retryable_route_open_code(code), "{code} should retry");
}
assert!(
!is_retryable_route_open_code(error_codes::UNKNOWN_MODULE),
"unknown_module is terminal"
);
assert!(!is_retryable_route_open_code(error_codes::MODULE_REMOVED));
assert!(!is_retryable_route_open_code("invalid_project_root"));
assert!(!is_retryable_route_open_code("route_rejected"));
assert!(
!is_retryable_route_open_code("capability_forbidden"),
"a capability policy refusal must not enter the route.open retry set"
);
}
#[test]
fn two_routes_record_distinct_reasons_and_reuse_clears_channel() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::new(),
ConsumerOptions::default(),
));
let (writer, _rx) = mpsc::channel(8);
shared.lock_inner().writer = Some(writer);
shared.lock_inner().daemon_ver = "0.39.0".into();
shared.install_ingress_handle(RouteHandle::new(7, 1, 1));
shared.install_ingress_handle(RouteHandle::new(9, 1, 1));
{
let mut inner = shared.lock_inner();
inner.route_modules.insert(7, "provider".into());
inner.route_modules.insert(9, "provider".into());
}
shared.control_push(br#"{"op":"route.closed","module_id":"provider","channels":[7],"reason":"scope_ended"}"#);
shared.control_push(
br#"{"op":"route.closing","module_id":"provider","channels":[9],"reason":"restart"}"#,
);
shared.control_push(
br#"{"op":"route.closing","module_id":"provider","channels":[7],"reason":"crash"}"#,
);
assert_eq!(
shared.lock_inner().route_end_reason(7),
RouteEndReason::Daemon(RouteCloseReason::ScopeEnded)
);
assert_eq!(
shared.lock_inner().route_end_reason(9),
RouteEndReason::Daemon(RouteCloseReason::Restart)
);
shared.control_push(
br#"{"op":"route.closed","module_id":"provider","channels":[9],"reason":"crash"}"#,
);
assert_eq!(
shared.lock_inner().route_end_reason(9),
RouteEndReason::Daemon(RouteCloseReason::Crash)
);
shared.control_push(br#"{"op":"route.closed","module_id":"provider","reason":"disable"}"#);
shared.install_ingress_handle(RouteHandle::new(7, 2, 1));
assert_eq!(
shared.lock_inner().route_end_reason(7),
RouteEndReason::Unknown
);
assert_eq!(shared.lock_inner().route_end(7).terminal, None);
}
#[test]
fn legacy_route_push_falls_back_only_when_channels_are_absent() {
let shared = reader_skew_tests::legacy_shared(PathBuf::new(), ConsumerOptions::default());
shared
.lock_inner()
.route_modules
.insert(7, "provider".into());
shared.control_push(
br#"{"op":"route.closed","module_id":"provider","channels":[],"reason":"crash"}"#,
);
assert_eq!(
shared.lock_inner().route_end_reason(7),
RouteEndReason::Unknown
);
shared.control_push(br#"{"op":"route.closing","module_id":"provider","reason":"restart"}"#);
assert_eq!(
shared.lock_inner().route_end_reason(7),
RouteEndReason::Daemon(RouteCloseReason::Restart)
);
shared.control_push(br#"{"op":"route.closed","module_id":"provider","reason":"crash"}"#);
assert_eq!(
shared.lock_inner().route_end_reason(7),
RouteEndReason::Daemon(RouteCloseReason::Crash)
);
}
#[tokio::test]
async fn sdk_teardown_paths_attach_caller_and_connection_loss_reasons() {
for local in [true, false] {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::new(),
ConsumerOptions::default(),
));
let (writer, _writer_rx) = mpsc::channel(8);
shared.lock_inner().writer = Some(writer);
shared.lock_inner().daemon_ver = "0.39.0".into();
let handle = RouteHandle::new(7, 1, 1);
shared.install_ingress_handle(handle);
let (tx, rx) = oneshot::channel();
let mut entry = PendingEntry::unary(tx, false, None, None);
entry.accepted = true;
shared.lock_inner().pending.insert(
PendingKey {
generation: 1,
channel: 7,
epoch: 1,
corr: 1,
},
entry,
);
if local {
shared
.close_handle(handle, &CloseRouteOptions::default())
.await
.unwrap();
} else {
shared.handle_generation_drop(1, "connection gone".into());
}
let err = rx.await.unwrap().into_call_result().err().unwrap();
assert!(matches!(err, CallError::OutcomeUnknown(_)));
assert_eq!(
err.outcome_cause(),
Some(if local {
OutcomeUnknownCause::RouteEnded
} else {
OutcomeUnknownCause::ConnectionFailed
})
);
let shared_failure = SharedCallFailure::from(err);
let err = shared_failure.into_call_error();
assert_eq!(
err.close_reason(),
Some(&if local {
RouteEndReason::ClosedByCaller
} else {
RouteEndReason::ConnectionLost
})
);
shared.close_sync("test complete");
}
}
#[tokio::test]
async fn route_end_reasons_preserve_pending_retry_classes() {
for reason in [
RouteEndReason::ClosedByCaller,
RouteEndReason::ConnectionLost,
RouteEndReason::Daemon(RouteCloseReason::ScopeEnded),
RouteEndReason::Unknown,
] {
for accepted in [false, true] {
let (tx, rx) = oneshot::channel();
let mut entry = PendingEntry::unary(tx, false, None, None);
entry.accepted = accepted;
settle_route_pending_entries(vec![entry], "route ended".into(), reason.clone());
let err = rx.await.unwrap().into_call_result().err().unwrap();
assert_eq!(err.close_reason(), Some(&reason));
assert_eq!(err.terminal(), None);
assert_eq!(matches!(err, CallError::OutcomeUnknown(_)), accepted);
assert_eq!(matches!(err, CallError::NotSent(_)), !accepted);
}
}
}
#[test]
fn route_closing_does_not_supply_or_overwrite_a_terminal_verdict() {
let shared = Shared::new(PathBuf::new(), ConsumerOptions::default());
shared.control_push(
br#"{"op":"route.closing","channels":[7],"reason":"restart","terminal":true}"#,
);
assert_eq!(shared.lock_inner().route_end(7).terminal, None);
shared.control_push(
br#"{"op":"route.closed","channels":[7],"reason":"restart","terminal":false}"#,
);
shared.control_push(
br#"{"op":"route.closing","channels":[7],"reason":"restart","terminal":true}"#,
);
assert_eq!(shared.lock_inner().route_end(7).terminal, Some(false));
assert_eq!(CallError::not_sent("not a route end").terminal(), None);
}
#[test]
fn route_close_reason_classifier_accepts_capability_denied_and_fails_closed_for_unknown() {
assert_eq!(
RouteCloseReason::from_wire("capability_denied"),
RouteCloseReason::CapabilityDenied
);
assert_eq!(
RouteCloseReason::from_wire("capability_denied").disposition(),
RouteCloseDisposition::MustNotReopen
);
assert_eq!(
RouteCloseReason::from_wire("future_policy_reason").disposition(),
RouteCloseDisposition::MustNotReopen,
"an unknown close reason must receive the strictest handling"
);
assert_eq!(
RouteCloseReason::from_wire("reload").disposition(),
RouteCloseDisposition::MayReopen,
"control proves the classifier can distinguish a conservative default"
);
}
#[tokio::test]
async fn module_removed_fails_fast_while_module_reloading_retries_at_the_same_route_open_call_site(
) {
async fn reject_route_open_attempts(code: &str, deadline: Duration) -> (usize, CallError) {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut receiver) = mpsc::channel(4);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
}
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let target = RouteTarget::ToolProvider {
module_id: "retry-polarity".to_string(),
};
let identity = BindIdentity::new("test".to_string(), code.to_string());
let options = CallOptions {
timeout: Duration::from_secs(2),
route_retry: RetryBackoff {
base: Duration::ZERO,
cap: Duration::ZERO,
max_attempts: 2,
},
route_retry_deadline: deadline,
..CallOptions::default()
};
let mut task =
tokio::spawn(async move { consumer.open_route(target, identity, options).await });
let mut attempts = 0usize;
let result = loop {
tokio::select! {
command = receiver.recv() => {
let Some(command) = command else {
break task.await.unwrap().expect_err("route.open must reject");
};
attempts += 1;
let body =
serde_json::to_vec(&ErrorBody::new(code, "test rejection")).unwrap();
assert!(
dispatch_frame(
&shared,
1,
Frame::build(
FrameType::Error,
Flags::new(false, Priority::Interactive, false),
0,
0,
command.frame.header.corr,
body,
)
.unwrap(),
)
.await
);
}
joined = &mut task => {
break joined.unwrap().expect_err("route.open must reject");
}
}
};
assert!(
receiver.try_recv().is_err(),
"a settled route.open must not queue another attempt"
);
(attempts, result)
}
let (reloading_attempts, reloading) =
reject_route_open_attempts(error_codes::MODULE_RELOADING, Duration::from_millis(150))
.await;
assert!(matches!(reloading, CallError::NotSent(_)));
assert!(
reloading_attempts > 2,
"reloading retries must run until the deadline, not an attempt cap \
(served {reloading_attempts} attempts against max_attempts=2)"
);
let (removed_attempts, removed) =
reject_route_open_attempts(error_codes::MODULE_REMOVED, Duration::from_millis(150))
.await;
assert!(matches!(removed, CallError::NotSent(_)));
assert_eq!(
removed_attempts, 1,
"terminal codes settle on the first answer"
);
}
fn seeded_random(seed: u64) -> Box<dyn Fn() -> f64 + Send + Sync> {
let state = AtomicU64::new(seed);
Box::new(move || {
let mut z = state
.fetch_add(0x9E37_79B9_7F4A_7C15, Ordering::Relaxed)
.wrapping_add(0x9E37_79B9_7F4A_7C15);
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^= z >> 31;
(z >> 11) as f64 / (1u64 << 53) as f64
})
}
fn route_open_stand_in(
random_unit: Box<dyn Fn() -> f64 + Send + Sync>,
) -> (Arc<Shared>, Arc<SubcConsumer>, mpsc::Receiver<WriteCommand>) {
let mut shared = reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
);
shared.random_unit = random_unit;
let shared = Arc::new(shared);
let (writer, receiver) = mpsc::channel(64);
shared.lock_inner().writer = Some(writer);
shared.lock_inner().daemon_ver = "0.39.0".into();
let consumer = Arc::new(SubcConsumer {
shared: Arc::clone(&shared),
});
(shared, consumer, receiver)
}
fn restart_target(module_id: &str) -> RouteTarget {
RouteTarget::ToolProvider {
module_id: module_id.to_string(),
}
}
fn restart_identity() -> BindIdentity {
BindIdentity::new("test".to_string(), "restart".to_string())
}
fn route_open_answer(corr: u64, channel: u16, refusal: Option<ErrorBody>) -> Frame {
match refusal {
None => response_frame(
0,
0,
corr,
serde_json::to_vec(&ClientControlResponse::RouteOpen {
request_window: None,
watch_window: None,
route_channel: channel,
route_epoch: 1,
})
.unwrap(),
),
Some(body) => Frame::build(
FrameType::Error,
Flags::new(false, Priority::Interactive, false),
0,
0,
corr,
serde_json::to_vec(&body).unwrap(),
)
.unwrap(),
}
}
async fn answer_route_opens<T>(
shared: &Arc<Shared>,
receiver: &mut mpsc::Receiver<WriteCommand>,
task: &mut JoinHandle<T>,
mut answer: impl FnMut(&str) -> Option<ErrorBody>,
) -> T {
let mut next_channel = 40u16;
loop {
tokio::select! {
command = receiver.recv() => {
let Some(command) = command else {
return (&mut *task).await.unwrap();
};
let request: serde_json::Value =
serde_json::from_slice(&command.frame.body).unwrap();
let module_id = request["target"]["module_id"].as_str().unwrap().to_string();
next_channel += 1;
let frame = route_open_answer(
command.frame.header.corr,
next_channel,
answer(&module_id),
);
assert!(dispatch_frame(shared, 1, frame).await);
}
joined = &mut *task => return joined.unwrap(),
}
}
}
fn reloading() -> ErrorBody {
ErrorBody::new(error_codes::MODULE_RELOADING, "module is reloading")
}
fn drain_call<'a>(
consumer: &'a SubcConsumer,
body: &'static [u8],
) -> impl Future<Output = Result<Vec<u8>, CallError>> + 'a {
consumer.call(
restart_target("plexus"),
restart_identity(),
body.to_vec(),
CallOptions {
timeout: Duration::from_secs(5),
..CallOptions::default()
},
)
}
async fn refuse_during_drain(shared: &Arc<Shared>, request: &Frame) {
let frame = Frame::build(
FrameType::Error,
request.header.flags,
request.header.channel,
request.header.epoch,
request.header.corr,
serde_json::to_vec(&reloading()).unwrap(),
)
.unwrap();
assert!(dispatch_frame(shared, 1, frame).await);
}
async fn reply_to_drain_call(shared: &Arc<Shared>, request: Frame) {
assert_eq!(request.header.ty, FrameType::Request);
assert!(
dispatch_frame(
shared,
1,
response_frame(
request.header.channel,
request.header.epoch,
request.header.corr,
request.body,
),
)
.await
);
}
#[tokio::test(start_paused = true)]
async fn shared_route_drain_delivers_both_refusals_and_resends_both_calls() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(1));
let (_, old) = open_answered(
&shared,
&consumer,
&mut receiver,
None,
41,
"initial shared route",
)
.await;
let mut seed = Box::pin(drain_call(&consumer, b"seed"));
assert_pending(seed.as_mut()).await;
reply_to_drain_call(&shared, receiver.recv().await.unwrap().frame).await;
assert_eq!(seed.await.unwrap(), b"seed");
shared.control_push(
&serde_json::to_vec(&serde_json::json!({
"op": "route.closing", "module_id": "plexus",
"channels": [old.channel], "terminal": false
}))
.unwrap(),
);
let mut first = Box::pin(drain_call(&consumer, b"first"));
let mut second = Box::pin(drain_call(&consumer, b"second"));
assert_pending(first.as_mut()).await;
assert_pending(second.as_mut()).await;
let first_command = receiver.recv().await.unwrap();
let second_command = receiver.recv().await.unwrap();
assert!(shared.mark_pending_accepted(first_command.pending.unwrap()));
assert!(shared.mark_pending_accepted(second_command.pending.unwrap()));
let first_request = first_command.frame;
let second_request = second_command.frame;
assert_eq!(first_request.header.channel, old.channel);
assert_eq!(second_request.header.channel, old.channel);
refuse_during_drain(&shared, &first_request).await;
assert_pending(first.as_mut()).await;
let (open_corr, _) = next_route_open(&mut receiver, "replacement route barrier").await;
refuse_during_drain(&shared, &second_request).await;
for (ty, channel, epoch, corr) in [
(FrameType::Goodbye, old.channel, old.epoch, 0),
(FrameType::Ping, 0, 0, 991),
] {
let frame = Frame::build(
ty,
Flags::new(false, Priority::Interactive, false),
channel,
epoch,
corr,
Vec::new(),
)
.unwrap();
assert!(dispatch_frame(&shared, 1, frame).await);
}
let pong = receiver.recv().await.unwrap().frame;
assert_eq!(pong.header.ty, FrameType::Pong);
assert_eq!(pong.header.corr, 991);
assert_pending(second.as_mut()).await;
assert!(dispatch_frame(&shared, 1, route_open_answer(open_corr, 42, None)).await);
assert_pending(first.as_mut()).await;
assert_pending(second.as_mut()).await;
while let Ok(command) = receiver.try_recv() {
assert_eq!(command.frame.header.channel, 42);
reply_to_drain_call(&shared, command.frame).await;
}
assert_eq!(first.await.expect("first call resends"), b"first");
assert_eq!(
second.await.expect("second refusal must not time out"),
b"second"
);
assert!(shared.lock_inner().pending.is_empty());
}
#[tokio::test(start_paused = true)]
async fn shared_route_drain_preserves_unrefused_reply_without_resending() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(1));
let (_, old) = open_answered(
&shared,
&consumer,
&mut receiver,
None,
41,
"initial shared route",
)
.await;
let mut first = Box::pin(drain_call(&consumer, b"refused"));
let mut second = Box::pin(drain_call(&consumer, b"executing"));
assert_pending(first.as_mut()).await;
assert_pending(second.as_mut()).await;
let first_request = receiver.recv().await.unwrap().frame;
let second_request = receiver.recv().await.unwrap().frame;
let executing_corr = second_request.header.corr;
assert_eq!(second_request.header.channel, old.channel);
let mut executing_requests = 1;
refuse_during_drain(&shared, &first_request).await;
assert_pending(first.as_mut()).await;
let (open_corr, _) = next_route_open(&mut receiver, "replacement route barrier").await;
assert_pending(second.as_mut()).await;
assert!(dispatch_frame(&shared, 1, route_open_answer(open_corr, 42, None)).await);
assert_pending(first.as_mut()).await;
assert_pending(second.as_mut()).await;
while let Ok(command) = receiver.try_recv() {
assert_eq!(command.frame.header.ty, FrameType::Request);
if command.frame.header.corr == executing_corr || command.frame.body == b"executing" {
executing_requests += 1;
}
reply_to_drain_call(&shared, command.frame).await;
}
reply_to_drain_call(&shared, second_request).await;
assert_eq!(first.await.expect("refused call resends"), b"refused");
assert_eq!(
second.await.expect("real old-route reply arrives"),
b"executing"
);
assert_eq!(executing_requests, 1, "unrefused call must never resend");
}
#[tokio::test(start_paused = true)]
async fn invalidated_route_ingress_is_uninstalled_after_last_pending_settles() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(1));
let (_, old) = open_answered(
&shared,
&consumer,
&mut receiver,
None,
41,
"initial shared route",
)
.await;
let mut first = Box::pin(drain_call(&consumer, b"first"));
let mut second = Box::pin(drain_call(&consumer, b"second"));
assert_pending(first.as_mut()).await;
assert_pending(second.as_mut()).await;
let first_request = receiver.recv().await.unwrap().frame;
let second_request = receiver.recv().await.unwrap().frame;
refuse_during_drain(&shared, &first_request).await;
assert_pending(first.as_mut()).await;
let _ = next_route_open(&mut receiver, "replacement route barrier").await;
assert_eq!(shared.ingress_handle(1, old.channel, old.epoch), Some(old));
assert!(route_reverse_request_registry(old.reverse_request_registry_id()).is_ok());
reply_to_drain_call(&shared, second_request).await;
assert_eq!(second.await.unwrap(), b"second");
assert_eq!(shared.ingress_handle(1, old.channel, old.epoch), None);
assert!(shared.lock_inner().retired_routes.is_empty());
assert!(route_reverse_request_registry(old.reverse_request_registry_id()).is_err());
drop(first);
}
#[tokio::test(start_paused = true)]
async fn invalidated_route_ingress_cleans_up_on_drop_deadline_goodbye_and_disconnect() {
for ending in ["drop", "deadline", "goodbye", "disconnect"] {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(1));
let (_, old) = open_answered(
&shared,
&consumer,
&mut receiver,
None,
41,
"initial shared route",
)
.await;
let mut first = Box::pin(drain_call(&consumer, b"refused"));
let mut second = Box::pin(drain_call(&consumer, b"executing"));
assert_pending(first.as_mut()).await;
assert_pending(second.as_mut()).await;
let first_request = receiver.recv().await.unwrap().frame;
let second_command = receiver.recv().await.unwrap();
assert!(shared.mark_pending_accepted(second_command.pending.unwrap()));
refuse_during_drain(&shared, &first_request).await;
assert_pending(first.as_mut()).await;
let _ = next_route_open(&mut receiver, "replacement route barrier").await;
match ending {
"drop" => drop(second),
"deadline" => {
assert_eq!(
second.await.unwrap_err().outcome_cause(),
Some(OutcomeUnknownCause::Deadline)
);
}
"goodbye" => {
let frame = Frame::build(
FrameType::Goodbye,
Flags::new(false, Priority::Interactive, false),
old.channel,
old.epoch,
0,
Vec::new(),
)
.unwrap();
assert!(dispatch_frame(&shared, 1, frame).await);
assert_eq!(
second.await.unwrap_err().outcome_cause(),
Some(OutcomeUnknownCause::RouteEnded)
);
}
"disconnect" => {
shared.handle_generation_drop(1, "test disconnect".into());
assert_eq!(
second.await.unwrap_err().outcome_cause(),
Some(OutcomeUnknownCause::ConnectionFailed)
);
}
_ => unreachable!(),
}
assert_eq!(
shared.ingress_handle(1, old.channel, old.epoch),
None,
"{ending}"
);
assert!(shared.lock_inner().retired_routes.is_empty(), "{ending}");
assert!(
route_reverse_request_registry(old.reverse_request_registry_id()).is_err(),
"{ending}"
);
drop(first);
}
}
#[tokio::test(start_paused = true)]
async fn invalidated_route_subscription_cancel_and_overflow_uninstall_ingress() {
for overflow in [false, true] {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(1));
let (_, old) = open_answered(
&shared,
&consumer,
&mut receiver,
None,
41,
"initial shared route",
)
.await;
let mut subscription = consumer
.subscribe_route(&old, b"watch".to_vec(), SubscribeOptions::default())
.await
.unwrap();
let request = receiver.recv().await.unwrap().frame;
let mut call = Box::pin(drain_call(&consumer, b"refused"));
assert_pending(call.as_mut()).await;
let refused = receiver.recv().await.unwrap().frame;
refuse_during_drain(&shared, &refused).await;
assert_pending(call.as_mut()).await;
let _ = next_route_open(&mut receiver, "replacement route barrier").await;
assert_eq!(shared.ingress_handle(1, old.channel, old.epoch), Some(old));
assert!(matches!(
consumer
.request(&old, b"new".to_vec(), CallOptions::default())
.await,
Err(CallError::StaleRouteHandle(_))
));
if overflow {
for _ in 0..=DEFAULT_SUBSCRIPTION_EVENT_BUFFER {
assert!(
dispatch_frame(
&shared,
1,
watch_terminal(&request, FrameType::StreamData, vec![1])
)
.await
);
}
assert_eq!(subscription.ended().await, SubscriptionEnd::Backpressure);
} else {
subscription.unsubscribe().unwrap();
assert_eq!(subscription.ended().await, SubscriptionEnd::Cancelled);
}
assert_eq!(
receiver.recv().await.unwrap().frame.header.ty,
FrameType::Cancel
);
assert_eq!(shared.ingress_handle(1, old.channel, old.epoch), None);
assert!(shared.lock_inner().retired_routes.is_empty());
drop(call);
}
}
#[tokio::test(start_paused = true)]
async fn the_default_retry_deadline_outlasts_a_60s_restart_and_the_open_succeeds() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(1));
let started = Instant::now();
let options = CallOptions {
timeout: Duration::from_secs(120),
..CallOptions::default()
};
let mut task = tokio::spawn(async move {
consumer
.open_route(restart_target("restarting"), restart_identity(), options)
.await
});
let opened = answer_route_opens(&shared, &mut receiver, &mut task, |_| {
(started.elapsed() < Duration::from_secs(60)).then(reloading)
})
.await;
let handle =
opened.unwrap_or_else(|err| panic!("the open must survive the restart: {err}"));
assert!(handle.channel > 40);
assert!(started.elapsed() >= Duration::from_secs(60));
}
#[tokio::test(start_paused = true)]
async fn a_call_timeout_shorter_than_the_retry_deadline_ends_the_retries_at_the_call_timeout() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(2));
let started = Instant::now();
let options = CallOptions {
timeout: Duration::from_secs(10),
..CallOptions::default()
};
let mut task = tokio::spawn(async move {
consumer
.open_route(restart_target("restarting"), restart_identity(), options)
.await
});
let err = answer_route_opens(&shared, &mut receiver, &mut task, |_| {
(started.elapsed() < Duration::from_secs(60)).then(reloading)
})
.await
.expect_err("the call timeout must end the retries before the module is back");
let elapsed = started.elapsed();
assert!(matches!(err, CallError::NotSent(_)), "got {err:?}");
assert!(
elapsed >= Duration::from_secs(9) && elapsed <= Duration::from_secs(10),
"the retries must stop at the 10s call timeout, stopped after {elapsed:?}"
);
}
#[tokio::test(start_paused = true)]
async fn two_routes_refused_together_retry_at_different_jittered_times() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(42));
let started = Instant::now();
let mut task = tokio::spawn(async move {
let alpha = consumer.open_route(
restart_target("alpha"),
restart_identity(),
CallOptions::default(),
);
let beta = consumer.open_route(
restart_target("beta"),
restart_identity(),
CallOptions::default(),
);
let (alpha, beta) = tokio::join!(alpha, beta);
(alpha.is_ok(), beta.is_ok())
});
let mut first_open = HashMap::<String, Duration>::new();
let mut retried_at = HashMap::<String, Duration>::new();
let opened = answer_route_opens(&shared, &mut receiver, &mut task, |module_id| {
let at = started.elapsed();
if first_open.contains_key(module_id) {
retried_at.insert(module_id.to_string(), at);
None
} else {
first_open.insert(module_id.to_string(), at);
Some(reloading())
}
})
.await;
assert_eq!(opened, (true, true));
assert_eq!(first_open["alpha"], first_open["beta"], "refused together");
for at in retried_at.values() {
assert!(
*at >= Duration::from_millis(50) && *at <= Duration::from_millis(100),
"retry at {at:?} is outside the jitter window"
);
}
assert_ne!(
retried_at["alpha"], retried_at["beta"],
"routes refused together must not retry in lock step"
);
}
#[tokio::test(start_paused = true)]
async fn twenty_concurrent_opens_never_have_more_than_eight_outstanding_at_the_daemon() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(3));
let mut task = tokio::spawn(async move {
let mut opens = tokio::task::JoinSet::new();
for i in 0..20 {
let consumer = Arc::clone(&consumer);
opens.spawn(async move {
consumer
.open_route(
restart_target(&format!("module-{i}")),
restart_identity(),
CallOptions::default(),
)
.await
});
}
let mut opened = 0usize;
while let Some(result) = opens.join_next().await {
result.unwrap().unwrap();
opened += 1;
}
opened
});
let mut held = Vec::new();
let mut max_outstanding = 0usize;
let mut next_channel = 40u16;
let opened = loop {
tokio::select! {
command = receiver.recv() => {
let Some(command) = command else {
break (&mut task).await.unwrap();
};
held.push(command.frame.header.corr);
max_outstanding = max_outstanding.max(held.len());
}
() = sleep(Duration::from_millis(25)), if !held.is_empty() => {
for corr in held.drain(..) {
next_channel += 1;
assert!(dispatch_frame(&shared, 1, route_open_answer(corr, next_channel, None)).await);
}
}
joined = &mut task => break joined.unwrap(),
}
};
assert_eq!(opened, 20);
assert_eq!(max_outstanding, MAX_ROUTE_OPENS_IN_FLIGHT);
}
#[tokio::test(start_paused = true)]
async fn an_exhausted_deadline_names_module_reloading_not_the_later_admission_pressure() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(4));
let options = CallOptions {
route_retry_deadline: Duration::from_secs(5),
..CallOptions::default()
};
let mut task = tokio::spawn(async move {
consumer
.open_route(restart_target("restarting"), restart_identity(), options)
.await
});
let mut opens = 0usize;
let err = answer_route_opens(&shared, &mut receiver, &mut task, |_| {
opens += 1;
Some(if opens <= 2 {
reloading().with_detail(serde_json::json!({ "reason": "restart" }))
} else {
ErrorBody::new(
error_codes::TARGET_UNAVAILABLE,
"connection already has 8 route.open binds in flight (limit 8); retry after one settles",
)
})
})
.await
.expect_err("the module never comes back");
assert!(opens > 2);
assert!(matches!(err, CallError::NotSent(_)), "got {err:?}");
assert_eq!(
err.route_open_refusal().map(|body| body.code.as_str()),
Some(error_codes::MODULE_RELOADING)
);
let message = err.to_string();
assert!(
message.starts_with(
"request not sent: route.open failed for target tool_provider:restarting: \
module_reloading (reason restart) (module is reloading); \
the most recent refusal was target_unavailable"
),
"got {message}"
);
}
#[tokio::test]
async fn close_route_flips_inflight_opening_so_a_racing_open_discards() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let key = RouteKey::new(
&RouteTarget::ToolProvider {
module_id: "m".into(),
},
&BindIdentity::new("h", "s"),
None,
None,
);
shared.lock_inner().openings.insert(
key.clone(),
Opening {
waiters: Vec::new(),
closed: false,
},
);
shared
.close_route(&key, &CloseRouteOptions::default())
.await;
assert!(
shared
.lock_inner()
.openings
.get(&key)
.is_some_and(|o| o.closed),
"close_route must flip the in-flight opening's closed flag (close-beats-reopen)"
);
let absent = RouteKey::new(
&RouteTarget::ToolProvider {
module_id: "absent".into(),
},
&BindIdentity::new("h", "s"),
None,
None,
);
shared
.close_route(&absent, &CloseRouteOptions::default())
.await;
}
#[test]
fn route_key_is_structured() {
let target = RouteTarget::InternalService {
module_id: "a\0b".into(),
service_id: "svc".into(),
};
let identity = BindIdentity::new("h", "s");
let key = RouteKey::new(&target, &identity, None, None);
assert!(matches!(key.target, RouteTargetKey::InternalService { .. }));
}
async fn assert_pending(mut future: Pin<&mut (impl Future + ?Sized)>) {
assert!(
std::future::poll_fn(|cx| Poll::Ready(future.as_mut().poll(cx).is_pending())).await
);
}
fn drop_and_replace_test_transport(shared: &Arc<Shared>) -> mpsc::Receiver<WriteCommand> {
let generation = shared.lock_inner().generation;
shared.lock_inner().reconnect = ReconnectState::Inline { generation };
shared.handle_generation_drop(generation, "test connection drop".into());
let (writer, receiver) = mpsc::channel(64);
let mut inner = shared.lock_inner();
inner.generation += 1;
inner.epoch += 1;
inner.next_corr = Some(1);
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.reconnect = ReconnectState::Idle;
receiver
}
#[tokio::test(start_paused = true)]
async fn route_open_single_flight_survives_generation_drop_during_backoff() {
let (shared, consumer, mut receiver) = route_open_stand_in(Box::new(|| 0.5));
let opts = CallOptions::default();
let opener = consumer.open_route(restart_target("m"), restart_identity(), opts.clone());
tokio::pin!(opener);
assert_pending(opener.as_mut()).await;
let first = receiver.try_recv().unwrap();
assert!(
dispatch_frame(
&shared,
1,
route_open_answer(first.frame.header.corr, 0, Some(reloading())),
)
.await
);
assert_pending(opener.as_mut()).await;
let waiting = consumer.open_route(restart_target("m"), restart_identity(), opts.clone());
tokio::pin!(waiting);
assert_pending(waiting.as_mut()).await;
let mut receiver = drop_and_replace_test_transport(&shared);
assert_pending(waiting.as_mut()).await;
let arriving = consumer.open_route(restart_target("m"), restart_identity(), opts.clone());
tokio::pin!(arriving);
assert_pending(arriving.as_mut()).await;
assert!(
receiver.try_recv().is_err(),
"a second opener bypassed the backoff"
);
tokio::time::advance(opts.route_retry.base).await;
assert_pending(opener.as_mut()).await;
let retry = receiver.try_recv().unwrap();
assert!(
dispatch_frame(
&shared,
2,
route_open_answer(retry.frame.header.corr, 42, None),
)
.await
);
let handle = opener.await.unwrap();
assert_eq!(waiting.await.unwrap(), handle);
assert_eq!(arriving.await.unwrap(), handle);
assert!(
receiver.try_recv().is_err(),
"more than one retry was emitted"
);
let inner = shared.lock_inner();
assert!(inner.openings.is_empty());
assert_eq!(inner.routes.len(), 1);
assert_eq!(inner.route_epochs.len(), 1);
}
#[tokio::test(start_paused = true)]
async fn close_route_during_backoff_survives_generation_drop() {
let (shared, consumer, mut receiver) = route_open_stand_in(Box::new(|| 0.5));
let opts = CallOptions::default();
let opener = consumer.open_route(restart_target("m"), restart_identity(), opts.clone());
tokio::pin!(opener);
assert_pending(opener.as_mut()).await;
let first = receiver.try_recv().unwrap();
assert!(
dispatch_frame(
&shared,
1,
route_open_answer(first.frame.header.corr, 0, Some(reloading())),
)
.await
);
assert_pending(opener.as_mut()).await;
consumer
.close_route(
restart_target("m"),
restart_identity(),
CloseRouteOptions::default(),
)
.await;
let mut receiver = drop_and_replace_test_transport(&shared);
tokio::time::advance(opts.route_retry.base).await;
assert_pending(opener.as_mut()).await;
let retry = receiver.try_recv().unwrap();
assert!(
dispatch_frame(
&shared,
2,
route_open_answer(retry.frame.header.corr, 42, None),
)
.await
);
let err = opener.await.expect_err("close must win over the retry");
assert!(matches!(err, CallError::NotSent(_)));
let goodbye = receiver
.try_recv()
.expect("discarded route must get GOODBYE");
assert_eq!(goodbye.frame.header.ty, FrameType::Goodbye);
assert_eq!(goodbye.frame.header.channel, 42);
assert!(receiver.try_recv().is_err());
let inner = shared.lock_inner();
assert!(inner.routes.is_empty());
assert!(inner.route_epochs.is_empty());
assert!(inner.openings.is_empty());
}
fn route_teardown_stand_in() -> (
Arc<Shared>,
SubcConsumer,
mpsc::Receiver<WriteCommand>,
tokio::io::DuplexStream,
RouteKey,
RouteState,
) {
let shared = writer_test_shared();
let (stream, peer) = tokio::io::duplex(1024);
shared
.install_initial(OpenedConnection {
stream: Box::new(stream),
daemon_origin_trusted: true,
daemon_ver: "0.39.0".into(),
})
.unwrap();
let (writer, rx) = mpsc::channel(1);
writer
.try_send(WriteCommand {
frame: response_frame(0, 0, 99, b"filler".to_vec()),
pending: None,
})
.unwrap();
let key = RouteKey::new(&restart_target("teardown"), &restart_identity(), None, None);
let route = RouteState {
handle: RouteHandle::new(17, 23, 1),
sem: Arc::new(Semaphore::new(1)),
watch_sem: None,
opted_in: false,
};
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.cache_route(key.clone(), route.clone());
}
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
(shared, consumer, rx, peer, key, route)
}
async fn assert_reopened_request_after_goodbye(
shared: &Arc<Shared>,
consumer: &SubcConsumer,
rx: &mut mpsc::Receiver<WriteCommand>,
old: RouteHandle,
) {
let reopened = RouteHandle::new(old.channel, old.epoch + 1, old.connection_token());
shared
.install_one_shot_route(RouteState {
handle: reopened,
sem: Arc::new(Semaphore::new(1)),
watch_sem: None,
opted_in: false,
})
.unwrap();
let request = consumer.request(&reopened, b"reopened".to_vec(), CallOptions::default());
tokio::pin!(request);
assert_pending(request.as_mut()).await;
let command = rx.recv().await.unwrap();
assert_eq!(command.frame.header.ty, FrameType::Goodbye);
assert_eq!(
(command.frame.header.channel, command.frame.header.epoch),
(old.channel, old.epoch)
);
assert_pending(request.as_mut()).await;
let command = rx.recv().await.unwrap();
assert_eq!(command.frame.header.ty, FrameType::Request);
assert_eq!(command.frame.header.channel, reopened.channel);
assert_eq!(command.frame.header.epoch, reopened.epoch);
dispatch_frame(
shared,
reopened.connection_token(),
response_frame(
reopened.channel,
reopened.epoch,
command.frame.header.corr,
Vec::new(),
),
)
.await;
request.await.unwrap();
}
#[tokio::test]
async fn route_teardown_close_waits_for_goodbye_before_channel_reuse() {
for by_handle in [false, true] {
let (shared, consumer, mut rx, _peer, key, route) = route_teardown_stand_in();
let close = async {
if by_handle {
consumer
.close_handle(&route.handle, CloseRouteOptions::default())
.await
.unwrap();
} else {
consumer
.close_route(
restart_target("teardown"),
restart_identity(),
CloseRouteOptions::default(),
)
.await;
}
};
tokio::pin!(close);
assert_pending(close.as_mut()).await;
assert!(route.sem.is_closed());
assert!(!shared.lock_inner().routes.contains_key(&key));
assert_eq!(
shared.lock_inner().route_epochs.get(&route.handle.channel),
Some(&route.handle)
);
let mut reopened = route.clone();
reopened.handle = RouteHandle::new(route.handle.channel, route.handle.epoch + 1, 1);
assert!(shared.install_one_shot_route(reopened).is_err());
assert_eq!(rx.recv().await.unwrap().frame.body, b"filler");
close.await;
assert_reopened_request_after_goodbye(&shared, &consumer, &mut rx, route.handle).await;
consumer.close().await;
}
}
#[tokio::test]
async fn route_teardown_retirement_waits_for_goodbye_before_channel_reuse() {
let (shared, consumer, mut rx, _peer, key, route) = route_teardown_stand_in();
shared.invalidate_route_with_deadline(&key, Some(route.handle), true);
assert!(route.sem.is_closed());
assert_eq!(
shared.lock_inner().route_epochs.get(&route.handle.channel),
Some(&route.handle)
);
let mut reopened = route.clone();
reopened.handle = RouteHandle::new(route.handle.channel, route.handle.epoch + 1, 1);
assert!(shared.install_one_shot_route(reopened).is_err());
assert_eq!(rx.recv().await.unwrap().frame.body, b"filler");
tokio::time::timeout(Duration::from_secs(1), async {
while shared
.lock_inner()
.route_epochs
.contains_key(&route.handle.channel)
{
tokio::task::yield_now().await;
}
})
.await
.expect("retirement must queue its GOODBYE after writer capacity frees");
assert_reopened_request_after_goodbye(&shared, &consumer, &mut rx, route.handle).await;
consumer.close().await;
}
#[tokio::test]
async fn route_teardown_retirements_flush_in_order() {
let (shared, consumer, mut rx, _peer, first_key, first) = route_teardown_stand_in();
let second_key = RouteKey::new(&restart_target("second"), &restart_identity(), None, None);
let mut second = first.clone();
second.handle = RouteHandle::new(first.handle.channel + 1, 31, 1);
shared
.lock_inner()
.cache_route(second_key.clone(), second.clone());
shared.invalidate_route_with_deadline(&first_key, Some(first.handle), true);
shared.invalidate_route_with_deadline(&second_key, Some(second.handle), true);
assert_eq!(rx.recv().await.unwrap().frame.body, b"filler");
for handle in [first.handle, second.handle] {
let command = tokio::time::timeout(Duration::from_secs(1), rx.recv())
.await
.unwrap()
.unwrap();
assert_eq!(command.frame.header.ty, FrameType::Goodbye);
assert_eq!(
(command.frame.header.channel, command.frame.header.epoch),
(handle.channel, handle.epoch)
);
assert!(!shared
.lock_inner()
.route_epochs
.contains_key(&handle.channel));
}
assert!(shared.lock_inner().retirement_queue.is_empty());
assert!(shared.lock_inner().retirement_goodbyes.is_empty());
assert!(rx.try_recv().is_err());
consumer.close().await;
}
#[tokio::test]
async fn route_teardown_dead_connection_ends_writer_wait_and_discards_retirements() {
for death in 0..3 {
let (shared, consumer, mut rx, _peer, key, route) = route_teardown_stand_in();
let opts = CloseRouteOptions::default();
let close = shared.close_route(&key, &opts);
tokio::pin!(close);
assert_pending(close.as_mut()).await;
match death {
0 => rx.close(),
1 => shared.handle_generation_drop(1, "test connection lost".into()),
_ => shared.close_sync("test connection closed"),
}
tokio::time::timeout(Duration::from_secs(1), close)
.await
.unwrap();
assert!(!shared
.lock_inner()
.route_epochs
.contains_key(&route.handle.channel));
assert_eq!(rx.recv().await.unwrap().frame.body, b"filler");
assert!(rx.try_recv().is_err());
consumer.close().await;
let (shared, consumer, mut rx, _peer, key, route) = route_teardown_stand_in();
shared.invalidate_route_with_deadline(&key, Some(route.handle), true);
if death == 1 {
shared.handle_generation_drop(1, "test connection lost".into());
} else {
consumer.close().await;
}
assert!(shared.lock_inner().retirement_queue.is_empty());
assert!(shared.lock_inner().retirement_goodbyes.is_empty());
assert_eq!(rx.recv().await.unwrap().frame.body, b"filler");
assert!(rx.try_recv().is_err());
consumer.close().await;
}
}
#[tokio::test]
async fn route_channel_index_tracks_lookup_close_and_generation_drop() {
let shared = writer_test_shared();
let (writer, _rx) = mpsc::channel(32);
let mut expected = Vec::new();
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
for channel in 1..=8 {
let key = RouteKey::new(
&RouteTarget::ToolProvider {
module_id: format!("module-{channel}"),
},
&BindIdentity::new("test", format!("session-{channel}")),
None,
None,
);
let route = RouteState {
handle: RouteHandle::new(channel, channel.into(), 1),
sem: Arc::new(Semaphore::new(DEFAULT_ROUTE_WINDOW)),
watch_sem: None,
opted_in: false,
};
inner.cache_route(key.clone(), route.clone());
expected.push((key, route));
}
}
for (key, expected_route) in &expected {
let resolved = shared
.route_state(expected_route.handle)
.expect("an indexed route handle should resolve");
assert_eq!(resolved.handle, expected_route.handle);
assert!(Arc::ptr_eq(&resolved.sem, &expected_route.sem));
let inner = shared.lock_inner();
assert_eq!(
inner.route_by_channel.get(&expected_route.handle.channel),
Some(key)
);
assert_eq!(
inner.route_epochs.get(&expected_route.handle.channel),
Some(&expected_route.handle)
);
assert!(inner
.routes
.get(key)
.is_some_and(|route| route.handle == expected_route.handle));
}
let (closed_key, closed_route) = &expected[3];
shared
.close_route(closed_key, &CloseRouteOptions::default())
.await;
{
let inner = shared.lock_inner();
assert!(!inner.routes.contains_key(closed_key));
assert!(!inner
.route_by_channel
.contains_key(&closed_route.handle.channel));
assert!(!inner
.route_epochs
.contains_key(&closed_route.handle.channel));
}
assert!(matches!(
shared.route_state(closed_route.handle),
Err(CallError::StaleRouteHandle(handle)) if handle == closed_route.handle
));
shared.handle_generation_drop(1, "test generation dropped".into());
{
let inner = shared.lock_inner();
assert!(inner.routes.is_empty());
assert!(inner.route_by_channel.is_empty());
assert!(inner.route_epochs.is_empty());
}
shared.close_sync("test complete");
}
#[tokio::test]
async fn stale_push_is_not_delivered_after_connection_generation_changes() {
let shared = writer_test_shared();
let old_handle = RouteHandle::new(7, 1, 1);
let (writer, _writer_rx) = mpsc::channel(1);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.route_epochs.insert(old_handle.channel, old_handle);
}
let mut pushes = shared
.register_push_events(old_handle)
.expect("the old live route should accept a receiver");
{
let mut inner = shared.lock_inner();
inner.generation = 2;
inner.close_routes();
inner.route_epochs.clear();
inner.route_modules.clear();
inner.route_end_reasons.clear();
inner.legacy_channel_reasons.clear();
}
shared.route_push(old_handle, b"stale".to_vec());
assert!(
pushes.recv().await.is_none(),
"connection teardown must end the old receiver before a stale Push can arrive"
);
shared.close_sync("test complete");
}
#[test]
fn route_key_canonicalizes_consumer_capabilities() {
let target = RouteTarget::ToolProvider {
module_id: "aft".into(),
};
let identity = BindIdentity::new("h", "s");
let left = RouteKey::new(
&target,
&identity,
None,
Some(&["sampling".to_string(), "elicitation".to_string()]),
);
let right = RouteKey::new(
&target,
&identity,
None,
Some(&[
"elicitation".to_string(),
"sampling".to_string(),
"sampling".to_string(),
]),
);
assert_eq!(left, right);
}
#[tokio::test]
async fn routes_with_and_without_declared_consumer_capabilities_are_kept_apart() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(10));
let (undeclared_request, undeclared) = open_answered_with(
&shared,
&consumer,
&mut receiver,
CallOptions::default(),
74,
"the undeclared route opens first",
)
.await;
assert!(undeclared_request.get("consumer_capabilities").is_none());
let declared_opts = CallOptions::default()
.with_consumer_capability("tool-result/v1")
.unwrap();
let (declared_request, declared) = open_answered_with(
&shared,
&consumer,
&mut receiver,
declared_opts.clone(),
75,
"a declared open must not reuse the undeclared route",
)
.await;
assert_eq!(
declared_request["consumer_capabilities"],
serde_json::json!(["tool-result/v1"])
);
assert_ne!(undeclared, declared);
assert_eq!(
consumer
.open_route(restart_target("plexus"), restart_identity(), declared_opts)
.await
.unwrap(),
declared,
"the same declaration must reuse its cached route"
);
assert_eq!(
consumer
.open_route(
restart_target("plexus"),
restart_identity(),
CallOptions::default(),
)
.await
.unwrap(),
undeclared,
"an undeclared open must continue to reuse only its own route"
);
assert!(
receiver.try_recv().is_err(),
"both subsequent opens are cache hits"
);
shared.close_sync("test complete");
}
fn scope_selector(owner: &str, scope_ref: &str, scope_epoch: u64) -> ScopeSelector {
ScopeSelector::new(
subc_protocol::Principal::Reserved {
module_id: owner.to_string(),
},
scope_ref.to_string(),
Some(scope_epoch),
)
}
#[test]
fn route_key_separates_scopes_by_owner_ref_and_epoch_and_from_unscoped() {
let target = RouteTarget::ToolProvider {
module_id: "plexus".into(),
};
let identity = BindIdentity::new("h", "s");
let key = |scope: Option<ScopeSelector>| {
RouteKey::new(&target, &identity, None, None).with_scope(scope.as_ref())
};
let base = key(Some(scope_selector("owner", "session-a", 1)));
assert_eq!(base, key(Some(scope_selector("owner", "session-a", 1))));
for (label, other) in [
("epoch", key(Some(scope_selector("owner", "session-a", 2)))),
("ref", key(Some(scope_selector("owner", "session-b", 1)))),
("owner", key(Some(scope_selector("other", "session-a", 1)))),
("unscoped", key(None)),
] {
assert_ne!(base, other, "a different {label} must be a different route");
}
let mut no_epoch = scope_selector("owner", "session-a", 1);
no_epoch.scope_epoch = None;
assert_ne!(base, key(Some(no_epoch)));
}
async fn next_route_open(
receiver: &mut mpsc::Receiver<WriteCommand>,
why: &str,
) -> (u64, serde_json::Value) {
loop {
let command = tokio::time::timeout(Duration::from_secs(2), receiver.recv())
.await
.unwrap_or_else(|_| panic!("no route.open was sent: {why}"))
.expect("writer channel open");
if command.frame.header.channel != 0 || command.frame.header.ty != FrameType::Request {
continue;
}
let request: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
if request["op"] == "route.open" {
return (command.frame.header.corr, request);
}
}
}
async fn open_answered(
shared: &Arc<Shared>,
consumer: &Arc<SubcConsumer>,
receiver: &mut mpsc::Receiver<WriteCommand>,
scope: Option<ScopeSelector>,
channel: u16,
why: &str,
) -> (serde_json::Value, RouteHandle) {
let consumer = Arc::clone(consumer);
let task = tokio::spawn(async move {
match scope {
Some(scope) => {
consumer
.open_route_scoped(
restart_target("plexus"),
restart_identity(),
scope,
CallOptions::default(),
)
.await
}
None => {
consumer
.open_route(
restart_target("plexus"),
restart_identity(),
CallOptions::default(),
)
.await
}
}
});
let (corr, request) = next_route_open(receiver, why).await;
assert!(dispatch_frame(shared, 1, route_open_answer(corr, channel, None)).await);
let handle = task.await.unwrap().expect("the stand-in accepted the open");
(request, handle)
}
async fn open_answered_with(
shared: &Arc<Shared>,
consumer: &Arc<SubcConsumer>,
receiver: &mut mpsc::Receiver<WriteCommand>,
opts: CallOptions,
channel: u16,
why: &str,
) -> (serde_json::Value, RouteHandle) {
let consumer = Arc::clone(consumer);
let task = tokio::spawn(async move {
consumer
.open_route(restart_target("plexus"), restart_identity(), opts)
.await
});
let (corr, request) = next_route_open(receiver, why).await;
assert!(dispatch_frame(shared, 1, route_open_answer(corr, channel, None)).await);
let handle = task.await.unwrap().expect("the stand-in accepted the open");
(request, handle)
}
fn with_role_versions(entries: &[(&str, &str)]) -> CallOptions {
CallOptions {
role_versions: Some(
entries
.iter()
.map(|(role, version)| (role.to_string(), version.to_string()))
.collect(),
),
..CallOptions::default()
}
}
#[tokio::test]
async fn routes_differing_only_in_role_versions_are_kept_apart() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(9));
let (legacy_request, legacy) = open_answered_with(
&shared,
&consumer,
&mut receiver,
CallOptions::default(),
71,
"first, legacy open",
)
.await;
assert!(
legacy_request.get("role_versions").is_none(),
"{legacy_request}"
);
let (v1_request, v1) = open_answered_with(
&shared,
&consumer,
&mut receiver,
with_role_versions(&[("tool-provider", "v1")]),
72,
"a v1 open must not reuse the legacy route",
)
.await;
assert_eq!(
v1_request["role_versions"],
serde_json::json!({ "tool-provider": "v1" })
);
assert_ne!(legacy, v1);
let (v2_request, v2) = open_answered_with(
&shared,
&consumer,
&mut receiver,
with_role_versions(&[("tool-provider", "v2")]),
73,
"a v2 open must not reuse the v1 route",
)
.await;
assert_eq!(
v2_request["role_versions"],
serde_json::json!({ "tool-provider": "v2" })
);
assert!(![legacy, v1].contains(&v2));
let open = |opts: CallOptions| {
let consumer = Arc::clone(&consumer);
async move {
consumer
.open_route(restart_target("plexus"), restart_identity(), opts)
.await
.unwrap()
}
};
assert_eq!(
open(with_role_versions(&[("tool-provider", "v1")])).await,
v1
);
assert_eq!(open(CallOptions::default()).await, legacy);
assert_eq!(
open(CallOptions {
role_versions: Some(BTreeMap::new()),
..CallOptions::default()
})
.await,
legacy,
"an empty map declares nothing and reuses the legacy route"
);
assert!(
receiver.try_recv().is_err(),
"a cache hit must not send a route.open"
);
shared.close_sync("test complete");
}
#[tokio::test]
async fn scoped_routes_are_cached_per_owner_ref_and_epoch_and_apart_from_unscoped() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(8));
let selector = scope_selector("owner", "session-a", 1);
let (request, first) = open_answered(
&shared,
&consumer,
&mut receiver,
Some(selector.clone()),
61,
"first scoped open",
)
.await;
assert_eq!(request["scope"], serde_json::to_value(&selector).unwrap());
let again = consumer
.open_route_scoped(
restart_target("plexus"),
restart_identity(),
selector,
CallOptions::default(),
)
.await
.unwrap();
assert_eq!(again, first);
assert!(
receiver.try_recv().is_err(),
"a cache hit must not send a route.open"
);
let mut handles = vec![first];
for (channel, scope, why) in [
(
62,
Some(scope_selector("owner", "session-a", 2)),
"another epoch of the same scope must open its own route",
),
(
63,
Some(scope_selector("owner", "session-b", 1)),
"another scope ref must open its own route",
),
(
64,
Some(scope_selector("other-owner", "session-a", 1)),
"another owner's scope must open its own route",
),
(65, None, "an unscoped open must not reuse a scoped route"),
] {
let expected_scope = scope
.as_ref()
.map(|scope| serde_json::to_value(scope).unwrap());
let (request, handle) =
open_answered(&shared, &consumer, &mut receiver, scope, channel, why).await;
assert_eq!(
request.get("scope").cloned(),
expected_scope,
"the route.open must carry exactly the requested scope: {why}"
);
assert!(!handles.contains(&handle), "{why}");
handles.push(handle);
}
shared.close_sync("test complete");
}
#[tokio::test]
async fn a_dropped_scoped_route_reopens_with_the_same_selector() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(7));
let selector = scope_selector("owner", "session-a", 3);
let (first_request, first) = open_answered(
&shared,
&consumer,
&mut receiver,
Some(selector.clone()),
51,
"first scoped open",
)
.await;
let goodbye = Frame::build(
FrameType::Goodbye,
Flags::new(false, Priority::Interactive, false),
first.channel,
first.epoch,
0,
Vec::new(),
)
.unwrap();
assert!(dispatch_frame(&shared, 1, goodbye).await);
assert!(matches!(
shared.route_state(first),
Err(CallError::StaleRouteHandle(_))
));
let (second_request, second) = open_answered(
&shared,
&consumer,
&mut receiver,
Some(selector.clone()),
52,
"the reopen after the route dropped",
)
.await;
assert_eq!(
first_request["scope"],
serde_json::to_value(&selector).unwrap()
);
assert_eq!(second_request["scope"], first_request["scope"]);
assert_ne!(second, first);
shared.close_sync("test complete");
}
async fn refuse_scoped_opens(code: &str) -> (usize, CallError) {
refuse_scoped_operation(code, "open").await
}
async fn refuse_scoped_operation(code: &str, operation: &str) -> (usize, CallError) {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(9));
let selector = scope_selector("owner", "session-a", 1);
let options = CallOptions {
scope: Some(selector.clone()),
timeout: Duration::from_secs(2),
route_retry: RetryBackoff {
base: Duration::ZERO,
cap: Duration::ZERO,
max_attempts: 2,
},
route_retry_deadline: Duration::from_millis(150),
..CallOptions::default()
};
let operation = operation.to_string();
let mut task = tokio::spawn(async move {
match operation.as_str() {
"call" => consumer
.call(
restart_target("plexus"),
restart_identity(),
b"{}".to_vec(),
options,
)
.await
.map(|_| ()),
"subscribe" => consumer
.subscribe(
restart_target("plexus"),
restart_identity(),
b"{}".to_vec(),
SubscribeOptions {
scope: options.scope,
route_open_timeout: options.timeout,
route_retry: options.route_retry,
route_retry_deadline: options.route_retry_deadline,
..SubscribeOptions::default()
},
)
.await
.map(|_| ()),
"open" => consumer
.open_route_scoped(
restart_target("plexus"),
restart_identity(),
scope_selector("owner", "session-a", 1),
options,
)
.await
.map(|_| ()),
_ => unreachable!(),
}
});
let mut attempts = 0usize;
let err = loop {
tokio::select! {
command = receiver.recv() => {
let Some(command) = command else {
break task.await.unwrap().expect_err("route.open must be refused");
};
let open: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
assert_eq!(open["scope"], serde_json::to_value(&selector).unwrap());
attempts += 1;
let frame = route_open_answer(
command.frame.header.corr,
0,
Some(ErrorBody::new(code, "test refusal")),
);
assert!(dispatch_frame(&shared, 1, frame).await);
}
joined = &mut task => {
break joined.unwrap().expect_err("route.open must be refused");
}
}
};
shared.close_sync("test complete");
(attempts, err)
}
#[tokio::test]
async fn scoped_managed_operations_retry_only_transient_scope_refusals() {
for operation in ["call", "subscribe"] {
for (code, retryable) in [
(error_codes::SCOPE_NOT_SYNCED, true),
(error_codes::SCOPE_CHANGED, true),
(error_codes::SCOPE_ENDED, false),
(error_codes::SCOPE_NOT_LIVE, false),
(error_codes::SCOPE_EPOCH_REQUIRED, false),
(error_codes::SCOPE_NOT_CARRIER, false),
] {
let (attempts, err) = refuse_scoped_operation(code, operation).await;
if retryable {
assert!(attempts > 1, "{operation}: {code} must be retried");
} else {
assert_eq!(attempts, 1, "{operation}: {code} must be terminal");
}
assert!(matches!(err, CallError::NotSent(_)), "{operation}: {err}");
assert_eq!(err.never_forwarded(), Some(true));
assert_eq!(err.route_open_refusal().unwrap().code, code);
}
}
}
#[tokio::test]
async fn scoped_open_retries_not_synced_and_changed_and_ends_on_the_terminal_scope_codes() {
for code in [error_codes::SCOPE_NOT_SYNCED, error_codes::SCOPE_CHANGED] {
let (attempts, err) = refuse_scoped_opens(code).await;
assert!(attempts > 1, "{code} must be retried, served {attempts}");
assert_eq!(
err.route_open_refusal().map(|body| body.code.as_str()),
Some(code)
);
}
for code in [
error_codes::SCOPE_ENDED,
error_codes::SCOPE_NOT_LIVE,
error_codes::SCOPE_EPOCH_REQUIRED,
error_codes::SCOPE_NOT_CARRIER,
] {
let (attempts, err) = refuse_scoped_opens(code).await;
assert_eq!(attempts, 1, "{code} is terminal and must not be retried");
assert!(matches!(err, CallError::NotSent(_)));
assert_eq!(
err.route_open_refusal().map(|body| body.code.as_str()),
Some(code)
);
}
}
#[tokio::test]
async fn refusal_survives_a_retry_still_in_flight_at_the_deadline() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(11));
let options = CallOptions {
timeout: Duration::from_secs(5),
route_retry: RetryBackoff {
base: Duration::ZERO,
cap: Duration::ZERO,
max_attempts: 2,
},
route_retry_deadline: Duration::from_millis(300),
..CallOptions::default()
};
let mut task = tokio::spawn(async move {
consumer
.open_route(restart_target("warming"), restart_identity(), options)
.await
});
let generation = shared.lock_inner().generation;
let refusal = ErrorBody {
code: error_codes::MODULE_WARMING.to_string(),
message: "warming is not ready".to_string(),
detail: Some(serde_json::json!({"reason": "required_capability_unprovided"})),
};
let mut attempts = 0usize;
let mut held = false;
let err = loop {
tokio::select! {
command = receiver.recv() => {
let Some(command) = command else {
break task.await.unwrap().expect_err("route.open must fail");
};
if command.frame.header.ty != FrameType::Request {
continue;
}
attempts += 1;
if attempts == 1 {
let frame = route_open_answer(
command.frame.header.corr,
0,
Some(refusal.clone()),
);
assert!(dispatch_frame(&shared, generation, frame).await);
} else {
held = shared.mark_pending_accepted(
command.pending.expect("a route.open is a pending request"),
);
}
}
joined = &mut task => {
break joined.unwrap().expect_err("route.open must fail");
}
}
};
shared.close_sync("test complete");
assert_eq!(
attempts, 2,
"one refused attempt, then one held past the deadline"
);
assert!(
held,
"the second attempt must be accepted and left unanswered"
);
assert!(
err.is_not_sent(),
"a refused route never delivered the request: {err}"
);
let body = err
.route_open_refusal()
.unwrap_or_else(|| panic!("the refusal must survive the deadline: {err}"));
assert_eq!(body.code, error_codes::MODULE_WARMING);
assert_eq!(body.detail, refusal.detail);
}
#[test]
fn every_scope_close_reason_must_not_reopen() {
for reason in [
subc_protocol::RouteCloseReason::ScopeEnded,
subc_protocol::RouteCloseReason::ScopeCarrierRemoved,
subc_protocol::RouteCloseReason::ScopeDelegationChanged,
subc_protocol::RouteCloseReason::ScopeParentEnded,
] {
let wire = serde_json::to_value(reason).unwrap();
let wire = wire.as_str().expect("a close reason is a string");
assert!(wire.starts_with("scope_"), "{wire}");
assert_eq!(
RouteCloseReason::from_wire(wire).disposition(),
RouteCloseDisposition::MustNotReopen,
"a route closed for {wire} must not be reopened"
);
}
}
async fn watch_subscribed() -> (
SubcConsumer,
mpsc::Receiver<WriteCommand>,
Subscription,
Frame,
) {
let (shared, mut rx, handle) = reverse_test_shared(ReverseRequestRegistry::new());
shared.lock_inner().one_shot_routes.insert(
handle.channel,
RouteState {
handle,
sem: Arc::new(Semaphore::new(1)),
watch_sem: None,
opted_in: false,
},
);
let consumer = SubcConsumer { shared };
let subscription = consumer
.subscribe_route(&handle, b"watch".to_vec(), SubscribeOptions::default())
.await
.unwrap();
let request = rx.recv().await.unwrap().frame;
(consumer, rx, subscription, request)
}
fn watch_terminal(request: &Frame, ty: FrameType, body: Vec<u8>) -> Frame {
Frame::build(
ty,
Flags::new(false, Priority::Interactive, false),
request.header.channel,
request.header.epoch,
request.header.corr,
body,
)
.unwrap()
}
#[tokio::test]
async fn subscription_ended_preserves_response_body_and_closed_stays_ok() {
let (consumer, _rx, mut subscription, request) = watch_subscribed().await;
let body = vec![0, 255, 128, 1, 10];
assert!(
dispatch_frame(
&consumer.shared,
1,
watch_terminal(&request, FrameType::Response, body.clone())
)
.await
);
subscription.closed().await.unwrap();
assert_eq!(
subscription.ended().await,
SubscriptionEnd::Response(body.clone())
);
subscription.unsubscribe().unwrap();
assert_eq!(subscription.ended().await, SubscriptionEnd::Response(body));
assert_eq!(subscription.events().recv().await, None);
}
#[tokio::test]
async fn subscription_ended_distinguishes_empty_response_from_stream_end() {
let (consumer, _rx, mut subscription, request) = watch_subscribed().await;
assert!(
dispatch_frame(
&consumer.shared,
1,
watch_terminal(&request, FrameType::Response, Vec::new())
)
.await
);
assert_eq!(
subscription.ended().await,
SubscriptionEnd::Response(Vec::new())
);
subscription.closed().await.unwrap();
}
#[tokio::test]
async fn subscription_ended_reports_stream_end_and_closed_stays_ok() {
let (consumer, _rx, mut subscription, request) = watch_subscribed().await;
assert!(
dispatch_frame(
&consumer.shared,
1,
watch_terminal(&request, FrameType::StreamEnd, Vec::new())
)
.await
);
assert_eq!(subscription.ended().await, SubscriptionEnd::StreamEnd);
subscription.closed().await.unwrap();
}
#[tokio::test]
async fn subscription_ended_reports_module_error_code_and_detail() {
let (consumer, _rx, mut subscription, request) = watch_subscribed().await;
let error = ErrorBody {
code: "no_grant".to_string(),
message: "not accepted: no grant".to_string(),
detail: Some(serde_json::json!({"capability": "watch"})),
};
assert!(
dispatch_frame(
&consumer.shared,
1,
watch_terminal(
&request,
FrameType::Error,
serde_json::to_vec(&error).unwrap()
)
)
.await
);
assert_eq!(
subscription.ended().await,
SubscriptionEnd::Error(error.clone())
);
let CallError::Module(closed_error) = subscription.closed().await.unwrap_err() else {
panic!("closed must still return the module error");
};
assert_eq!(closed_error, error);
subscription.unsubscribe().unwrap();
assert_eq!(subscription.ended().await, SubscriptionEnd::Error(error));
}
#[tokio::test]
async fn subscription_ended_reports_unsubscribe_as_cancelled_and_ignores_late_terminal() {
let (consumer, mut rx, mut subscription, request) = watch_subscribed().await;
subscription.unsubscribe().unwrap();
subscription.unsubscribe().unwrap();
subscription.closed().await.unwrap();
assert_eq!(subscription.ended().await, SubscriptionEnd::Cancelled);
let cancel = rx.recv().await.unwrap().frame;
assert_eq!(cancel.header.ty, FrameType::Cancel);
assert_eq!(cancel.header.corr, request.header.corr);
assert!(rx.try_recv().is_err());
assert!(
dispatch_frame(
&consumer.shared,
1,
watch_terminal(&request, FrameType::Response, b"late".to_vec())
)
.await
);
assert_eq!(subscription.ended().await, SubscriptionEnd::Cancelled);
}
#[tokio::test]
async fn subscription_ended_retains_route_close_reason() {
let (consumer, _rx, mut subscription, request) = watch_subscribed().await;
let push = Frame::build(FrameType::Push, Flags::new(false, Priority::Interactive, false), 0, 0, 0,
serde_json::to_vec(&serde_json::json!({"op": "route.closed", "module_id": "watch", "reason": "capability_denied", "channels": [request.header.channel]})).unwrap()).unwrap();
assert!(dispatch_frame(&consumer.shared, 1, push).await);
assert!(
dispatch_frame(
&consumer.shared,
1,
watch_terminal(&request, FrameType::Goodbye, Vec::new())
)
.await
);
let reason = RouteEndReason::Daemon(RouteCloseReason::CapabilityDenied);
assert_eq!(
subscription.ended().await,
SubscriptionEnd::RouteClosed {
reason: reason.clone()
}
);
assert_eq!(
subscription.closed().await.unwrap_err().close_reason(),
Some(&reason)
);
}
#[tokio::test]
async fn subscription_ended_reports_connection_loss() {
let (consumer, _rx, mut subscription, _request) = watch_subscribed().await;
consumer
.shared
.handle_generation_drop(1, "fake connection lost".to_string());
assert_eq!(subscription.ended().await, SubscriptionEnd::ConnectionLost);
assert!(subscription.closed().await.is_err());
}
#[tokio::test]
async fn subscription_ended_waiter_wakes_on_stream_end() {
let (consumer, _rx, mut subscription, request) = watch_subscribed().await;
let (end, dispatched) = tokio::join!(subscription.ended(), async {
tokio::task::yield_now().await;
dispatch_frame(
&consumer.shared,
1,
watch_terminal(&request, FrameType::StreamEnd, Vec::new()),
)
.await
});
assert!(dispatched);
assert_eq!(end, SubscriptionEnd::StreamEnd);
subscription.closed().await.unwrap();
}
#[tokio::test]
async fn subscription_ended_reports_backpressure_without_hiding_closed_error() {
let (consumer, _rx, mut subscription, request) = watch_subscribed().await;
for _ in 0..=DEFAULT_SUBSCRIPTION_EVENT_BUFFER {
assert!(
dispatch_frame(
&consumer.shared,
1,
watch_terminal(&request, FrameType::StreamData, vec![1])
)
.await
);
}
assert_eq!(subscription.ended().await, SubscriptionEnd::Backpressure);
assert!(matches!(
subscription.closed().await,
Err(CallError::SubscriptionBackpressure(_))
));
}
#[tokio::test]
async fn subscription_ended_reports_client_close_as_route_closed_by_caller() {
let (consumer, _rx, mut subscription, _request) = watch_subscribed().await;
consumer.close().await;
assert_eq!(
subscription.ended().await,
SubscriptionEnd::RouteClosed {
reason: RouteEndReason::ClosedByCaller
}
);
assert!(subscription.closed().await.is_err());
}
#[test]
fn drain_pending_channel_can_skip_subscriptions() {
let mut pending = HashMap::new();
let generation = 7;
let channel = 11;
let handle = RouteHandle::new(channel, 3, generation);
let unary_key = PendingKey {
generation,
channel,
epoch: handle.epoch,
corr: 1,
};
let subscription_key = PendingKey {
generation,
channel,
epoch: handle.epoch,
corr: 2,
};
let (unary_tx, _unary_rx) = oneshot::channel();
pending.insert(unary_key, PendingEntry::unary(unary_tx, false, None, None));
let (events_tx, _events_rx) = mpsc::channel(1);
let (closed_tx, _closed_rx) = oneshot::channel();
let (ended_tx, _ended_rx) = watch::channel(None);
let permit = Arc::new(Semaphore::new(1))
.try_acquire_owned()
.expect("test semaphore permit should be available");
pending.insert(
subscription_key,
PendingEntry::subscription(
events_tx,
closed_tx,
ended_tx,
RoutePermit::legacy(permit),
Priority::Interactive,
),
);
let drained = drain_pending_handle(&mut pending, handle, false);
assert_eq!(drained.len(), 1);
assert!(pending.contains_key(&subscription_key));
let drained = drain_pending_handle(&mut pending, handle, true);
assert_eq!(drained.len(), 1);
assert!(pending.is_empty());
}
fn response_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
Frame::build(
FrameType::Response,
Flags::new(false, Priority::Interactive, false),
channel,
epoch,
corr,
body,
)
.unwrap()
}
#[tokio::test]
async fn stale_epoch_ingress_drops_without_settling_matching_corr() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, _rx) = mpsc::channel(4);
let current = RouteHandle::new(9, 2, 1);
let stale_key = PendingKey {
generation: 1,
channel: 9,
epoch: 1,
corr: 77,
};
let key = PendingKey {
generation: 1,
channel: 9,
epoch: 2,
corr: 77,
};
let (stale_tx, mut stale_response) = oneshot::channel();
let (tx, mut response) = oneshot::channel();
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.route_epochs.insert(9, current);
inner
.pending
.insert(stale_key, PendingEntry::unary(stale_tx, false, None, None));
inner
.pending
.insert(key, PendingEntry::unary(tx, false, None, None));
}
assert!(dispatch_frame(&shared, 1, response_frame(9, 1, 77, b"stale".to_vec())).await);
assert!(matches!(
response.try_recv(),
Err(oneshot::error::TryRecvError::Empty)
));
assert!(matches!(
stale_response.try_recv(),
Err(oneshot::error::TryRecvError::Empty)
));
assert!(shared.lock_inner().pending.contains_key(&stale_key));
assert!(shared.lock_inner().pending.contains_key(&key));
assert_eq!(shared.lock_inner().dropped_route_frames, 1);
assert!(dispatch_frame(&shared, 1, response_frame(9, 2, 77, b"current".to_vec())).await);
let PendingResult::Terminal(PendingTerminal::Response { body, .. }) =
response.await.unwrap()
else {
panic!("current epoch must settle its own pending request");
};
assert_eq!(body, b"current");
assert!(shared.lock_inner().pending.contains_key(&stale_key));
}
#[tokio::test]
async fn route_poll_response_must_echo_expected_handle_before_settling() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, _rx) = mpsc::channel(4);
let handle = RouteHandle::new(3, 9, 1);
let key = PendingKey {
generation: 1,
channel: 0,
epoch: 0,
corr: 88,
};
let (tx, mut response) = oneshot::channel();
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.route_epochs.insert(handle.channel, handle);
inner
.pending
.insert(key, PendingEntry::unary(tx, false, Some(handle), None));
}
let wrong = serde_json::to_vec(&ClientControlResponse::RoutePoll {
route_channel: handle.channel,
route_epoch: handle.epoch + 1,
status: Some("wrong".to_string()),
live: Some(true),
})
.unwrap();
assert!(dispatch_frame(&shared, 1, response_frame(0, 0, key.corr, wrong)).await);
assert!(matches!(
response.try_recv(),
Err(oneshot::error::TryRecvError::Empty)
));
assert!(shared.lock_inner().pending.contains_key(&key));
let correct = serde_json::to_vec(&ClientControlResponse::RoutePoll {
route_channel: handle.channel,
route_epoch: handle.epoch,
status: Some("ready".to_string()),
live: Some(true),
})
.unwrap();
assert!(dispatch_frame(&shared, 1, response_frame(0, 0, key.corr, correct)).await);
assert!(matches!(
response.await.unwrap(),
PendingResult::Terminal(PendingTerminal::Response { .. })
));
}
#[tokio::test]
async fn stale_connection_handle_emits_no_request_cancel_or_goodbye() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(4);
let stale = RouteHandle::new(4, 1, 1);
let current = RouteHandle::new(4, 1, 2);
{
let mut inner = shared.lock_inner();
inner.generation = 2;
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.route_epochs.insert(4, current);
}
let err = shared
.send_request(RequestSend {
binary: false,
expected_handle: Some(stale),
channel: stale.channel,
epoch: stale.epoch,
body: b"request".to_vec(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline: Instant::now() + Duration::from_millis(10),
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await
.unwrap_err();
assert!(matches!(err, CallError::StaleRouteHandle(handle) if handle == stale));
shared.send_cancel(stale, 8, Priority::Interactive);
assert!(!shared.send_route_goodbye(stale, false));
assert!(
rx.try_recv().is_err(),
"stale operations must not queue frames"
);
}
#[tokio::test]
async fn route_poll_refusal_preserves_daemon_code_message_and_detail() {
let (shared, consumer, mut receiver) = route_open_stand_in(Box::new(|| 0.5));
let handle = RouteHandle::new(3, 9, 1);
shared
.lock_inner()
.route_epochs
.insert(handle.channel, handle);
let poll = consumer.poll_route(&handle, PollKind::Liveness, Duration::from_secs(1));
tokio::pin!(poll);
assert_pending(poll.as_mut()).await;
let request = receiver.try_recv().unwrap();
let refusal = ErrorBody {
code: "unknown_route".into(),
message: "the route is no longer present".into(),
detail: Some(serde_json::json!({ "route_channel": 3, "route_epoch": 9 })),
};
assert!(
dispatch_frame(
&shared,
1,
channel_zero_frame(
FrameType::Error,
request.frame.header.corr,
serde_json::to_vec(&refusal).unwrap(),
),
)
.await
);
let err = poll.await.unwrap_err();
assert_eq!(err.code(), Some("unknown_route"));
let CallError::Module(body) = err else {
panic!("a delivered refusal must not be classified as a send failure");
};
assert_eq!(body.code, refusal.code);
assert_eq!(body.message, refusal.message);
assert_eq!(body.detail, refusal.detail);
}
#[tokio::test]
async fn late_route_open_queues_goodbye_and_full_queue_closes_connection() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(2);
let key = PendingKey {
generation: 1,
channel: 0,
epoch: 0,
corr: 41,
};
let (tx, response) = oneshot::channel();
drop(response);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner
.pending
.insert(key, PendingEntry::unary(tx, true, None, None));
}
let body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
request_window: None,
watch_window: None,
route_channel: 12,
route_epoch: 7,
})
.unwrap();
assert!(dispatch_frame(&shared, 1, response_frame(0, 0, 41, body)).await);
let cleanup = rx.recv().await.unwrap().frame;
assert_eq!(cleanup.header.ty, FrameType::Goodbye);
assert_eq!((cleanup.header.channel, cleanup.header.epoch), (12, 7));
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions {
reconnect_backoff: RetryBackoff {
max_attempts: 1,
..RetryBackoff::default()
},
..ConsumerOptions::default()
},
));
let (writer, _rx) = mpsc::channel(1);
let filler = response_frame(0, 0, 1, Vec::new());
writer
.try_send(WriteCommand {
frame: filler,
pending: None,
})
.unwrap();
let key = PendingKey {
generation: 1,
channel: 0,
epoch: 0,
corr: 42,
};
let (tx, response) = oneshot::channel();
drop(response);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner
.pending
.insert(key, PendingEntry::unary(tx, true, None, None));
}
let body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
request_window: None,
watch_window: None,
route_channel: 13,
route_epoch: 8,
})
.unwrap();
assert!(dispatch_frame(&shared, 1, response_frame(0, 0, 42, body)).await);
assert!(shared.lock_inner().writer.is_none());
}
#[tokio::test]
async fn correlation_allocator_emits_max_once_then_closes_without_reuse() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions {
reconnect_backoff: RetryBackoff {
max_attempts: 1,
..RetryBackoff::default()
},
..ConsumerOptions::default()
},
));
let (writer, mut rx) = mpsc::channel(4);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.next_corr = Some(u64::MAX);
}
let request_shared = Arc::clone(&shared);
let request = tokio::spawn(async move {
request_shared
.send_request(RequestSend {
binary: false,
expected_handle: None,
channel: 0,
epoch: 0,
body: Vec::new(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline: Instant::now() + Duration::from_secs(1),
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await
});
let command = rx.recv().await.unwrap();
assert_eq!(command.frame.header.corr, u64::MAX);
assert!(dispatch_frame(&shared, 1, response_frame(0, 0, u64::MAX, Vec::new()),).await);
assert!(request.await.unwrap().is_ok());
let exhausted = shared
.send_request(RequestSend {
binary: false,
expected_handle: None,
channel: 0,
epoch: 0,
body: Vec::new(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline: Instant::now() + Duration::from_millis(10),
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await
.unwrap_err();
assert!(matches!(exhausted, CallError::NotSent(_)));
assert!(rx.try_recv().is_err());
assert!(shared.lock_inner().writer.is_none());
}
#[tokio::test]
async fn managed_call_deadline_bounds_flow_control_wait() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(4);
let target = RouteTarget::ToolProvider {
module_id: "flow-controlled".to_string(),
};
let identity = BindIdentity::new("test".to_string(), "deadline".to_string());
let consumer_identity = Some(ConsumerIdentity {
module_id: "caller".to_string(),
launch_nonce: "nonce".to_string(),
});
let first_opts = CallOptions {
timeout: Duration::from_secs(1),
consumer_identity: consumer_identity.clone(),
..CallOptions::default()
};
let second_opts = CallOptions {
timeout: Duration::from_millis(25),
consumer_identity,
..CallOptions::default()
};
let key = RouteKey::new(
&target,
&identity,
first_opts.consumer_identity.as_ref(),
None,
);
let handle = RouteHandle::new(5, 3, 1);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.cache_route(
key,
RouteState {
handle,
sem: Arc::new(Semaphore::new(1)),
watch_sem: None,
opted_in: false,
},
);
}
let first = tokio::spawn({
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let target = target.clone();
let identity = identity.clone();
async move {
consumer
.call(target, identity, b"first".to_vec(), first_opts)
.await
}
});
let first_frame = rx
.recv()
.await
.expect("the first request should enter the fake daemon queue");
assert_eq!(first_frame.frame.header.ty, FrameType::Request);
assert_eq!(first_frame.frame.body, b"first");
assert!(shared.mark_pending_accepted(
first_frame
.pending
.expect("request commands retain their pending key"),
));
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let result = tokio::time::timeout(
Duration::from_millis(250),
consumer.call(target, identity, b"second".to_vec(), second_opts),
)
.await
.expect("a flow-controlled call must finish at its own deadline")
.unwrap_err();
assert!(matches!(result, CallError::NotSent(_)));
assert!(
rx.try_recv().is_err(),
"the timed-out second request must not reach the fake daemon"
);
first.abort();
let _ = first.await;
}
#[tokio::test]
async fn admitted_route_open_emits_one_frame_without_retrying_daemon_errors() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions {
call_timeout: Duration::from_secs(1),
..ConsumerOptions::default()
},
));
let (writer, mut rx) = mpsc::channel(4);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
}
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let target = RouteTarget::ToolProvider {
module_id: "admitted-target".to_string(),
};
let identity = BindIdentity::new("test".to_string(), "admitted".to_string());
let task = tokio::spawn(async move {
consumer
.open_route_with_admission_facts(
target,
identity,
serde_json::json!({"schema": 1, "verified_class": "member"}),
)
.await
});
let command = rx.recv().await.expect("one route.open must be queued");
let request: ClientControlRequest = serde_json::from_slice(&command.frame.body).unwrap();
let ClientControlRequest::RouteOpen {
admission_facts, ..
} = request
else {
panic!("expected route.open")
};
assert_eq!(
admission_facts,
Some(serde_json::json!({"schema": 1, "verified_class": "member"}))
);
let error_body = serde_json::to_vec(&ErrorBody {
code: "admission_facts_not_permitted".to_string(),
message: "not permitted".to_string(),
detail: None,
})
.unwrap();
assert!(
dispatch_frame(
&shared,
1,
Frame::build(
FrameType::Error,
Flags::new(false, Priority::Interactive, false),
0,
0,
command.frame.header.corr,
error_body,
)
.unwrap(),
)
.await
);
let result = task.await.unwrap();
assert!(matches!(result, Err(CallError::NotSent(_))));
assert!(rx.try_recv().is_err(), "one-shot route.open must not retry");
}
async fn open_admitted_route_for_test(
opts: Option<CallOptions>,
) -> (
Arc<Shared>,
SubcConsumer,
ClientControlRequest,
Result<RouteHandle, CallError>,
mpsc::Receiver<WriteCommand>,
) {
let shared = writer_test_shared();
let (writer, mut rx) = mpsc::channel(8);
let generation = {
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.generation
};
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let target = RouteTarget::ToolProvider {
module_id: "cerebellum".to_string(),
};
let identity = BindIdentity::new("test".to_string(), "admitted".to_string());
let facts = serde_json::json!({"schema": 1, "verified_class": "member"});
let task = tokio::spawn(async move {
let result = match opts {
Some(opts) => {
consumer
.open_route_with_admission_facts_and_options(target, identity, facts, opts)
.await
}
None => {
consumer
.open_route_with_admission_facts(target, identity, facts)
.await
}
};
(consumer, result)
});
let command = rx.recv().await.expect("one route.open must be queued");
let request: ClientControlRequest = serde_json::from_slice(&command.frame.body).unwrap();
let response = serde_json::to_vec(&ClientControlResponse::RouteOpen {
request_window: None,
watch_window: None,
route_channel: 21,
route_epoch: 4,
})
.unwrap();
assert!(
dispatch_frame(
&shared,
generation,
Frame::build(
FrameType::Response,
Flags::new(false, Priority::Interactive, false),
0,
0,
command.frame.header.corr,
response,
)
.unwrap(),
)
.await
);
let (consumer, result) = tokio::time::timeout(Duration::from_secs(1), task)
.await
.expect("admitted route.open resolves")
.unwrap();
(shared, consumer, request, result, rx)
}
#[tokio::test]
async fn admitted_route_open_refusal_keeps_the_daemon_code_and_detail() {
let shared = writer_test_shared();
let (writer, mut rx) = mpsc::channel(8);
let generation = {
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.generation
};
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let task = tokio::spawn(async move {
let result = consumer
.open_route_with_admission_facts(
RouteTarget::ToolProvider {
module_id: "cerebellum".to_string(),
},
BindIdentity::new("test".to_string(), "admitted".to_string()),
serde_json::json!({"schema": 1, "verified_class": "member"}),
)
.await;
(consumer, result)
});
let command = rx.recv().await.expect("one route.open must be queued");
let refusal = ErrorBody {
code: "module_warming".to_string(),
message: "cerebellum is not ready".to_string(),
detail: Some(serde_json::json!({"reason": "declared_not_ready"})),
};
assert!(
dispatch_frame(
&shared,
generation,
Frame::build(
FrameType::Error,
Flags::new(false, Priority::Interactive, false),
0,
0,
command.frame.header.corr,
serde_json::to_vec(&refusal).unwrap(),
)
.unwrap(),
)
.await
);
let (_consumer, result) = tokio::time::timeout(Duration::from_secs(1), task)
.await
.expect("admitted route.open resolves")
.unwrap();
let error = result.expect_err("a refused open is an error");
assert!(
error.is_not_sent(),
"a refused open never reached the module: {error}"
);
let body = error
.route_open_refusal()
.unwrap_or_else(|| panic!("the refusal's code must survive: {error}"));
assert_eq!(body.code, "module_warming");
assert_eq!(
body.detail,
Some(serde_json::json!({"reason": "declared_not_ready"}))
);
assert!(
error.to_string().contains("tool_provider:cerebellum"),
"the error names its target: {error}"
);
}
#[tokio::test]
async fn admitted_route_without_options_opens_and_declares_no_reverse_capabilities() {
let (_shared, _consumer, request, result, _rx) = open_admitted_route_for_test(None).await;
let ClientControlRequest::RouteOpen {
consumer_capabilities,
admission_facts,
..
} = request
else {
panic!("expected route.open")
};
assert_eq!(consumer_capabilities, None);
assert!(admission_facts.is_some());
let handle = result.expect("admitted route opens");
assert_eq!((handle.channel, handle.epoch), (21, 4));
}
#[tokio::test]
async fn admitted_route_with_options_delivers_elicitation_to_its_handler() {
let opts = CallOptions::default();
let seen = Arc::new(Mutex::new(None));
let seen_by_handler = Arc::clone(&seen);
let reply_body = br#"{"jsonrpc":"2.0","id":3,"result":{"action":"accept"}}"#.to_vec();
let expected_reply = reply_body.clone();
opts.reverse_requests
.on_request("elicitation", move |body, ctx| {
*seen_by_handler
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()) = Some((body, ctx.corr));
let reply_body = reply_body.clone();
async move { reply_body }
})
.unwrap();
let (shared, _consumer, request, result, mut rx) =
open_admitted_route_for_test(Some(opts)).await;
let ClientControlRequest::RouteOpen {
consumer_capabilities,
admission_facts,
..
} = request
else {
panic!("expected route.open")
};
assert_eq!(consumer_capabilities, Some(vec!["elicitation".to_string()]));
assert_eq!(
admission_facts,
Some(serde_json::json!({"schema": 1, "verified_class": "member"}))
);
let handle = result.expect("admitted route opens");
let request_body =
br#"{"jsonrpc":"2.0","id":3,"method":"elicitation/create","params":{}}"#.to_vec();
assert!(
dispatch_frame(
&shared,
handle.connection_token(),
reverse_request_frame(handle, 900, request_body.clone()),
)
.await
);
let reply = tokio::time::timeout(Duration::from_secs(1), rx.recv())
.await
.expect("the handler replies")
.unwrap()
.frame;
assert_eq!(reply.header.ty, FrameType::Response);
assert_eq!(reply.header.corr, 900);
assert_eq!(reply.header.channel, 21);
assert_eq!(reply.body, expected_reply);
assert_eq!(
seen.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone(),
Some((request_body, 900))
);
}
#[tokio::test]
async fn route_open_waiter_deadline_is_not_sent_without_writing() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(4);
let target = RouteTarget::ToolProvider {
module_id: "single-flight".to_string(),
};
let identity = BindIdentity::new("test".to_string(), "route-open".to_string());
let opts = CallOptions {
timeout: Duration::from_millis(25),
consumer_identity: Some(ConsumerIdentity {
module_id: "caller".to_string(),
launch_nonce: "nonce".to_string(),
}),
..CallOptions::default()
};
let key = RouteKey::new(&target, &identity, opts.consumer_identity.as_ref(), None);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.openings.insert(
key,
Opening {
waiters: Vec::new(),
closed: false,
},
);
}
let consumer = SubcConsumer { shared };
let result = tokio::time::timeout(
Duration::from_millis(250),
consumer.open_route(target, identity, opts),
)
.await
.expect("a route.open waiter must finish at its own deadline")
.unwrap_err();
assert!(matches!(result, CallError::NotSent(_)));
assert!(
rx.try_recv().is_err(),
"a timed-out route.open waiter must not write a control frame"
);
}
async fn managed_op_with_unanswered_route_open(subscribe: bool) -> CallError {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(4);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
}
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let target = RouteTarget::ToolProvider {
module_id: "slow-route-open".to_string(),
};
let identity = BindIdentity::new("test".to_string(), "slow-route-open".to_string());
let task = tokio::spawn(async move {
if subscribe {
consumer
.subscribe(
target,
identity,
b"{}".to_vec(),
SubscribeOptions {
route_open_timeout: Duration::from_millis(200),
route_retry_deadline: Duration::from_millis(100),
..SubscribeOptions::default()
},
)
.await
.map(|_| ())
} else {
consumer
.call(
target,
identity,
b"{}".to_vec(),
CallOptions {
timeout: Duration::from_millis(200),
route_retry_deadline: Duration::from_millis(100),
..CallOptions::default()
},
)
.await
.map(|_| ())
}
});
let command = rx.recv().await.expect("route.open must be queued");
assert_eq!(command.frame.header.channel, 0);
let request: ClientControlRequest = serde_json::from_slice(&command.frame.body).unwrap();
assert!(matches!(request, ClientControlRequest::RouteOpen { .. }));
assert!(shared.mark_pending_accepted(command.pending.expect("route.open is tracked")));
let err = tokio::time::timeout(Duration::from_secs(2), task)
.await
.expect("the managed op must settle at its own deadline")
.unwrap()
.expect_err("an unanswered route.open cannot yield a route");
while let Ok(command) = rx.try_recv() {
assert_eq!(
command.frame.header.channel, 0,
"no data-plane frame may be written without an open route"
);
}
err
}
#[tokio::test]
async fn call_whose_route_open_times_out_after_send_is_not_sent() {
let err = managed_op_with_unanswered_route_open(false).await;
assert!(
matches!(err, CallError::NotSent(_)),
"call body never left the client, got {err:?}"
);
}
#[tokio::test]
async fn subscribe_whose_route_open_times_out_after_send_is_not_sent() {
let err = managed_op_with_unanswered_route_open(true).await;
assert!(
matches!(err, CallError::NotSent(_)),
"subscription request never left the client, got {err:?}"
);
}
#[tokio::test]
async fn route_poll_deadline_bounds_writer_capacity() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(1);
let handle = RouteHandle::new(8, 4, 1);
writer
.try_send(WriteCommand {
frame: response_frame(0, 0, 99, Vec::new()),
pending: None,
})
.expect("the fake daemon queue should accept its filler frame");
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.route_epochs.insert(handle.channel, handle);
}
let consumer = SubcConsumer { shared };
let result = tokio::time::timeout(
Duration::from_millis(250),
consumer.poll_route(&handle, PollKind::Liveness, Duration::from_millis(25)),
)
.await
.expect("a control request must finish at its own deadline")
.unwrap_err();
assert!(matches!(result, CallError::NotSent(_)));
assert_eq!(
rx.recv()
.await
.expect("the filler must still be the only queued frame")
.frame
.header
.corr,
99
);
assert!(
rx.try_recv().is_err(),
"the timed-out control request must not reach the fake daemon"
);
}
#[tokio::test]
async fn subscription_deadline_bounds_writer_capacity() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(1);
let handle = RouteHandle::new(9, 2, 1);
let route_sem = Arc::new(Semaphore::new(1));
writer
.try_send(WriteCommand {
frame: response_frame(0, 0, 100, Vec::new()),
pending: None,
})
.expect("the fake daemon queue should accept its filler frame");
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.daemon_ver = "0.39.0".into();
inner.cache_route(
RouteKey::new(
&RouteTarget::ToolProvider {
module_id: "subscriptions".to_string(),
},
&BindIdentity::new("test".to_string(), "subscription".to_string()),
None,
None,
),
RouteState {
handle,
sem: Arc::clone(&route_sem),
watch_sem: None,
opted_in: false,
},
);
}
let consumer = SubcConsumer { shared };
let result = tokio::time::timeout(
Duration::from_millis(250),
consumer.subscribe_route(
&handle,
b"subscribe".to_vec(),
SubscribeOptions {
route_open_timeout: Duration::from_millis(25),
..SubscribeOptions::default()
},
),
)
.await
.expect("a subscription must finish at its route-open deadline");
let result = match result {
Ok(_) => panic!("a subscription blocked before writing must time out"),
Err(err) => err,
};
assert!(matches!(result, CallError::NotSent(_)));
assert_eq!(
rx.recv()
.await
.expect("the filler must still be the only queued frame")
.frame
.header
.corr,
100
);
assert!(
rx.try_recv().is_err(),
"the timed-out subscription must not reach the fake daemon"
);
assert!(
route_sem.try_acquire().is_ok(),
"a pre-write subscription timeout must release its route credit"
);
}
#[test]
fn catalog_list_deserializes_golden_reply_and_ignores_unknown_fields() {
let mut reply: serde_json::Value = serde_json::from_str(include_str!(
"../../subc-control/tests/golden/client_control_response_catalog_list.json"
))
.expect("the catalog.list golden reply must be valid JSON");
reply["future_top_level"] = serde_json::json!(true);
reply["modules"][0]["future_module_field"] = serde_json::json!("ignored");
let catalog: CatalogList =
serde_json::from_value(reply).expect("catalog.list should tolerate additive fields");
assert_eq!(catalog.generation, 7);
assert_eq!(catalog.modules.len(), 1);
assert!(catalog.subc_ops.iter().any(|op| op == "catalog.list"));
let tools = catalog.modules[0]
.roles
.iter()
.find_map(|role| match role {
subc_protocol::manifest::ProviderRole::ToolProvider { tools, .. } => Some(tools),
_ => None,
})
.expect("the golden module must advertise a tool_provider role");
let tool = tools
.first()
.expect("the golden tool_provider role must advertise a tool");
assert!(!tool.name.is_empty());
assert_eq!(
tool.schema.get("type").and_then(serde_json::Value::as_str),
Some("object")
);
assert!(matches!(
tool.execution_mode,
subc_protocol::manifest::ExecutionMode::Pure
));
}
#[tokio::test]
async fn catalog_list_sends_an_unfiltered_channel_zero_request() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(1);
shared.lock_inner().writer = Some(writer);
shared.lock_inner().daemon_ver = "0.39.0".into();
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let request = tokio::spawn(async move { consumer.catalog_list().await });
let command = rx
.recv()
.await
.expect("catalog.list must queue a channel-0 request");
assert_eq!(command.frame.header.channel, 0);
let body: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
assert_eq!(body["op"], "catalog.list");
assert!(
body.get("module_id").is_none(),
"catalog.list must request the complete catalog without a module filter"
);
let response = serde_json::to_vec(&ClientControlResponse::CatalogList {
generation: 9,
modules: Vec::new(),
subc_ops: vec!["catalog.list".to_string()],
})
.unwrap();
assert!(
dispatch_frame(
&shared,
1,
response_frame(0, 0, command.frame.header.corr, response),
)
.await
);
let catalog = request.await.unwrap().unwrap();
assert_eq!(catalog.generation, 9);
assert!(catalog.modules.is_empty());
}
#[tokio::test]
async fn spawn_snapshot_sends_a_channel_zero_request_and_returns_the_snapshot() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(1);
shared.lock_inner().writer = Some(writer);
shared.lock_inner().daemon_ver = "0.39.0".into();
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let request = tokio::spawn(async move { consumer.spawn_snapshot().await });
let command = rx
.recv()
.await
.expect("supervisor.spawn_snapshot must queue a channel-0 request");
assert_eq!(command.frame.header.channel, 0);
let body: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
assert_eq!(body["op"], "supervisor.spawn_snapshot");
let snapshot = SpawnSnapshot {
cursor: SpawnCursor {
daemon_incarnation: "incarnation-a".to_string(),
seq: 41,
},
ring_bound: 256,
live: vec![LiveSpawn {
module_id: "participant".to_string(),
spawn_generation: 3,
pid: 4242,
spawned_at_ms: 1_700_000_000_000,
}],
};
let response = serde_json::to_vec(&ClientControlResponse::SupervisorSpawnSnapshot {
snapshot: snapshot.clone(),
})
.unwrap();
assert!(
dispatch_frame(
&shared,
1,
response_frame(0, 0, command.frame.header.corr, response),
)
.await
);
assert_eq!(request.await.unwrap().unwrap(), snapshot);
}
#[tokio::test]
async fn spawn_snapshot_refusal_keeps_the_daemon_code() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(1);
shared.lock_inner().writer = Some(writer);
shared.lock_inner().daemon_ver = "0.39.0".into();
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let request = tokio::spawn(async move { consumer.spawn_snapshot().await });
let command = rx
.recv()
.await
.expect("supervisor.spawn_snapshot must queue a channel-0 request");
let refusal = serde_json::to_vec(&ErrorBody {
code: "op_not_permitted".to_string(),
message: "refused by the test".to_string(),
detail: None,
})
.unwrap();
let frame = Frame::build(
FrameType::Error,
Flags::new(false, Priority::Interactive, false),
0,
0,
command.frame.header.corr,
refusal,
)
.unwrap();
assert!(dispatch_frame(&shared, 1, frame).await);
let error = request.await.unwrap().unwrap_err();
assert_eq!(error.code(), Some("op_not_permitted"));
}
fn spawn_cursor(seq: u64) -> SpawnCursor {
SpawnCursor {
daemon_incarnation: "incarnation-a".to_string(),
seq,
}
}
fn spawn_event(seq: u64, kind: SpawnEventKind, generation: u64) -> SpawnEvent {
SpawnEvent {
cursor: spawn_cursor(seq),
kind,
module_id: "participant".to_string(),
spawn_generation: generation,
pid: 4242,
exit_code: (kind == SpawnEventKind::Exited).then_some(0),
exit_signal: None,
}
}
fn channel_zero_frame(ty: FrameType, corr: u64, body: Vec<u8>) -> Frame {
Frame::build(
ty,
Flags::new(false, Priority::Interactive, false),
0,
0,
corr,
body,
)
.unwrap()
}
fn spawn_error_frame(corr: u64, code: &str, detail: serde_json::Value) -> Frame {
channel_zero_frame(
FrameType::Error,
corr,
serde_json::to_vec(&ErrorBody {
code: code.to_string(),
message: "sent by the test".to_string(),
detail: Some(detail),
})
.unwrap(),
)
}
async fn spawn_subscribed(
since: Option<SpawnCursor>,
) -> (
Arc<Shared>,
mpsc::Receiver<WriteCommand>,
SpawnSubscription,
Frame,
) {
let shared = writer_test_shared();
let (writer, mut rx) = mpsc::channel(8);
shared.lock_inner().writer = Some(writer);
shared.lock_inner().daemon_ver = "0.39.0".into();
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let subscription = consumer
.spawn_subscribe(since)
.await
.expect("supervisor.spawn_subscribe must be sent");
std::mem::forget(consumer);
let command = rx
.recv()
.await
.expect("supervisor.spawn_subscribe must queue a channel-0 request");
(shared, rx, subscription, command.frame)
}
#[tokio::test]
async fn spawn_subscribe_sends_its_cursor_on_channel_zero_and_yields_events_until_stream_end() {
let (shared, _rx, mut subscription, request) =
spawn_subscribed(Some(spawn_cursor(7))).await;
assert_eq!(request.header.channel, 0);
assert_eq!(request.header.ty, FrameType::Request);
let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap();
assert_eq!(
body,
serde_json::json!({
"op": "supervisor.spawn_subscribe",
"since": {"daemon_incarnation": "incarnation-a", "seq": 7},
})
);
let corr = request.header.corr;
let events = [
spawn_event(8, SpawnEventKind::Exited, 3),
spawn_event(9, SpawnEventKind::Spawned, 4),
];
for event in &events {
let body = serde_json::to_vec(event).unwrap();
assert!(
dispatch_frame(
&shared,
1,
channel_zero_frame(FrameType::StreamData, corr, body)
)
.await
);
}
assert!(
dispatch_frame(
&shared,
1,
channel_zero_frame(FrameType::StreamEnd, corr, Vec::new())
)
.await
);
assert_eq!(subscription.next().await.unwrap(), Some(events[0].clone()));
assert_eq!(subscription.next().await.unwrap(), Some(events[1].clone()));
assert_eq!(subscription.next().await.unwrap(), None);
assert_eq!(
subscription.next().await.unwrap(),
None,
"a finished stream stays finished"
);
}
#[tokio::test]
async fn spawn_subscribe_without_a_cursor_omits_since() {
let (_shared, _rx, _subscription, request) = spawn_subscribed(None).await;
let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap();
assert_eq!(
body,
serde_json::json!({"op": "supervisor.spawn_subscribe"})
);
}
#[tokio::test]
async fn spawn_cursor_refusals_surface_as_coded_errors_with_their_detail() {
let (shared, _rx, mut subscription, request) =
spawn_subscribed(Some(spawn_cursor(7))).await;
let frame = spawn_error_frame(
request.header.corr,
"spawn_cursor_incarnation_mismatch",
serde_json::json!({"current_daemon_incarnation": "incarnation-b"}),
);
assert!(dispatch_frame(&shared, 1, frame).await);
let error = subscription.next().await.unwrap_err();
assert_eq!(error.code(), Some("spawn_cursor_incarnation_mismatch"));
let SpawnStreamError::CursorIncarnationMismatch {
current_daemon_incarnation,
body,
} = &error
else {
panic!("expected the incarnation refusal, got {error:?}");
};
assert_eq!(current_daemon_incarnation, "incarnation-b");
assert_eq!(body.code, "spawn_cursor_incarnation_mismatch");
assert_eq!(subscription.next().await.unwrap(), None);
let (shared, _rx, mut subscription, request) =
spawn_subscribed(Some(spawn_cursor(1))).await;
let frame = spawn_error_frame(
request.header.corr,
"spawn_cursor_too_old",
serde_json::json!({"oldest_retained_cursor": spawn_cursor(90)}),
);
assert!(dispatch_frame(&shared, 1, frame).await);
let error = subscription.next().await.unwrap_err();
assert_eq!(error.code(), Some("spawn_cursor_too_old"));
let SpawnStreamError::CursorTooOld {
oldest_retained_cursor,
..
} = &error
else {
panic!("expected the too-old refusal, got {error:?}");
};
assert_eq!(oldest_retained_cursor, &spawn_cursor(90));
}
#[tokio::test]
async fn a_lagged_spawn_stream_yields_its_queued_events_then_the_lag_error() {
let (shared, _rx, mut subscription, request) =
spawn_subscribed(Some(spawn_cursor(7))).await;
let corr = request.header.corr;
let queued = spawn_event(8, SpawnEventKind::Exited, 3);
assert!(
dispatch_frame(
&shared,
1,
channel_zero_frame(
FrameType::StreamData,
corr,
serde_json::to_vec(&queued).unwrap()
),
)
.await
);
let frame = spawn_error_frame(
corr,
"spawn_subscriber_lagged",
serde_json::json!({"first_undelivered_cursor": spawn_cursor(9)}),
);
assert!(dispatch_frame(&shared, 1, frame).await);
assert_eq!(subscription.next().await.unwrap(), Some(queued));
let error = subscription.next().await.unwrap_err();
assert_eq!(error.code(), Some("spawn_subscriber_lagged"));
let SpawnStreamError::SubscriberLagged {
first_undelivered_cursor,
..
} = &error
else {
panic!("expected the lag error, got {error:?}");
};
assert_eq!(first_undelivered_cursor, &spawn_cursor(9));
}
#[tokio::test]
async fn dropping_a_spawn_subscription_sends_a_channel_zero_cancel_for_its_corr() {
let (_shared, mut rx, subscription, request) =
spawn_subscribed(Some(spawn_cursor(7))).await;
drop(subscription);
let cancel = tokio::time::timeout(Duration::from_secs(1), rx.recv())
.await
.expect("dropping the subscription must queue a Cancel")
.expect("the writer stays open");
assert_eq!(cancel.frame.header.ty, FrameType::Cancel);
assert_eq!(cancel.frame.header.channel, 0);
assert_eq!(cancel.frame.header.corr, request.header.corr);
}
#[tokio::test]
async fn reconnect_completion_between_wait_decision_and_registration_wakes_caller() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/subc-client-rs-reconnect-wakeup"),
ConsumerOptions::default(),
));
let (sender, _receiver) = mpsc::channel(EGRESS_BUFFER);
{
let mut inner = shared.lock_inner();
inner.reconnect = ReconnectState::Inline {
generation: inner.generation,
};
}
let completing = Arc::clone(&shared);
*shared.before_wait.lock().unwrap() = Some(Box::new(move || {
completing.lock_inner().writer = Some(sender);
completing.lock_inner().daemon_ver = "0.39.0".into();
completing.notify.notify_waiters();
}));
let result = shared
.ensure_connected_for_call(Instant::now() + Duration::from_millis(30))
.await;
assert!(
result.is_ok(),
"connected caller should not time out: {result:?}"
);
assert!(shared.before_wait.lock().unwrap().is_none());
}
#[tokio::test]
async fn catalog_list_deadline_is_not_sent_when_reconnection_stays_down() {
let shared = Arc::new(reader_skew_tests::legacy_shared(
PathBuf::from("/tmp/subc-client-rs-catalog-list-unavailable"),
ConsumerOptions {
call_timeout: Duration::from_millis(25),
reconnect_backoff: RetryBackoff {
base: Duration::from_millis(1),
cap: Duration::from_millis(1),
max_attempts: 100,
},
..ConsumerOptions::default()
},
));
let consumer = SubcConsumer { shared };
let result = tokio::time::timeout(Duration::from_millis(250), consumer.catalog_list())
.await
.expect("catalog.list must finish at its configured deadline")
.unwrap_err();
assert!(matches!(result, CallError::NotSent(_)));
}
fn consumer_with_malformed_connection_file(
) -> (cortexkit_test_support::ScratchDir, SubcConsumer) {
let temp = cortexkit_test_support::ScratchDir::new("consumer-malformed-connection");
let path = temp.path().join("connection.json");
std::fs::write(&path, b"not JSON").unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
}
let shared = Arc::new(reader_skew_tests::legacy_shared(
path,
ConsumerOptions {
call_timeout: Duration::from_millis(100),
reconnect_backoff: RetryBackoff {
base: Duration::from_millis(20),
cap: Duration::from_millis(40),
max_attempts: 6,
},
..ConsumerOptions::default()
},
));
(temp, SubcConsumer { shared })
}
#[tokio::test]
async fn catalog_list_backs_off_after_malformed_connection_file_and_close_cancels_it() {
let (_temp, consumer) = consumer_with_malformed_connection_file();
let call = consumer.catalog_list();
tokio::pin!(call);
assert_pending(call.as_mut()).await;
consumer.close().await;
let err = call.await.unwrap_err();
assert!(matches!(err, CallError::NotSent(_)));
assert!(err.to_string().contains("consumer closed"));
}
#[tokio::test]
async fn spawn_snapshot_backs_off_after_malformed_connection_file_and_close_cancels_it() {
let (_temp, consumer) = consumer_with_malformed_connection_file();
let call = consumer.spawn_snapshot();
tokio::pin!(call);
assert_pending(call.as_mut()).await;
consumer.close().await;
let err = call.await.unwrap_err();
assert!(matches!(err, CallError::NotSent(_)));
assert!(err.to_string().contains("consumer closed"));
}
}
#[cfg(test)]
mod launch_nonce_redaction_tests {
use super::*;
const NONCE: &str = "nonce-f00dfeed1234abcd";
#[test]
fn option_types_and_route_keys_never_print_the_nonce() {
let identity = ConsumerIdentity {
module_id: "wernicke".to_string(),
launch_nonce: NONCE.to_string(),
};
let call = CallOptions {
consumer_identity: Some(identity.clone()),
..CallOptions::default()
};
let key = ConsumerIdentityKey::from(&identity);
for printed in [format!("{call:?}"), format!("{key:?}")] {
assert!(!printed.contains(NONCE), "launch nonce printed: {printed}");
}
}
}