use std::{
collections::{BTreeSet, HashMap},
error::Error,
fmt,
future::Future,
io,
path::{Path, PathBuf},
pin::Pin,
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc, Mutex, MutexGuard, OnceLock,
},
task::{Context, Poll},
time::{Duration, SystemTime, UNIX_EPOCH},
};
use subc_control::{
CatalogEntry, ClientControlRequest, ClientControlResponse, ConsumerIdentity, PollKind,
};
pub use subc_control::{LiveSpawn, SpawnCursor, SpawnEvent, SpawnEventKind, SpawnSnapshot};
use subc_protocol::{
error_codes, manifest::is_valid_capability_identifier, AdmissionClass, BindIdentity, ErrorBody,
Flags, Frame, FrameBuildError, FrameType, Priority, RouteTarget, SUBC_MODULE_ID_ENV,
};
use crate::RouteHandle;
use subc_transport::{
authenticate_client, connection_file, read_frame, write_frame, AuthError, ConnectionFileError,
FrameIoError,
};
use tokio::{
io::{AsyncWrite, AsyncWriteExt, BufWriter},
net::{tcp::OwnedReadHalf, TcpStream},
sync::{mpsc, oneshot, Notify, OwnedSemaphorePermit, Semaphore},
task::JoinHandle,
time::{sleep, sleep_until, timeout_at, Instant},
};
use tokio_util::sync::CancellationToken;
const DEFAULT_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(2);
pub const DEFAULT_CALL_TIMEOUT: Duration = Duration::from_secs(30);
pub const DEFAULT_ROUTE_RETRY_DEADLINE: Duration = Duration::from_secs(90);
pub const MAX_ROUTE_OPENS_IN_FLIGHT: usize = 8;
const DEFAULT_RESTORED_DEBOUNCE: Duration = Duration::from_millis(250);
pub const DEFAULT_LIVENESS_PROBE_WINDOW: Duration = Duration::from_secs(2);
const EGRESS_BUFFER: usize = 128;
const DEFAULT_ROUTE_WINDOW: usize = 1024;
const DEFAULT_SUBSCRIPTION_EVENT_BUFFER: usize = 128;
const DEFAULT_PUSH_EVENT_BUFFER: usize = 128;
const REVERSE_REQUEST_UNHANDLED: &str = "reverse_request_unhandled";
const SPAWN_EVENT_BUFFER: usize = 8192;
pub const SPAWN_CURSOR_INCARNATION_MISMATCH: &str = "spawn_cursor_incarnation_mismatch";
pub const SPAWN_CURSOR_TOO_OLD: &str = "spawn_cursor_too_old";
pub const SPAWN_SUBSCRIBER_LAGGED: &str = "spawn_subscriber_lagged";
type ReverseRequestFuture =
Pin<Box<dyn Future<Output = Result<Vec<u8>, ReverseRequestError>> + Send + 'static>>;
type ReverseRequestHandler =
Arc<dyn Fn(Vec<u8>, ReverseRequestContext) -> ReverseRequestFuture + Send + Sync + 'static>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ReverseRequestContext {
pub corr: u64,
pub method: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ReverseRequestError {
message: String,
}
impl ReverseRequestError {
pub fn new(message: impl Into<String>) -> Self {
Self {
message: message.into(),
}
}
}
impl fmt::Display for ReverseRequestError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.message)
}
}
impl Error for ReverseRequestError {}
#[derive(Default)]
struct ReverseRequestRegistryState {
handlers: HashMap<String, ReverseRequestHandler>,
declared_families: Option<BTreeSet<String>>,
}
#[derive(Clone, Default)]
pub struct ReverseRequestRegistry {
inner: Arc<Mutex<ReverseRequestRegistryState>>,
}
impl fmt::Debug for ReverseRequestRegistry {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ReverseRequestRegistry")
.field("capabilities", &self.capabilities())
.finish()
}
}
impl PartialEq for ReverseRequestRegistry {
fn eq(&self, other: &Self) -> bool {
self.capabilities() == other.capabilities()
}
}
impl Eq for ReverseRequestRegistry {}
impl ReverseRequestRegistry {
pub fn new() -> Self {
Self::default()
}
pub fn on_request<F, Fut>(
&self,
method_family: impl Into<String>,
handler: F,
) -> Result<(), ReverseRequestRegistrationError>
where
F: Fn(Vec<u8>, ReverseRequestContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Vec<u8>> + Send + 'static,
{
self.on_request_fallible(method_family, move |body, ctx| {
let future = handler(body, ctx);
async move { Ok(future.await) }
})
}
pub fn on_request_fallible<F, Fut>(
&self,
method_family: impl Into<String>,
handler: F,
) -> Result<(), ReverseRequestRegistrationError>
where
F: Fn(Vec<u8>, ReverseRequestContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<Vec<u8>, ReverseRequestError>> + Send + 'static,
{
let method_family = method_family.into();
if !is_valid_method_family(&method_family) {
return Err(ReverseRequestRegistrationError::InvalidFamily(
method_family,
));
}
let mut state = self
.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if state
.declared_families
.as_ref()
.is_some_and(|families| !families.contains(&method_family))
{
return Err(ReverseRequestRegistrationError::NotDeclared(method_family));
}
state.handlers.insert(
method_family,
Arc::new(move |body, ctx| Box::pin(handler(body, ctx))),
);
Ok(())
}
fn capabilities(&self) -> Vec<String> {
let state = self
.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let mut capabilities = state.handlers.keys().cloned().collect::<Vec<_>>();
capabilities.sort();
capabilities
}
fn handler(&self, method_family: &str) -> Option<ReverseRequestHandler> {
self.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.handlers
.get(method_family)
.cloned()
}
pub(crate) fn seal(&self) {
let mut state = self
.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if state.declared_families.is_none() {
state.declared_families = Some(state.handlers.keys().cloned().collect());
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ReverseRequestRegistrationError {
InvalidFamily(String),
NotDeclared(String),
NotConsumerRoute,
}
impl fmt::Display for ReverseRequestRegistrationError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidFamily(family) => {
write!(f, "invalid reverse-request method family {family:?}")
}
Self::NotDeclared(family) => write!(
f,
"reverse-request capability {family:?} was not registered before route.open"
),
Self::NotConsumerRoute => {
f.write_str("route handle does not belong to a consumer route")
}
}
}
}
impl Error for ReverseRequestRegistrationError {}
fn is_valid_method_family(value: &str) -> bool {
let mut segments = value.split(['.', '_', '-']);
segments.next().is_some_and(|segment| {
segment
.as_bytes()
.first()
.is_some_and(u8::is_ascii_lowercase)
&& segment
.bytes()
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit())
}) && segments.all(|segment| {
!segment.is_empty()
&& segment
.bytes()
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit())
})
}
static NEXT_REVERSE_REQUEST_REGISTRY_ID: AtomicU64 = AtomicU64::new(1);
static REVERSE_REQUEST_REGISTRIES: OnceLock<Mutex<HashMap<u64, ReverseRequestRegistry>>> =
OnceLock::new();
fn reverse_request_registries() -> &'static Mutex<HashMap<u64, ReverseRequestRegistry>> {
REVERSE_REQUEST_REGISTRIES.get_or_init(|| Mutex::new(HashMap::new()))
}
pub(crate) fn install_reverse_request_registry(registry: ReverseRequestRegistry) -> u64 {
let id = NEXT_REVERSE_REQUEST_REGISTRY_ID
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |id| id.checked_add(1))
.expect("reverse-request route registry id exhausted");
reverse_request_registries()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.insert(id, registry);
id
}
pub(crate) fn route_reverse_request_registry(
id: u64,
) -> Result<ReverseRequestRegistry, ReverseRequestRegistrationError> {
if id == 0 {
return Err(ReverseRequestRegistrationError::NotConsumerRoute);
}
reverse_request_registries()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.get(&id)
.cloned()
.ok_or(ReverseRequestRegistrationError::NotConsumerRoute)
}
fn release_reverse_request_registry(handle: RouteHandle) {
let id = handle.reverse_request_registry_id();
if id != 0 {
reverse_request_registries()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.remove(&id);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RetryBackoff {
pub base: Duration,
pub cap: Duration,
pub max_attempts: usize,
}
impl Default for RetryBackoff {
fn default() -> Self {
Self {
base: Duration::from_millis(100),
cap: Duration::from_secs(2),
max_attempts: 6,
}
}
}
impl RetryBackoff {
fn delay_after_attempt(self, attempt: usize) -> Duration {
let mut delay = self.base;
for _ in 1..attempt {
delay = (delay * 2).min(self.cap);
}
delay
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ConsumerOptions {
pub handshake_timeout: Duration,
pub call_timeout: Duration,
pub reconnect_backoff: RetryBackoff,
pub restored_debounce: Duration,
pub liveness_probe_window: Duration,
}
impl Default for ConsumerOptions {
fn default() -> Self {
Self {
handshake_timeout: DEFAULT_HANDSHAKE_TIMEOUT,
call_timeout: DEFAULT_CALL_TIMEOUT,
reconnect_backoff: RetryBackoff::default(),
restored_debounce: DEFAULT_RESTORED_DEBOUNCE,
liveness_probe_window: DEFAULT_LIVENESS_PROBE_WINDOW,
}
}
}
#[derive(Debug, Clone)]
pub struct CloseRouteOptions {
pub drain: bool,
pub drain_timeout: Duration,
pub consumer_identity: Option<ConsumerIdentity>,
pub reverse_requests: ReverseRequestRegistry,
}
impl Default for CloseRouteOptions {
fn default() -> Self {
Self {
drain: false,
drain_timeout: DEFAULT_CALL_TIMEOUT,
consumer_identity: None,
reverse_requests: ReverseRequestRegistry::new(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CallOptions {
pub timeout: Duration,
pub priority: Priority,
pub admission_class: AdmissionClass,
pub route_retry: RetryBackoff,
pub route_retry_deadline: Duration,
pub consumer_identity: Option<ConsumerIdentity>,
pub reverse_requests: ReverseRequestRegistry,
}
impl Default for CallOptions {
fn default() -> Self {
Self {
timeout: DEFAULT_CALL_TIMEOUT,
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
route_retry: RetryBackoff::default(),
route_retry_deadline: DEFAULT_ROUTE_RETRY_DEADLINE,
consumer_identity: None,
reverse_requests: ReverseRequestRegistry::new(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SubscribeOptions {
pub priority: Priority,
pub admission_class: AdmissionClass,
pub event_buffer: usize,
pub route_retry: RetryBackoff,
pub route_retry_deadline: Duration,
pub route_open_timeout: Duration,
pub consumer_identity: Option<ConsumerIdentity>,
pub reverse_requests: ReverseRequestRegistry,
}
impl Default for SubscribeOptions {
fn default() -> Self {
Self {
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
event_buffer: DEFAULT_SUBSCRIPTION_EVENT_BUFFER,
route_retry: RetryBackoff::default(),
route_retry_deadline: DEFAULT_ROUTE_RETRY_DEADLINE,
route_open_timeout: DEFAULT_CALL_TIMEOUT,
consumer_identity: None,
reverse_requests: ReverseRequestRegistry::new(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ConnectionState {
Dropped,
Restored { epoch: u64 },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RoutePollResult {
pub handle: RouteHandle,
pub status: Option<String>,
pub live: Option<bool>,
}
#[derive(Debug, Clone, serde::Deserialize, PartialEq)]
pub struct CatalogList {
pub generation: u64,
#[serde(default)]
pub modules: Vec<CatalogEntry>,
#[serde(default)]
pub subc_ops: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PushEvent {
pub handle: RouteHandle,
pub body: Vec<u8>,
}
enum DroppedPush {
Delivered,
NoReceiver,
ReceiverFull,
}
#[derive(Debug, Clone)]
pub struct ControlPush {
pub op: String,
pub body: serde_json::Value,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RouteCloseReason {
Reload,
Restart,
Disable,
Crash,
CapabilityDenied,
Unknown(String),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RouteCloseDisposition {
MayReopen,
MustNotReopen,
}
impl RouteCloseReason {
pub fn from_wire(reason: &str) -> Self {
match reason {
"reload" => Self::Reload,
"restart" => Self::Restart,
"disable" => Self::Disable,
"crash" => Self::Crash,
"capability_denied" => Self::CapabilityDenied,
other => Self::Unknown(other.to_string()),
}
}
pub fn disposition(&self) -> RouteCloseDisposition {
match self {
Self::Reload | Self::Restart => RouteCloseDisposition::MayReopen,
Self::Disable | Self::Crash | Self::CapabilityDenied | Self::Unknown(_) => {
RouteCloseDisposition::MustNotReopen
}
}
}
}
impl ControlPush {
pub fn route_close_reason(&self) -> Option<RouteCloseReason> {
matches!(self.op.as_str(), "route.closing" | "route.closed")
.then(|| {
self.body
.get("reason")?
.as_str()
.map(RouteCloseReason::from_wire)
})
.flatten()
}
}
pub struct SubcConsumer {
shared: Arc<Shared>,
}
pub struct Subscription {
events: mpsc::Receiver<Vec<u8>>,
closed: SubscriptionClosed,
cancel: SubscriptionCancel,
}
impl Subscription {
pub fn events(&mut self) -> &mut mpsc::Receiver<Vec<u8>> {
&mut self.events
}
pub fn closed(&mut self) -> &mut SubscriptionClosed {
&mut self.closed
}
pub fn unsubscribe(&self) -> Result<(), CallError> {
self.cancel.unsubscribe()
}
}
impl Drop for Subscription {
fn drop(&mut self) {
let _ = self.cancel.unsubscribe();
}
}
pub struct SpawnSubscription {
inner: Subscription,
finished: bool,
}
impl SpawnSubscription {
pub async fn next(&mut self) -> Result<Option<SpawnEvent>, SpawnStreamError> {
if self.finished {
return Ok(None);
}
if let Some(body) = self.inner.events().recv().await {
return serde_json::from_slice::<SpawnEvent>(&body)
.map(Some)
.map_err(|err| {
self.finished = true;
let _ = self.inner.unsubscribe();
SpawnStreamError::Decode(err.to_string())
});
}
let terminal = self.inner.closed().await;
self.finished = true;
match terminal {
Ok(()) => Ok(None),
Err(err) => Err(SpawnStreamError::from_call_error(err)),
}
}
pub fn unsubscribe(&self) -> Result<(), CallError> {
self.inner.unsubscribe()
}
}
#[derive(Debug)]
pub enum SpawnStreamError {
CursorIncarnationMismatch {
current_daemon_incarnation: String,
body: Box<ErrorBody>,
},
CursorTooOld {
oldest_retained_cursor: SpawnCursor,
body: Box<ErrorBody>,
},
SubscriberLagged {
first_undelivered_cursor: SpawnCursor,
body: Box<ErrorBody>,
},
Call(CallError),
Decode(String),
}
impl SpawnStreamError {
fn from_call_error(err: CallError) -> Self {
let CallError::Module(body) = err else {
return Self::Call(err);
};
let detail = |key: &str| body.detail.as_ref().and_then(|detail| detail.get(key));
let cursor = |key: &str| {
detail(key).and_then(|value| serde_json::from_value::<SpawnCursor>(value.clone()).ok())
};
match body.code.as_str() {
SPAWN_CURSOR_INCARNATION_MISMATCH => {
match detail("current_daemon_incarnation").and_then(|value| value.as_str()) {
Some(current) => Self::CursorIncarnationMismatch {
current_daemon_incarnation: current.to_string(),
body: Box::new(body),
},
None => Self::Call(CallError::Module(body)),
}
}
SPAWN_CURSOR_TOO_OLD => match cursor("oldest_retained_cursor") {
Some(oldest_retained_cursor) => Self::CursorTooOld {
oldest_retained_cursor,
body: Box::new(body),
},
None => Self::Call(CallError::Module(body)),
},
SPAWN_SUBSCRIBER_LAGGED => match cursor("first_undelivered_cursor") {
Some(first_undelivered_cursor) => Self::SubscriberLagged {
first_undelivered_cursor,
body: Box::new(body),
},
None => Self::Call(CallError::Module(body)),
},
_ => Self::Call(CallError::Module(body)),
}
}
pub fn code(&self) -> Option<&str> {
match self {
Self::CursorIncarnationMismatch { .. } => Some(SPAWN_CURSOR_INCARNATION_MISMATCH),
Self::CursorTooOld { .. } => Some(SPAWN_CURSOR_TOO_OLD),
Self::SubscriberLagged { .. } => Some(SPAWN_SUBSCRIBER_LAGGED),
Self::Call(err) => err.code(),
Self::Decode(_) => None,
}
}
}
impl fmt::Display for SpawnStreamError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::CursorIncarnationMismatch {
current_daemon_incarnation,
..
} => write!(
f,
"{SPAWN_CURSOR_INCARNATION_MISMATCH}: the daemon is incarnation \
{current_daemon_incarnation}"
),
Self::CursorTooOld {
oldest_retained_cursor,
..
} => write!(
f,
"{SPAWN_CURSOR_TOO_OLD}: the oldest retained event is seq {}",
oldest_retained_cursor.seq
),
Self::SubscriberLagged {
first_undelivered_cursor,
..
} => write!(
f,
"{SPAWN_SUBSCRIBER_LAGGED}: the first undelivered event is seq {}",
first_undelivered_cursor.seq
),
Self::Call(err) => write!(f, "spawn stream ended: {err}"),
Self::Decode(message) => write!(f, "spawn event did not decode: {message}"),
}
}
}
impl Error for SpawnStreamError {}
pub struct SubscriptionClosed {
rx: oneshot::Receiver<Result<(), CallError>>,
}
impl Unpin for SubscriptionClosed {}
impl Future for SubscriptionClosed {
type Output = Result<(), CallError>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let this = self.get_mut();
match Pin::new(&mut this.rx).poll(cx) {
Poll::Ready(Ok(result)) => Poll::Ready(result),
Poll::Ready(Err(_)) => Poll::Ready(Err(CallError::outcome_unknown(
"subscription closed result channel dropped",
))),
Poll::Pending => Poll::Pending,
}
}
}
struct SubscriptionCancel {
shared: Arc<Shared>,
key: PendingKey,
priority: Priority,
cancelled: AtomicBool,
}
impl SubscriptionCancel {
fn new(shared: Arc<Shared>, key: PendingKey, priority: Priority) -> Self {
Self {
shared,
key,
priority,
cancelled: AtomicBool::new(false),
}
}
fn unsubscribe(&self) -> Result<(), CallError> {
let handle = RouteHandle::new(self.key.channel, self.key.epoch, self.key.generation);
self.shared.validate_current_handle(handle)?;
if self.cancelled.swap(true, Ordering::AcqRel) {
return Ok(());
}
self.shared
.unsubscribe_subscription(self.key, self.priority)
}
}
impl SubcConsumer {
pub async fn connect(
connection_file: &Path,
opts: ConsumerOptions,
) -> Result<Self, ConsumerError> {
let opened = open_connection(connection_file, opts.handshake_timeout).await?;
let shared = Arc::new(Shared::new(connection_file.to_path_buf(), opts));
shared.install_initial(opened)?;
Ok(Self { shared })
}
pub async fn connect_default(opts: ConsumerOptions) -> Result<Self, ConsumerError> {
let discovered = connection_file::discover(None)
.map_err(|source| ConsumerError::Discovery { source })?;
let opened =
open_connection_with_info(&discovered.path, &discovered.info, opts.handshake_timeout)
.await?;
let shared = Arc::new(Shared::new(discovered.path, opts));
shared.install_initial(opened)?;
Ok(Self { shared })
}
pub async fn open_route(
&self,
target: RouteTarget,
identity: BindIdentity,
opts: CallOptions,
) -> Result<RouteHandle, CallError> {
let deadline = Instant::now() + opts.timeout;
let consumer_identity = route_open_consumer_identity(&opts);
let consumer_capabilities = route_open_consumer_capabilities(&opts);
let key = RouteKey::new(
&target,
&identity,
consumer_identity.as_ref(),
consumer_capabilities.as_deref(),
);
let params = RouteOpenParams {
target: &target,
identity: &identity,
consumer_identity: &consumer_identity,
consumer_capabilities: &consumer_capabilities,
reverse_requests: &opts.reverse_requests,
};
self.shared
.ensure_route(&key, ¶ms, &opts, deadline)
.await
.map(|route| route.handle)
}
pub async fn open_route_with_admission_facts(
&self,
target: RouteTarget,
identity: BindIdentity,
facts: serde_json::Value,
) -> Result<RouteHandle, CallError> {
let deadline = Instant::now() + self.shared.opts.call_timeout;
self.open_admitted_route(
target,
identity,
facts,
&CallOptions::default(),
None,
deadline,
)
.await
}
pub async fn open_route_with_admission_facts_and_options(
&self,
target: RouteTarget,
identity: BindIdentity,
facts: serde_json::Value,
opts: CallOptions,
) -> Result<RouteHandle, CallError> {
let deadline = Instant::now() + opts.timeout;
let reverse_requests = opts.reverse_requests.clone();
self.open_admitted_route(
target,
identity,
facts,
&opts,
Some(reverse_requests),
deadline,
)
.await
}
async fn open_admitted_route(
&self,
target: RouteTarget,
identity: BindIdentity,
facts: serde_json::Value,
opts: &CallOptions,
reverse_requests: Option<ReverseRequestRegistry>,
deadline: Instant,
) -> Result<RouteHandle, CallError> {
let consumer_capabilities = if reverse_requests.is_some() {
route_open_consumer_capabilities(opts)
} else {
None
};
let target_label = route_target_label(&target);
let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
target,
identity,
consumer_identity: route_open_consumer_identity(opts),
consumer_capabilities,
admission_facts: Some(facts),
scope: None,
})
.map_err(|err| CallError::not_sent(format!("failed to encode route.open: {err}")))?;
let terminal = self
.shared
.paced_route_open(body, deadline, reverse_requests.clone())
.await?;
let (generation, body) = match terminal {
TerminalFrame::Response {
generation, body, ..
} => (generation, body),
TerminalFrame::Error { body, .. } => {
return Err(CallError::route_open_refused(target_label, body));
}
_ => {
return Err(CallError::not_sent(
"route.open returned a non-response frame",
));
}
};
let ClientControlResponse::RouteOpen {
route_channel,
route_epoch,
} = serde_json::from_slice(&body).map_err(|err| {
CallError::not_sent(format!("failed to decode route.open response: {err}"))
})?
else {
return Err(CallError::not_sent(
"route.open returned an unexpected control response",
));
};
let handle = self
.shared
.ingress_handle(generation, route_channel, route_epoch)
.unwrap_or_else(|| match reverse_requests {
Some(reverse_requests) => RouteHandle::new_consumer(
route_channel,
route_epoch,
generation,
reverse_requests,
),
None => RouteHandle::new(route_channel, route_epoch, generation),
});
let route = RouteState {
handle,
sem: Arc::new(Semaphore::new(DEFAULT_ROUTE_WINDOW)),
};
self.shared.install_one_shot_route(route.clone())?;
Ok(route.handle)
}
pub async fn catalog_list(&self) -> Result<CatalogList, CallError> {
let deadline = Instant::now() + self.shared.opts.call_timeout;
let body = serde_json::to_vec(&serde_json::json!({
"op": subc_control::ops::CATALOG_LIST,
}))
.map_err(|err| CallError::not_sent(format!("failed to encode catalog.list: {err}")))?;
loop {
match self
.shared
.control_call(body.clone(), deadline, false, None)
.await
{
Ok(TerminalFrame::Response { body, .. }) => {
let response =
serde_json::from_slice::<ClientControlResponse>(&body).map_err(|err| {
CallError::not_sent(format!(
"failed to decode catalog.list response: {err}"
))
})?;
let ClientControlResponse::CatalogList {
generation,
modules,
subc_ops,
} = response
else {
return Err(CallError::not_sent(
"catalog.list returned an unexpected control response",
));
};
return Ok(CatalogList {
generation,
modules,
subc_ops,
});
}
Ok(TerminalFrame::Error { body, .. }) => return Err(CallError::Module(body)),
Ok(TerminalFrame::StreamEnd) => {
return Err(CallError::not_sent("catalog.list returned StreamEnd"));
}
Err(err)
if is_retryable_catalog_transport_error(&err) && Instant::now() < deadline =>
{
continue;
}
Err(err) => return Err(err),
}
}
}
pub async fn spawn_snapshot(&self) -> Result<SpawnSnapshot, CallError> {
let deadline = Instant::now() + self.shared.opts.call_timeout;
let body = serde_json::to_vec(&ClientControlRequest::SupervisorSpawnSnapshot {}).map_err(
|err| CallError::not_sent(format!("failed to encode supervisor.spawn_snapshot: {err}")),
)?;
loop {
match self
.shared
.control_call(body.clone(), deadline, false, None)
.await
{
Ok(TerminalFrame::Response { body, .. }) => {
let response =
serde_json::from_slice::<ClientControlResponse>(&body).map_err(|err| {
CallError::not_sent(format!(
"failed to decode supervisor.spawn_snapshot response: {err}"
))
})?;
let ClientControlResponse::SupervisorSpawnSnapshot { snapshot } = response
else {
return Err(CallError::not_sent(
"supervisor.spawn_snapshot returned an unexpected control response",
));
};
return Ok(snapshot);
}
Ok(TerminalFrame::Error { body, .. }) => return Err(CallError::Module(body)),
Ok(TerminalFrame::StreamEnd) => {
return Err(CallError::not_sent(
"supervisor.spawn_snapshot returned StreamEnd",
));
}
Err(err)
if is_retryable_catalog_transport_error(&err) && Instant::now() < deadline =>
{
continue;
}
Err(err) => return Err(err),
}
}
}
pub async fn spawn_subscribe(
&self,
since: Option<SpawnCursor>,
) -> Result<SpawnSubscription, CallError> {
let deadline = Instant::now() + self.shared.opts.call_timeout;
let body = serde_json::to_vec(&ClientControlRequest::SupervisorSpawnSubscribe { since })
.map_err(|err| {
CallError::not_sent(format!(
"failed to encode supervisor.spawn_subscribe: {err}"
))
})?;
loop {
self.shared.ensure_connected_for_call(deadline).await?;
let permit = Arc::new(Semaphore::new(1))
.acquire_owned()
.await
.map_err(|_| CallError::not_sent("spawn subscription permit closed"))?;
match self
.shared
.send_subscription(SubscriptionSend {
expected_handle: None,
channel: 0,
epoch: 0,
body: body.clone(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
event_buffer: SPAWN_EVENT_BUFFER,
deadline,
permit,
})
.await
{
Ok(inner) => {
return Ok(SpawnSubscription {
inner,
finished: false,
})
}
Err(err) if err.is_not_sent() && Instant::now() < deadline => continue,
Err(err) => return Err(err),
}
}
}
pub async fn resolve_provider(&self, capability: &str) -> Result<String, CallError> {
let claimants = self.resolve_providers(capability).await?;
match claimants.as_slice() {
[] => Err(CallError::CapabilityUnprovided {
capability: capability.to_string(),
}),
[claimant] => Ok(claimant.clone()),
_ => Err(CallError::CapabilityAmbiguous {
capability: capability.to_string(),
claimants,
}),
}
}
pub async fn resolve_providers(&self, capability: &str) -> Result<Vec<String>, CallError> {
validate_capability_for_resolution(capability)?;
let catalog = self.catalog_list().await?;
Ok(capability_claimants(&catalog, capability))
}
pub async fn request(
&self,
handle: &RouteHandle,
body: Vec<u8>,
opts: CallOptions,
) -> Result<Vec<u8>, CallError> {
let deadline = Instant::now() + opts.timeout;
let route = self.shared.route_state(*handle)?;
let permit = match timeout_at(deadline, Arc::clone(&route.sem).acquire_owned()).await {
Ok(Ok(permit)) => permit,
Ok(Err(_)) => return Err(CallError::StaleRouteHandle(*handle)),
Err(_) => {
return Err(CallError::not_sent(
"call deadline elapsed waiting for route flow-control",
))
}
};
let result = self
.shared
.send_request(RequestSend {
expected_handle: Some(*handle),
channel: handle.channel,
epoch: handle.epoch,
body,
priority: opts.priority,
admission_class: opts.admission_class,
deadline,
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await;
drop(permit);
match result? {
TerminalFrame::Response { body, .. } => Ok(body),
TerminalFrame::StreamEnd => Ok(Vec::new()),
TerminalFrame::Error { body, .. } => Err(CallError::Module(body)),
}
}
pub async fn subscribe_route(
&self,
handle: &RouteHandle,
body: Vec<u8>,
opts: SubscribeOptions,
) -> Result<Subscription, CallError> {
let deadline = Instant::now() + opts.route_open_timeout;
let route = self.shared.route_state(*handle)?;
let permit = match timeout_at(deadline, Arc::clone(&route.sem).acquire_owned()).await {
Ok(Ok(permit)) => permit,
Ok(Err(_)) => return Err(CallError::StaleRouteHandle(*handle)),
Err(_) => {
return Err(CallError::not_sent(
"subscription deadline elapsed waiting for route flow-control",
))
}
};
self.shared
.send_subscription(SubscriptionSend {
expected_handle: Some(*handle),
channel: handle.channel,
epoch: handle.epoch,
body,
priority: opts.priority,
admission_class: opts.admission_class,
event_buffer: opts.event_buffer,
deadline,
permit,
})
.await
}
pub async fn poll_route(
&self,
handle: &RouteHandle,
kind: PollKind,
timeout: Duration,
) -> Result<RoutePollResult, CallError> {
let deadline = Instant::now() + timeout;
let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
route_channel: handle.channel,
route_epoch: handle.epoch,
kind,
})
.map_err(|err| CallError::not_sent(format!("failed to encode route.poll: {err}")))?;
let terminal = self
.shared
.send_request(RequestSend {
expected_handle: Some(*handle),
channel: 0,
epoch: 0,
body,
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline,
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await?;
let TerminalFrame::Response { body, .. } = terminal else {
return Err(CallError::not_sent(
"route.poll returned a non-response frame",
));
};
let ClientControlResponse::RoutePoll {
route_channel,
route_epoch,
status,
live,
} = serde_json::from_slice(&body)
.map_err(|err| CallError::not_sent(format!("failed to decode route.poll: {err}")))?
else {
return Err(CallError::not_sent(
"route.poll returned an unexpected control response",
));
};
if route_channel != handle.channel || route_epoch != handle.epoch {
return Err(CallError::not_sent(
"route.poll response echoed a different route handle",
));
}
self.shared.validate_current_handle(*handle)?;
Ok(RoutePollResult {
handle: *handle,
status,
live,
})
}
pub fn dropped_route_frames(&self) -> u64 {
self.shared.lock_inner().dropped_route_frames
}
pub fn push_events(
&self,
handle: &RouteHandle,
) -> Result<mpsc::Receiver<PushEvent>, CallError> {
self.shared.register_push_events(*handle)
}
pub fn pushes_dropped_no_receiver(&self) -> u64 {
self.shared
.pushes_dropped_no_receiver
.load(Ordering::Relaxed)
}
pub fn pushes_dropped_receiver_full(&self) -> u64 {
self.shared
.pushes_dropped_receiver_full
.load(Ordering::Relaxed)
}
pub fn control_pushes(&self, capacity: usize) -> mpsc::Receiver<ControlPush> {
let (sender, receiver) = mpsc::channel(capacity.max(1));
self.shared.lock_inner().control_push_receiver = Some(sender);
receiver
}
pub fn control_pushes_dropped(&self) -> u64 {
self.shared.control_pushes_dropped.load(Ordering::Relaxed)
}
pub async fn call(
&self,
target: RouteTarget,
identity: BindIdentity,
body: Vec<u8>,
opts: CallOptions,
) -> Result<Vec<u8>, CallError> {
let call_deadline = Instant::now() + opts.timeout;
let mut retried_unknown_channel = false;
let consumer_identity = route_open_consumer_identity(&opts);
let consumer_capabilities = route_open_consumer_capabilities(&opts);
let route_key = RouteKey::new(
&target,
&identity,
consumer_identity.as_ref(),
consumer_capabilities.as_deref(),
);
let route_open = RouteOpenParams {
target: &target,
identity: &identity,
consumer_identity: &consumer_identity,
consumer_capabilities: &consumer_capabilities,
reverse_requests: &opts.reverse_requests,
};
loop {
let route = self
.shared
.ensure_route(&route_key, &route_open, &opts, call_deadline)
.await
.map_err(request_not_sent_after_route_open_failure)?;
let permit =
match timeout_at(call_deadline, Arc::clone(&route.sem).acquire_owned()).await {
Ok(Ok(permit)) => permit,
Ok(Err(_)) => {
return Err(CallError::not_sent("route flow-control semaphore closed"));
}
Err(_) => {
return Err(CallError::not_sent(
"call deadline elapsed waiting for route flow-control",
));
}
};
if !self.shared.route_is_current(&route_key, &route) {
drop(permit);
self.shared
.sleep_until_retry(call_deadline, opts.route_retry.base)
.await?;
continue;
}
let response = self
.shared
.send_request(RequestSend {
expected_handle: Some(route.handle),
channel: route.handle.channel,
epoch: route.handle.epoch,
body: body.clone(),
priority: opts.priority,
admission_class: opts.admission_class,
deadline: call_deadline,
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await;
drop(permit);
match response {
Ok(TerminalFrame::Response { body, .. }) => return Ok(body),
Ok(TerminalFrame::StreamEnd) => return Ok(Vec::new()),
Ok(TerminalFrame::Error { body, flags })
if error_codes::is_established_route_dead(flags, &body.code)
&& !retried_unknown_channel
&& Instant::now() < call_deadline =>
{
retried_unknown_channel = true;
self.shared.invalidate_route(&route_key, Some(route.handle));
continue;
}
Ok(TerminalFrame::Error { body, .. }) => return Err(CallError::Module(body)),
Err(err) if err.is_not_sent() && Instant::now() < call_deadline => {
self.shared.invalidate_route(&route_key, Some(route.handle));
self.shared.ensure_connected_for_call(call_deadline).await?;
continue;
}
Err(err) => return Err(err),
}
}
}
pub async fn subscribe(
&self,
target: RouteTarget,
identity: BindIdentity,
body: Vec<u8>,
opts: SubscribeOptions,
) -> Result<Subscription, CallError> {
let open_deadline = Instant::now() + opts.route_open_timeout;
let route_opts = CallOptions {
timeout: opts.route_open_timeout,
priority: opts.priority,
admission_class: opts.admission_class,
route_retry: opts.route_retry,
route_retry_deadline: opts.route_retry_deadline,
consumer_identity: opts.consumer_identity.clone(),
reverse_requests: opts.reverse_requests.clone(),
};
let consumer_identity = route_open_consumer_identity(&route_opts);
let consumer_capabilities = route_open_consumer_capabilities(&route_opts);
let route_key = RouteKey::new(
&target,
&identity,
consumer_identity.as_ref(),
consumer_capabilities.as_deref(),
);
let route_open = RouteOpenParams {
target: &target,
identity: &identity,
consumer_identity: &consumer_identity,
consumer_capabilities: &consumer_capabilities,
reverse_requests: &opts.reverse_requests,
};
loop {
let route = self
.shared
.ensure_route(&route_key, &route_open, &route_opts, open_deadline)
.await
.map_err(request_not_sent_after_route_open_failure)?;
let permit =
match timeout_at(open_deadline, Arc::clone(&route.sem).acquire_owned()).await {
Ok(Ok(permit)) => permit,
Ok(Err(_)) => {
return Err(CallError::not_sent("route flow-control semaphore closed"));
}
Err(_) => {
return Err(CallError::not_sent(
"subscription open deadline elapsed waiting for route flow-control",
));
}
};
if !self.shared.route_is_current(&route_key, &route) {
drop(permit);
self.shared
.sleep_until_retry(open_deadline, opts.route_retry.base)
.await?;
continue;
}
match self
.shared
.send_subscription(SubscriptionSend {
expected_handle: Some(route.handle),
channel: route.handle.channel,
epoch: route.handle.epoch,
body: body.clone(),
priority: opts.priority,
admission_class: opts.admission_class,
event_buffer: opts.event_buffer,
deadline: open_deadline,
permit,
})
.await
{
Ok(subscription) => return Ok(subscription),
Err(err) if err.is_not_sent() && Instant::now() < open_deadline => {
self.shared.invalidate_route(&route_key, Some(route.handle));
self.shared.ensure_connected_for_call(open_deadline).await?;
continue;
}
Err(err) => return Err(err),
}
}
}
pub async fn close_route(
&self,
target: RouteTarget,
identity: BindIdentity,
opts: CloseRouteOptions,
) {
let consumer_identity = close_route_consumer_identity(&opts);
let consumer_capabilities = close_route_consumer_capabilities(&opts);
let key = RouteKey::new(
&target,
&identity,
consumer_identity.as_ref(),
consumer_capabilities.as_deref(),
);
self.shared.close_route(&key, &opts).await;
}
pub async fn close_handle(
&self,
handle: &RouteHandle,
opts: CloseRouteOptions,
) -> Result<(), CallError> {
self.shared.close_handle(*handle, &opts).await
}
pub fn current_epoch(&self) -> u64 {
self.shared.lock_inner().epoch
}
pub fn on_connection_state(&self, cb: impl Fn(ConnectionState) + Send + 'static) {
self.shared
.lock_inner()
.callbacks
.push(Arc::new(Mutex::new(Box::new(cb))));
}
pub async fn close(&self) {
self.shared.close_sync("consumer closed");
tokio::task::yield_now().await;
}
}
impl Drop for SubcConsumer {
fn drop(&mut self) {
self.shared.close_sync("consumer dropped");
}
}
#[derive(Debug)]
pub enum ConsumerError {
Discovery {
source: connection_file::DiscoveryError,
},
ConnectionFile {
path: PathBuf,
source: ConnectionFileError,
},
NoEndpoint {
path: PathBuf,
},
Connect {
path: PathBuf,
endpoint: String,
source: io::Error,
},
Auth {
path: PathBuf,
endpoint: String,
source: AuthError,
},
Closed,
}
impl fmt::Display for ConsumerError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Discovery { source } => source.fmt(f),
Self::ConnectionFile { path, source } => write!(
f,
"failed to read subc connection file '{}': {source}",
path.display()
),
Self::NoEndpoint { path } => {
write!(
f,
"subc connection file '{}' has no endpoints",
path.display()
)
}
Self::Connect {
path,
endpoint,
source,
} => write!(
f,
"failed to connect to subc endpoint {endpoint} from '{}': {source}",
path.display()
),
Self::Auth {
path,
endpoint,
source,
} => write!(
f,
"failed to authenticate to subc endpoint {endpoint} from '{}': {source}",
path.display()
),
Self::Closed => write!(f, "consumer closed"),
}
}
}
impl Error for ConsumerError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Discovery { source } => Some(source),
Self::ConnectionFile { source, .. } => Some(source),
Self::Connect { source, .. } => Some(source),
Self::Auth { source, .. } => Some(source),
Self::NoEndpoint { .. } | Self::Closed => None,
}
}
}
#[derive(Debug)]
pub enum CallError {
NotSent(Box<dyn Error + Send + Sync>),
OutcomeUnknown(Box<dyn Error + Send + Sync>),
Module(ErrorBody),
SubscriptionBackpressure(Box<dyn Error + Send + Sync>),
StaleRouteHandle(RouteHandle),
CapabilityUnprovided { capability: String },
CapabilityAmbiguous {
capability: String,
claimants: Vec<String>,
},
InvalidCapabilityIdentifier { capability: String },
}
impl CallError {
pub fn code(&self) -> Option<&str> {
match self {
Self::Module(body) => Some(&body.code),
Self::CapabilityUnprovided { .. } => Some("capability_unprovided"),
Self::CapabilityAmbiguous { .. } => Some("capability_ambiguous"),
Self::InvalidCapabilityIdentifier { .. } => Some("invalid_capability_identifier"),
Self::NotSent(_)
| Self::OutcomeUnknown(_)
| Self::SubscriptionBackpressure(_)
| Self::StaleRouteHandle(_) => None,
}
}
fn not_sent(reason: impl Into<String>) -> Self {
Self::NotSent(Box::new(SimpleError(reason.into())))
}
fn outcome_unknown(reason: impl Into<String>) -> Self {
Self::OutcomeUnknown(Box::new(SimpleError(reason.into())))
}
fn is_not_sent(&self) -> bool {
matches!(self, Self::NotSent(_))
}
pub fn route_open_refusal(&self) -> Option<&ErrorBody> {
match self {
Self::NotSent(err) => err
.downcast_ref::<RouteOpenRefused>()
.map(|refused| &refused.body),
_ => None,
}
}
fn subscription_backpressure(reason: impl Into<String>) -> Self {
Self::SubscriptionBackpressure(Box::new(SimpleError(reason.into())))
}
}
impl fmt::Display for CallError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NotSent(err) => write!(f, "request not sent: {err}"),
Self::OutcomeUnknown(err) => write!(f, "request outcome unknown: {err}"),
Self::Module(body) => write!(f, "module error {}: {}", body.code, body.message),
Self::SubscriptionBackpressure(err) => {
write!(f, "subscription event channel backpressure: {err}")
}
Self::StaleRouteHandle(handle) => write!(f, "stale route handle: {handle:?}"),
Self::CapabilityUnprovided { capability } => {
write!(
f,
"capability_unprovided: no catalog claimant for {capability}"
)
}
Self::CapabilityAmbiguous {
capability,
claimants,
} => write!(
f,
"capability_ambiguous: multiple catalog claimants for {capability}: {claimants:?}"
),
Self::InvalidCapabilityIdentifier { capability } => write!(
f,
"invalid_capability_identifier: malformed capability identifier {capability:?}"
),
}
}
}
impl Error for CallError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::NotSent(err)
| Self::OutcomeUnknown(err)
| Self::SubscriptionBackpressure(err) => Some(err.as_ref()),
Self::Module(_)
| Self::StaleRouteHandle(_)
| Self::CapabilityUnprovided { .. }
| Self::CapabilityAmbiguous { .. }
| Self::InvalidCapabilityIdentifier { .. } => None,
}
}
}
#[derive(Debug, Clone)]
struct RouteOpenRefused {
target: String,
body: ErrorBody,
most_recent: Option<ErrorBody>,
}
impl fmt::Display for RouteOpenRefused {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"route.open failed for target {}: {}",
self.target,
DescribeRefusal(&self.body)
)?;
if let Some(most_recent) = &self.most_recent {
write!(
f,
"; the most recent refusal was {}",
DescribeRefusal(most_recent)
)?;
}
Ok(())
}
}
impl Error for RouteOpenRefused {}
struct DescribeRefusal<'a>(&'a ErrorBody);
impl fmt::Display for DescribeRefusal<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.0.code)?;
if let Some(reason) = refusal_reason(self.0) {
write!(f, " (reason {reason})")?;
}
write!(f, " ({})", self.0.message)
}
}
fn refusal_reason(body: &ErrorBody) -> Option<&str> {
body.detail.as_ref()?.get("reason")?.as_str()
}
const ADMISSION_PRESSURE_REASONS: [&str; 2] = ["open_admission_full", "target_binds_full"];
const ADMISSION_PRESSURE_MESSAGES: [&str; 2] =
["route.open binds in flight", "route.bind relays in flight"];
fn route_open_refusal_rank(body: &ErrorBody) -> u8 {
if body.code == error_codes::MODULE_RELOADING || body.code == error_codes::MODULE_WARMING {
return 3;
}
let reason = refusal_reason(body);
if body.code == "delegation_not_registered" || reason == Some("delegation_not_registered") {
return 2;
}
if body.code == error_codes::TARGET_UNAVAILABLE {
let admission_pressure = match reason {
Some(reason) => ADMISSION_PRESSURE_REASONS.contains(&reason),
None => ADMISSION_PRESSURE_MESSAGES
.iter()
.any(|marker| body.message.contains(marker)),
};
if admission_pressure {
return 1;
}
}
0
}
#[derive(Default)]
struct RouteOpenRefusals {
best: Option<ErrorBody>,
most_recent: Option<ErrorBody>,
}
impl RouteOpenRefusals {
fn record(&mut self, body: ErrorBody) {
let outranks = self
.best
.as_ref()
.is_none_or(|best| route_open_refusal_rank(&body) >= route_open_refusal_rank(best));
if outranks {
self.best = Some(body);
self.most_recent = None;
} else {
self.most_recent = Some(body);
}
}
fn into_error(self, target: String) -> Option<CallError> {
let body = self.best?;
Some(CallError::NotSent(Box::new(RouteOpenRefused {
target,
body,
most_recent: self.most_recent,
})))
}
}
fn route_target_label(target: &RouteTarget) -> String {
match target {
RouteTarget::ToolProvider { module_id } => format!("tool_provider:{module_id}"),
RouteTarget::ManagementSurface { module_id } => format!("management_surface:{module_id}"),
RouteTarget::InternalService {
module_id,
service_id,
} => format!("internal_service:{module_id}:{service_id}"),
}
}
impl CallError {
fn route_open_refused(target: String, body: ErrorBody) -> Self {
Self::NotSent(Box::new(RouteOpenRefused {
target,
body,
most_recent: None,
}))
}
}
#[derive(Debug)]
struct SimpleError(String);
impl fmt::Display for SimpleError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.0)
}
}
impl Error for SimpleError {}
type Callback = Arc<Mutex<Box<dyn Fn(ConnectionState) + Send + 'static>>>;
type OpeningWaiter = oneshot::Sender<Result<RouteState, SharedCallFailure>>;
struct Opening {
waiters: Vec<OpeningWaiter>,
closed: bool,
}
struct Shared {
connection_file: PathBuf,
opts: ConsumerOptions,
inner: Mutex<Inner>,
notify: Notify,
#[cfg(test)]
before_wait: Mutex<Option<Box<dyn FnOnce() + Send>>>,
close_token: CancellationToken,
last_inbound_ms: AtomicU64,
liveness_probe_running: AtomicBool,
pushes_dropped_no_receiver: AtomicU64,
control_pushes_dropped: AtomicU64,
pushes_dropped_receiver_full: AtomicU64,
random_unit: Box<dyn Fn() -> f64 + Send + Sync>,
route_open_slots: Semaphore,
}
struct Inner {
generation: u64,
epoch: u64,
next_corr: Option<u64>,
writer: Option<mpsc::Sender<WriteCommand>>,
pending: HashMap<PendingKey, PendingEntry>,
routes: HashMap<RouteKey, RouteState>,
route_by_channel: HashMap<u16, RouteKey>,
one_shot_routes: HashMap<u16, RouteState>,
route_epochs: HashMap<u16, RouteHandle>,
push_event_receivers: HashMap<RouteHandle, mpsc::Sender<PushEvent>>,
control_push_receiver: Option<mpsc::Sender<ControlPush>>,
dropped_route_frames: u64,
openings: HashMap<RouteKey, Opening>,
callbacks: Vec<Callback>,
closed: bool,
reconnect: ReconnectState,
restored_token: u64,
reader_task: Option<JoinHandle<()>>,
writer_task: Option<JoinHandle<()>>,
}
impl Inner {
fn cache_route(&mut self, key: RouteKey, route: RouteState) -> RouteState {
let cached = self.routes.entry(key.clone()).or_insert(route).clone();
let previous = self
.route_by_channel
.insert(cached.handle.channel, key.clone());
debug_assert!(previous.as_ref().is_none_or(|previous| previous == &key));
self.route_epochs
.insert(cached.handle.channel, cached.handle);
cached
}
fn remove_route(&mut self, key: &RouteKey) -> Option<RouteState> {
let route = self.routes.remove(key)?;
let indexed = self.route_by_channel.remove(&route.handle.channel);
debug_assert_eq!(indexed.as_ref(), Some(key));
release_reverse_request_registry(route.handle);
Some(route)
}
fn remove_route_by_handle(&mut self, handle: RouteHandle) -> Option<RouteState> {
if let Some(key) = self.route_by_channel.get(&handle.channel).cloned() {
let matches = self
.routes
.get(&key)
.is_some_and(|route| route.handle == handle);
debug_assert!(matches);
if matches {
return self.remove_route(&key);
}
}
let route = self
.one_shot_routes
.get(&handle.channel)
.is_some_and(|route| route.handle == handle)
.then(|| self.one_shot_routes.remove(&handle.channel))
.flatten();
if let Some(route) = &route {
release_reverse_request_registry(route.handle);
}
route
}
fn drain_routes(&mut self) -> Vec<RouteState> {
self.route_by_channel.clear();
let routes = self
.routes
.drain()
.map(|(_, route)| route)
.chain(self.one_shot_routes.drain().map(|(_, route)| route))
.collect::<Vec<_>>();
for route in &routes {
release_reverse_request_registry(route.handle);
}
routes
}
fn close_routes(&mut self) {
self.push_event_receivers.clear();
for route in self.drain_routes() {
route.sem.close();
}
}
}
impl Shared {
fn new(connection_file: PathBuf, opts: ConsumerOptions) -> Self {
Self {
connection_file,
opts,
inner: Mutex::new(Inner {
generation: 1,
epoch: 1,
next_corr: Some(1),
writer: None,
pending: HashMap::new(),
routes: HashMap::new(),
route_by_channel: HashMap::new(),
one_shot_routes: HashMap::new(),
route_epochs: HashMap::new(),
push_event_receivers: HashMap::new(),
control_push_receiver: None,
dropped_route_frames: 0,
openings: HashMap::new(),
callbacks: Vec::new(),
closed: false,
reconnect: ReconnectState::Idle,
restored_token: 0,
reader_task: None,
writer_task: None,
}),
notify: Notify::new(),
#[cfg(test)]
before_wait: Mutex::new(None),
close_token: CancellationToken::new(),
last_inbound_ms: AtomicU64::new(0),
liveness_probe_running: AtomicBool::new(false),
pushes_dropped_no_receiver: AtomicU64::new(0),
control_pushes_dropped: AtomicU64::new(0),
pushes_dropped_receiver_full: AtomicU64::new(0),
random_unit: Box::new(default_random_unit),
route_open_slots: Semaphore::new(MAX_ROUTE_OPENS_IN_FLIGHT),
}
}
fn jittered(&self, delay: Duration) -> Duration {
let unit = (self.random_unit)();
let unit = if unit.is_finite() {
unit.clamp(0.0, 1.0)
} else {
0.0
};
let half = delay / 2;
half + half.mul_f64(unit)
}
async fn paced_route_open(
self: &Arc<Self>,
body: Vec<u8>,
deadline: Instant,
reverse_requests: Option<ReverseRequestRegistry>,
) -> Result<TerminalFrame, CallError> {
let _slot = match timeout_at(deadline, self.route_open_slots.acquire()).await {
Ok(Ok(permit)) => permit,
Ok(Err(_)) => return Err(CallError::not_sent("consumer closed")),
Err(_) => {
return Err(CallError::not_sent(format!(
"deadline elapsed waiting for one of {MAX_ROUTE_OPENS_IN_FLIGHT} route.open slots"
)));
}
};
self.control_call(body, deadline, true, reverse_requests)
.await
}
fn lock_inner(&self) -> MutexGuard<'_, Inner> {
self.inner
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
fn install_initial(self: &Arc<Self>, opened: OpenedConnection) -> Result<(), ConsumerError> {
self.install_connection(opened, InstallKind::Initial)
.map(|_| ())
}
fn install_reconnected(
self: &Arc<Self>,
opened: OpenedConnection,
) -> Result<(u64, u64), ConsumerError> {
let generation_epoch = self.install_connection(opened, InstallKind::Reconnect)?;
self.notify.notify_waiters();
Ok(generation_epoch)
}
fn install_connection(
self: &Arc<Self>,
opened: OpenedConnection,
kind: InstallKind,
) -> Result<(u64, u64), ConsumerError> {
if self.close_token.is_cancelled() {
return Err(ConsumerError::Closed);
}
let (reader, writer) = opened.stream.into_split();
let (tx, rx) = mpsc::channel(EGRESS_BUFFER);
let (generation, epoch, old_reader, old_writer) = {
let mut inner = self.lock_inner();
if inner.closed {
return Err(ConsumerError::Closed);
}
let (generation, epoch) = match kind {
InstallKind::Initial => (inner.generation, inner.epoch),
InstallKind::Reconnect => {
inner.generation = inner
.generation
.checked_add(1)
.ok_or(ConsumerError::Closed)?;
inner.epoch = inner.epoch.checked_add(1).ok_or(ConsumerError::Closed)?;
(inner.generation, inner.epoch)
}
};
inner.close_routes();
inner.route_epochs.clear();
inner.next_corr = Some(1);
inner.writer = Some(tx);
self.last_inbound_ms.store(0, Ordering::Release);
(
generation,
epoch,
inner.reader_task.take(),
inner.writer_task.take(),
)
};
if let Some(handle) = old_reader {
handle.abort();
}
if let Some(handle) = old_writer {
handle.abort();
}
let reader_shared = Arc::clone(self);
let reader_task = tokio::spawn(async move {
reader_loop(reader_shared, reader, generation).await;
});
let writer_shared = Arc::clone(self);
let writer_task = tokio::spawn(async move {
writer_loop(writer_shared, writer, rx, generation).await;
});
{
let mut inner = self.lock_inner();
if inner.closed || inner.generation != generation {
reader_task.abort();
writer_task.abort();
return Err(ConsumerError::Closed);
}
inner.reader_task = Some(reader_task);
inner.writer_task = Some(writer_task);
}
Ok((generation, epoch))
}
async fn ensure_connected_for_call(
self: &Arc<Self>,
deadline: Instant,
) -> Result<(), CallError> {
loop {
if Instant::now() >= deadline {
return Err(CallError::not_sent(
"call deadline elapsed waiting for reconnection",
));
}
let notified = self.notify.notified();
tokio::pin!(notified);
let action = {
let mut inner = self.lock_inner();
notified.as_mut().enable();
if inner.closed {
return Err(CallError::not_sent("consumer closed"));
}
if inner.writer.is_some() {
return Ok(());
}
let generation = inner.generation;
let reconnect_is_live = match &inner.reconnect {
ReconnectState::Idle => false,
ReconnectState::Inline { generation: active } => *active == generation,
ReconnectState::Background {
generation: active,
task,
} => *active == generation && !task.is_finished(),
};
if reconnect_is_live {
EnsureAction::Wait
} else {
let stale_task =
match std::mem::replace(&mut inner.reconnect, ReconnectState::Idle) {
ReconnectState::Background { task, .. } => Some(task),
ReconnectState::Idle | ReconnectState::Inline { .. } => None,
};
inner.reconnect = ReconnectState::Inline { generation };
EnsureAction::Lead {
generation,
stale_task,
}
}
};
match action {
EnsureAction::Wait => {
#[cfg(test)]
{
let hook = self.before_wait.lock().unwrap().take();
if let Some(hook) = hook {
hook();
}
}
timeout_at(deadline, notified).await.map_err(|_| {
CallError::not_sent("call deadline elapsed waiting for reconnection")
})?;
}
EnsureAction::Lead {
generation,
stale_task,
} => {
if let Some(handle) = stale_task {
handle.abort();
}
let mut guard = InlineReconnectGuard::new(Arc::clone(self), generation);
let result = timeout_at(deadline, self.reconnect_with_retry(generation)).await;
guard.finish();
return match result {
Ok(Ok(())) => Ok(()),
Ok(Err(err)) => Err(CallError::not_sent(err.to_string())),
Err(_) => Err(CallError::not_sent(
"call deadline elapsed waiting for reconnection",
)),
};
}
}
}
}
fn spawn_reconnect(self: &Arc<Self>, generation: u64) -> bool {
let stale_task = {
let mut inner = self.lock_inner();
if inner.closed || inner.writer.is_some() || inner.generation != generation {
return false;
}
let should_spawn = match &inner.reconnect {
ReconnectState::Idle => true,
ReconnectState::Inline { generation: active } => *active < generation,
ReconnectState::Background {
generation: active,
task,
} => *active < generation || (*active == generation && task.is_finished()),
};
if !should_spawn {
return false;
}
let stale_task = match std::mem::replace(&mut inner.reconnect, ReconnectState::Idle) {
ReconnectState::Background { task, .. } => Some(task),
ReconnectState::Idle | ReconnectState::Inline { .. } => None,
};
let shared = Arc::clone(self);
let handle = tokio::spawn(async move {
let _result = shared.reconnect_with_retry(generation).await;
shared.finish_background_reconnect(generation);
});
inner.reconnect = ReconnectState::Background {
generation,
task: handle,
};
stale_task
};
if let Some(handle) = stale_task {
handle.abort();
}
true
}
async fn reconnect_with_retry(
self: &Arc<Self>,
reconnect_generation: u64,
) -> Result<(), ConsumerError> {
let mut last_error: Option<ConsumerError> = None;
for attempt in 1..=self.opts.reconnect_backoff.max_attempts {
if self.close_token.is_cancelled() {
return Err(ConsumerError::Closed);
}
if !self.reconnect_attempt_is_current(reconnect_generation) {
return Ok(());
}
match open_connection(&self.connection_file, self.opts.handshake_timeout).await {
Ok(opened) => {
if !self.reconnect_attempt_is_current(reconnect_generation) {
return Ok(());
}
let (generation, epoch) = self.install_reconnected(opened)?;
self.schedule_restored(generation, epoch);
return Ok(());
}
Err(err) => {
let transient = is_reconnect_transient(&err);
last_error = Some(err);
if !transient || attempt >= self.opts.reconnect_backoff.max_attempts {
break;
}
let delay =
self.jittered(self.opts.reconnect_backoff.delay_after_attempt(attempt));
tokio::select! {
() = self.close_token.cancelled() => return Err(ConsumerError::Closed),
() = sleep(delay) => {}
}
}
}
}
Err(last_error.unwrap_or(ConsumerError::Closed))
}
fn reconnect_attempt_is_current(&self, generation: u64) -> bool {
let inner = self.lock_inner();
!inner.closed
&& inner.writer.is_none()
&& inner.generation == generation
&& matches!(
&inner.reconnect,
ReconnectState::Inline { generation: active }
| ReconnectState::Background {
generation: active,
..
} if *active == generation
)
}
fn finish_inline_reconnect(&self, generation: u64) {
let finished = {
let mut inner = self.lock_inner();
if matches!(
&inner.reconnect,
ReconnectState::Inline { generation: active } if *active == generation
) {
inner.reconnect = ReconnectState::Idle;
true
} else {
false
}
};
if finished {
self.notify.notify_waiters();
}
}
fn finish_background_reconnect(&self, generation: u64) {
let completed_task = {
let mut inner = self.lock_inner();
if !matches!(
&inner.reconnect,
ReconnectState::Background { generation: active, .. } if *active == generation
) {
None
} else {
match std::mem::replace(&mut inner.reconnect, ReconnectState::Idle) {
ReconnectState::Background { task, .. } => Some(task),
ReconnectState::Idle | ReconnectState::Inline { .. } => unreachable!(),
}
}
};
if completed_task.is_some() {
drop(completed_task);
self.notify.notify_waiters();
}
}
fn schedule_restored(self: &Arc<Self>, generation: u64, epoch: u64) {
let token = {
let mut inner = self.lock_inner();
inner.restored_token = inner.restored_token.saturating_add(1);
inner.restored_token
};
let shared = Arc::clone(self);
tokio::spawn(async move {
tokio::select! {
() = shared.close_token.cancelled() => {}
() = sleep(shared.opts.restored_debounce) => {
let should_emit = {
let inner = shared.lock_inner();
!inner.closed
&& inner.generation == generation
&& inner.epoch == epoch
&& inner.restored_token == token
&& inner.writer.is_some()
};
if should_emit {
shared.emit_connection_state(ConnectionState::Restored { epoch });
}
}
}
});
}
fn install_one_shot_route(&self, route: RouteState) -> Result<(), CallError> {
let handle = route.handle;
let mut inner = self.lock_inner();
if inner.closed || inner.generation != handle.connection_token() || inner.writer.is_none() {
return Err(CallError::StaleRouteHandle(handle));
}
if inner
.route_epochs
.get(&handle.channel)
.is_some_and(|existing| *existing != handle)
{
return Err(CallError::not_sent(
"daemon returned a route channel already in use",
));
}
inner.one_shot_routes.insert(handle.channel, route);
inner.route_epochs.insert(handle.channel, handle);
Ok(())
}
async fn ensure_route(
self: &Arc<Self>,
key: &RouteKey,
route_open: &RouteOpenParams<'_>,
opts: &CallOptions,
call_deadline: Instant,
) -> Result<RouteState, CallError> {
loop {
let action = {
let mut inner = self.lock_inner();
if inner.closed {
return Err(CallError::not_sent("consumer closed"));
}
if let Some(route) = inner.routes.get(key) {
if route.handle.connection_token() == inner.generation && inner.writer.is_some()
{
return Ok(route.clone());
}
}
if let Some(opening) = inner.openings.get_mut(key) {
let (tx, rx) = oneshot::channel();
opening.waiters.push(tx);
RouteOpenAction::Wait(rx)
} else {
inner.openings.insert(
key.clone(),
Opening {
waiters: Vec::new(),
closed: false,
},
);
RouteOpenAction::Lead
}
};
match action {
RouteOpenAction::Wait(rx) => match timeout_at(call_deadline, rx).await {
Ok(Ok(Ok(route))) => return Ok(route),
Ok(Ok(Err(err))) => return Err(err.into_call_error()),
Ok(Err(_)) => continue,
Err(_) => {
return Err(CallError::not_sent(
"call deadline elapsed waiting for route.open",
));
}
},
RouteOpenAction::Lead => {
let mut guard = OpeningGuard::new(Arc::clone(self), key.clone());
let result = self
.open_route_with_retry(key, route_open, opts, call_deadline)
.await
.map_err(SharedCallFailure::from);
guard.finish(result.clone());
return result.map_err(SharedCallFailure::into_call_error);
}
}
}
}
async fn open_route_with_retry(
self: &Arc<Self>,
key: &RouteKey,
route_open: &RouteOpenParams<'_>,
opts: &CallOptions,
call_deadline: Instant,
) -> Result<RouteState, CallError> {
let route_deadline = (Instant::now() + opts.route_retry_deadline).min(call_deadline);
let mut attempt = 0usize;
let mut refusals = RouteOpenRefusals::default();
let expired = |err: CallError, refusals: &mut RouteOpenRefusals| {
if err.is_not_sent() && Instant::now() >= route_deadline {
if let Some(refused) = std::mem::take(refusals).into_error(key.target_label()) {
return refused;
}
}
err
};
loop {
attempt = attempt.saturating_add(1);
let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
target: route_open.target.clone(),
identity: route_open.identity.clone(),
consumer_identity: route_open.consumer_identity.clone(),
consumer_capabilities: route_open.consumer_capabilities.clone(),
admission_facts: None,
scope: None,
})
.map_err(|err| CallError::not_sent(format!("failed to encode route.open: {err}")))?;
match self
.paced_route_open(
body,
route_deadline,
Some(route_open.reverse_requests.clone()),
)
.await
{
Ok(TerminalFrame::Response {
generation, body, ..
}) => {
let response =
serde_json::from_slice::<ClientControlResponse>(&body).map_err(|err| {
CallError::not_sent(format!(
"failed to decode route.open response: {err}"
))
})?;
let ClientControlResponse::RouteOpen {
route_channel,
route_epoch,
} = response
else {
return Err(CallError::not_sent(
"route.open returned an unexpected control response",
));
};
let handle = self
.ingress_handle(generation, route_channel, route_epoch)
.unwrap_or_else(|| {
RouteHandle::new_consumer(
route_channel,
route_epoch,
generation,
route_open.reverse_requests.clone(),
)
});
let route = RouteState {
handle,
sem: Arc::new(Semaphore::new(DEFAULT_ROUTE_WINDOW)),
};
let install = {
let mut inner = self.lock_inner();
if inner.closed {
return Err(CallError::not_sent("consumer closed"));
}
let closed_during_open = inner.openings.get(key).is_some_and(|o| o.closed);
if closed_during_open
|| inner.generation != generation
|| inner.writer.is_none()
{
RouteInstall::Discard {
closed: closed_during_open,
}
} else {
let cached = inner.cache_route(key.clone(), route.clone());
RouteInstall::Cached(cached)
}
};
match install {
RouteInstall::Cached(cached) => return Ok(cached),
RouteInstall::Discard { closed } => {
if closed {
self.send_route_goodbye(route.handle, true);
self.uninstall_route_handle(route.handle);
return Err(CallError::not_sent(
"route was closed before route.open completed",
));
}
}
}
self.sleep_until_retry(route_deadline, self.jittered(opts.route_retry.base))
.await?;
}
Ok(TerminalFrame::Error { body, .. }) => {
if !is_retryable_route_open_code(&body.code) {
return Err(CallError::route_open_refused(key.target_label(), body));
}
refusals.record(body);
if Instant::now() >= route_deadline {
return Err(std::mem::take(&mut refusals)
.into_error(key.target_label())
.unwrap_or_else(|| {
CallError::not_sent("route.open retry deadline elapsed")
}));
}
let delay = self.jittered(opts.route_retry.delay_after_attempt(attempt));
if let Err(err) = self.sleep_until_retry(route_deadline, delay).await {
return Err(expired(err, &mut refusals));
}
continue;
}
Ok(TerminalFrame::StreamEnd) => {
return Err(CallError::not_sent("route.open returned StreamEnd"));
}
Err(err) if err.is_not_sent() && Instant::now() < route_deadline => {
let delay = self.jittered(opts.route_retry.delay_after_attempt(attempt));
if let Err(err) = self.sleep_until_retry(route_deadline, delay).await {
return Err(expired(err, &mut refusals));
}
}
Err(err) => return Err(expired(err, &mut refusals)),
}
}
#[allow(unreachable_code)]
Err(CallError::not_sent(format!(
"route.open retry deadline elapsed for target {}",
key.target_label()
)))
}
async fn sleep_until_retry(&self, deadline: Instant, delay: Duration) -> Result<(), CallError> {
if Instant::now() >= deadline {
return Err(CallError::not_sent("retry deadline elapsed"));
}
let bounded = delay.min(deadline.saturating_duration_since(Instant::now()));
tokio::select! {
() = self.close_token.cancelled() => Err(CallError::not_sent("consumer closed")),
() = sleep(bounded) => Ok(()),
}
}
async fn control_call(
self: &Arc<Self>,
body: Vec<u8>,
deadline: Instant,
retain_late_route_open: bool,
route_open_reverse_requests: Option<ReverseRequestRegistry>,
) -> Result<TerminalFrame, CallError> {
self.ensure_connected_for_call(deadline).await?;
self.send_request(RequestSend {
expected_handle: None,
channel: 0,
epoch: 0,
body,
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline,
retain_late_route_open,
route_open_reverse_requests,
})
.await
}
async fn send_request(
self: &Arc<Self>,
request: RequestSend,
) -> Result<TerminalFrame, CallError> {
let RequestSend {
expected_handle,
channel,
epoch,
body,
priority,
admission_class,
deadline,
retain_late_route_open,
route_open_reverse_requests,
} = request;
if Instant::now() >= deadline {
return Err(CallError::not_sent(
"call deadline elapsed before request was sent",
));
}
let (generation, corr, writer) = {
let mut inner = self.lock_inner();
if inner.closed {
return Err(CallError::not_sent("consumer closed"));
}
let generation = inner.generation;
if let Some(expected) = expected_handle {
let route_pair_matches =
channel == 0 || (expected.channel == channel && expected.epoch == epoch);
if expected.connection_token() != generation
|| !route_pair_matches
|| inner.route_epochs.get(&expected.channel) != Some(&expected)
{
return Err(CallError::StaleRouteHandle(expected));
}
}
let Some(writer) = inner.writer.clone() else {
return Err(CallError::not_sent("subc connection is down before send"));
};
let Some(corr) = inner.next_corr else {
drop(inner);
self.handle_generation_drop(
generation,
"channel-0 correlation allocator exhausted".to_string(),
);
return Err(CallError::not_sent(
"correlation allocator exhausted; connection closed",
));
};
inner.next_corr = corr.checked_add(1);
(generation, corr, writer)
};
let frame = Frame::build(
FrameType::Request,
Flags::new(false, priority, false).with_admission_class(admission_class),
channel,
epoch,
corr,
body,
)
.map_err(|err| CallError::not_sent(format!("failed to build request frame: {err}")))?;
let key = PendingKey {
generation,
channel,
epoch,
corr,
};
let (tx, rx) = oneshot::channel();
{
let mut inner = self.lock_inner();
if inner.closed || inner.generation != generation || inner.writer.is_none() {
return Err(CallError::not_sent(
"connection changed before request registration",
));
}
if let Some(expected) = expected_handle {
if inner.route_epochs.get(&expected.channel) != Some(&expected) {
return Err(CallError::StaleRouteHandle(expected));
}
}
let expected_control_handle = (channel == 0 && !retain_late_route_open)
.then_some(expected_handle)
.flatten();
inner.pending.insert(
key,
PendingEntry::unary(
tx,
retain_late_route_open,
expected_control_handle,
route_open_reverse_requests,
),
);
}
let mut registration =
PendingRegistration::new(Arc::clone(self), key, retain_late_route_open);
match timeout_at(
deadline,
writer.send(WriteCommand {
frame,
pending: Some(key),
}),
)
.await
{
Ok(Ok(())) => {}
Ok(Err(_)) => {
let accepted = registration.remove_pending().unwrap_or(false);
return Err(classify_failure(
accepted,
"writer task closed before accepting request",
));
}
Err(_) => {
let _ = registration.remove_pending();
return Err(CallError::not_sent(
"call deadline elapsed waiting for writer capacity",
));
}
}
tokio::select! {
result = timeout_at(deadline, rx) => match result {
Ok(Ok(result)) => {
registration.disarm();
result.into_call_result()
}
Ok(Err(_)) => {
registration.disarm();
Err(CallError::not_sent("pending response channel closed"))
}
Err(_) => {
let accepted = if retain_late_route_open {
registration.disarm();
self.pending_accepted(key).unwrap_or(false)
} else {
registration.remove_pending().unwrap_or(false)
};
if accepted {
self.spawn_liveness_probe();
}
Err(classify_failure(
accepted,
format!("request on channel {channel} timed out at its deadline"),
))
}
},
() = self.close_token.cancelled() => {
let accepted = registration.remove_pending().unwrap_or(false);
Err(classify_failure(accepted, "consumer closed while request was pending"))
}
}
}
fn spawn_liveness_probe(self: &Arc<Self>) {
if self.liveness_probe_running.swap(true, Ordering::AcqRel) {
return;
}
let allocation = {
let mut inner = self.lock_inner();
let control_pending = inner
.pending
.keys()
.any(|key| key.generation == inner.generation && key.channel == 0);
if inner.closed || control_pending || !matches!(&inner.reconnect, ReconnectState::Idle)
{
None
} else {
match (inner.next_corr, inner.writer.clone()) {
(Some(corr), Some(writer)) => {
inner.next_corr = corr.checked_add(1);
Some((inner.generation, corr, writer))
}
(None, _) | (_, None) => None,
}
}
};
let Some((generation, corr, writer)) = allocation else {
self.liveness_probe_running.store(false, Ordering::Release);
return;
};
let t0 = epoch_millis();
let ping = match Frame::build(
FrameType::Ping,
Flags::new(false, Priority::Interactive, false),
0,
0,
corr,
Vec::new(),
) {
Ok(ping) => ping,
Err(_) => {
self.liveness_probe_running.store(false, Ordering::Release);
return;
}
};
let window = self.opts.liveness_probe_window;
let shared = Arc::clone(self);
tokio::spawn(async move {
let window_end = Instant::now() + window;
let _ = timeout_at(
window_end,
writer.send(WriteCommand {
frame: ping,
pending: None,
}),
)
.await;
sleep_until(window_end).await;
let control_pending_now = {
let inner = shared.lock_inner();
inner
.pending
.keys()
.any(|key| key.generation == generation && key.channel == 0)
};
if !shared.close_token.is_cancelled()
&& !control_pending_now && shared.generation_is_current(generation)
&& shared.last_inbound_ms.load(Ordering::Acquire) < t0
{
shared.handle_generation_drop(
generation,
format!(
"liveness probe convicted a half-open socket: no inbound frame for {}ms after a channel-0 Ping",
window.as_millis()
),
);
}
shared
.liveness_probe_running
.store(false, Ordering::Release);
});
}
async fn send_subscription(
self: &Arc<Self>,
subscription: SubscriptionSend,
) -> Result<Subscription, CallError> {
let SubscriptionSend {
expected_handle,
channel,
epoch,
body,
priority,
admission_class,
event_buffer,
deadline,
permit,
} = subscription;
if Instant::now() >= deadline {
return Err(CallError::not_sent(
"subscription deadline elapsed before request was sent",
));
}
let (generation, corr, writer) = {
let mut inner = self.lock_inner();
if inner.closed {
return Err(CallError::not_sent("consumer closed"));
}
let generation = inner.generation;
if let Some(expected) = expected_handle {
if expected.connection_token() != generation
|| expected.channel != channel
|| expected.epoch != epoch
|| inner.route_epochs.get(&channel) != Some(&expected)
{
return Err(CallError::StaleRouteHandle(expected));
}
}
let Some(writer) = inner.writer.clone() else {
return Err(CallError::not_sent("subc connection is down before send"));
};
let Some(corr) = inner.next_corr else {
drop(inner);
self.handle_generation_drop(
generation,
"correlation allocator exhausted".to_string(),
);
return Err(CallError::not_sent(
"correlation allocator exhausted; connection closed",
));
};
inner.next_corr = corr.checked_add(1);
(generation, corr, writer)
};
let frame = Frame::build(
FrameType::Request,
Flags(
Flags::new(false, priority, false)
.with_admission_class(admission_class)
.0,
),
channel,
epoch,
corr,
body,
)
.map_err(|err| CallError::not_sent(format!("failed to build request frame: {err}")))?;
let key = PendingKey {
generation,
channel,
epoch,
corr,
};
let (events_tx, events_rx) = mpsc::channel(event_buffer.max(1));
let (closed_tx, closed_rx) = oneshot::channel();
{
let mut inner = self.lock_inner();
if inner.closed || inner.generation != generation || inner.writer.is_none() {
return Err(CallError::not_sent(
"connection changed before subscription registration",
));
}
if let Some(expected) = expected_handle {
if inner.route_epochs.get(&expected.channel) != Some(&expected) {
return Err(CallError::StaleRouteHandle(expected));
}
}
inner.pending.insert(
key,
PendingEntry::subscription(events_tx, closed_tx, permit, priority),
);
}
let mut registration = PendingRegistration::new(Arc::clone(self), key, false);
match timeout_at(
deadline,
writer.send(WriteCommand {
frame,
pending: Some(key),
}),
)
.await
{
Ok(Ok(())) => {}
Ok(Err(_)) => {
let accepted = registration.remove_pending().unwrap_or(false);
return Err(classify_failure(
accepted,
"writer task closed before accepting subscription request",
));
}
Err(_) => {
let _ = registration.remove_pending();
return Err(CallError::not_sent(
"subscription deadline elapsed waiting for writer capacity",
));
}
}
registration.disarm();
Ok(Subscription {
events: events_rx,
closed: SubscriptionClosed { rx: closed_rx },
cancel: SubscriptionCancel::new(Arc::clone(self), key, priority),
})
}
fn unsubscribe_subscription(
&self,
key: PendingKey,
priority: Priority,
) -> Result<(), CallError> {
let handle = RouteHandle::new(key.channel, key.epoch, key.generation);
self.validate_current_handle(handle)?;
let entry = self.lock_inner().pending.remove(&key);
if let Some(entry) = entry {
entry.settle_subscription_result(Ok(()));
self.send_cancel(handle, key.corr, priority);
}
Ok(())
}
fn route_stream_data(&self, key: PendingKey, body: Vec<u8>) {
let overflow = {
let mut inner = self.lock_inner();
let Some(entry) = inner.pending.get(&key) else {
return;
};
match entry.try_send_stream_data(body) {
Ok(()) | Err(StreamDataDelivery::NotSubscription) => return,
Err(StreamDataDelivery::Full) => {
let priority = entry
.subscription_priority()
.unwrap_or(Priority::Interactive);
let entry = inner.pending.remove(&key);
entry.map(|entry| {
(
entry,
priority,
"subscription event channel filled; reader dropped the stream instead of blocking",
)
})
}
Err(StreamDataDelivery::Closed) => {
let priority = entry
.subscription_priority()
.unwrap_or(Priority::Interactive);
let entry = inner.pending.remove(&key);
entry.map(|entry| {
(
entry,
priority,
"subscription event receiver closed before the stream ended",
)
})
}
}
};
if let Some((entry, priority, reason)) = overflow {
entry.settle_call_error(CallError::subscription_backpressure(reason));
self.send_cancel(
RouteHandle::new(key.channel, key.epoch, key.generation),
key.corr,
priority,
);
}
}
fn register_push_events(
&self,
handle: RouteHandle,
) -> Result<mpsc::Receiver<PushEvent>, CallError> {
let (events_tx, events_rx) = mpsc::channel(DEFAULT_PUSH_EVENT_BUFFER);
let mut inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| inner.writer.is_none()
|| inner.route_epochs.get(&handle.channel) != Some(&handle)
{
return Err(CallError::StaleRouteHandle(handle));
}
inner.push_event_receivers.insert(handle, events_tx);
Ok(events_rx)
}
fn route_push(&self, handle: RouteHandle, body: Vec<u8>) {
let should_count_drop = {
let mut inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| inner.route_epochs.get(&handle.channel) != Some(&handle)
{
return;
}
match inner.push_event_receivers.get(&handle) {
None => DroppedPush::NoReceiver,
Some(events) => match events.try_send(PushEvent { handle, body }) {
Ok(()) => DroppedPush::Delivered,
Err(mpsc::error::TrySendError::Closed(_)) => {
inner.push_event_receivers.remove(&handle);
DroppedPush::NoReceiver
}
Err(mpsc::error::TrySendError::Full(_)) => DroppedPush::ReceiverFull,
},
}
};
match should_count_drop {
DroppedPush::Delivered => {}
DroppedPush::NoReceiver => {
self.pushes_dropped_no_receiver
.fetch_add(1, Ordering::Relaxed);
}
DroppedPush::ReceiverFull => {
self.pushes_dropped_receiver_full
.fetch_add(1, Ordering::Relaxed);
}
}
}
fn control_push(&self, body: &[u8]) {
let parsed = serde_json::from_slice::<serde_json::Value>(body)
.ok()
.and_then(|value| {
let op = value.get("op")?.as_str()?.to_string();
Some(ControlPush { op, body: value })
});
let delivered = match parsed {
None => false,
Some(push) => {
let mut inner = self.lock_inner();
match inner.control_push_receiver.as_ref() {
None => false,
Some(sender) => match sender.try_send(push) {
Ok(()) => true,
Err(mpsc::error::TrySendError::Closed(_)) => {
inner.control_push_receiver = None;
false
}
Err(mpsc::error::TrySendError::Full(_)) => false,
},
}
}
};
if !delivered {
self.control_pushes_dropped.fetch_add(1, Ordering::Relaxed);
}
}
fn send_cancel(&self, handle: RouteHandle, corr: u64, priority: Priority) {
let writer = {
let inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| !route_is_installed(&inner, handle)
{
return;
}
inner.writer.clone()
};
let Some(writer) = writer else {
return;
};
let Ok(frame) = Frame::build(
FrameType::Cancel,
Flags::new(false, priority, false),
handle.channel,
handle.epoch,
corr,
Vec::new(),
) else {
return;
};
let _ = writer.try_send(WriteCommand {
frame,
pending: None,
});
}
fn mark_pending_accepted(&self, key: PendingKey) -> bool {
let mut inner = self.lock_inner();
if inner.closed || inner.generation != key.generation {
return false;
}
let Some(entry) = inner.pending.get_mut(&key) else {
return false;
};
entry.accepted = true;
true
}
fn settle_pending(self: &Arc<Self>, key: PendingKey, terminal: PendingTerminal) {
let entry = self.lock_inner().pending.remove(&key);
let Some(entry) = entry else {
return;
};
if entry.retain_late_route_open && entry.completion_is_closed() {
if let PendingTerminal::Response { generation, body } = &terminal {
if let Ok(ClientControlResponse::RouteOpen {
route_channel,
route_epoch,
}) = serde_json::from_slice::<ClientControlResponse>(body)
{
let handle = RouteHandle::new(route_channel, route_epoch, *generation);
self.send_route_goodbye(handle, true);
self.uninstall_route_handle(handle);
}
}
return;
}
entry.settle_terminal(terminal);
}
fn pending_accepted(&self, key: PendingKey) -> Option<bool> {
self.lock_inner()
.pending
.get(&key)
.map(|entry| entry.accepted)
}
fn handle_generation_drop(self: &Arc<Self>, generation: u64, reason: String) {
let (should_emit, pending, openings, callbacks) = {
let mut inner = self.lock_inner();
if inner.closed || inner.generation != generation || inner.writer.is_none() {
return;
}
inner.writer = None;
inner.restored_token = inner.restored_token.saturating_add(1);
inner.close_routes();
inner.route_epochs.clear();
let pending = drain_pending_generation(&mut inner.pending, generation);
let openings = drain_openings(&mut inner.openings);
let callbacks = inner.callbacks.clone();
(true, pending, openings, callbacks)
};
if should_emit {
settle_pending_entries(pending, reason.clone());
fail_openings(openings, SharedCallFailure::not_sent(reason.clone()));
emit_callbacks(callbacks, ConnectionState::Dropped);
self.notify.notify_waiters();
let _ = self.spawn_reconnect(generation);
}
}
fn close_sync(&self, reason: &str) {
let (pending, openings, routes, reader, writer, reconnect) = {
let mut inner = self.lock_inner();
if inner.closed {
return;
}
inner.closed = true;
inner.writer = None;
inner.route_epochs.clear();
inner.push_event_receivers.clear();
self.close_token.cancel();
let reconnect = match std::mem::replace(&mut inner.reconnect, ReconnectState::Idle) {
ReconnectState::Background { task, .. } => Some(task),
ReconnectState::Idle | ReconnectState::Inline { .. } => None,
};
(
inner
.pending
.drain()
.map(|(_, entry)| entry)
.collect::<Vec<_>>(),
inner
.openings
.drain()
.map(|(_, opening)| opening.waiters)
.collect::<Vec<_>>(),
inner.drain_routes(),
inner.reader_task.take(),
inner.writer_task.take(),
reconnect,
)
};
for route in routes {
route.sem.close();
}
if let Some(handle) = reader {
handle.abort();
}
if let Some(handle) = writer {
handle.abort();
}
if let Some(handle) = reconnect {
handle.abort();
}
settle_pending_entries(pending, reason.to_string());
fail_openings(openings, SharedCallFailure::not_sent(reason.to_string()));
self.notify.notify_waiters();
}
fn validate_current_handle(&self, handle: RouteHandle) -> Result<(), CallError> {
let inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| inner.writer.is_none()
|| !route_is_installed(&inner, handle)
{
Err(CallError::StaleRouteHandle(handle))
} else {
Ok(())
}
}
fn route_state(&self, handle: RouteHandle) -> Result<RouteState, CallError> {
let inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| inner.writer.is_none()
|| inner.route_epochs.get(&handle.channel) != Some(&handle)
{
return Err(CallError::StaleRouteHandle(handle));
}
let route = inner
.route_by_channel
.get(&handle.channel)
.and_then(|key| inner.routes.get(key))
.or_else(|| inner.one_shot_routes.get(&handle.channel));
debug_assert!(route.is_none_or(|route| route.handle == handle));
route
.filter(|route| route.handle == handle)
.cloned()
.ok_or(CallError::StaleRouteHandle(handle))
}
fn route_is_current(&self, key: &RouteKey, route: &RouteState) -> bool {
let inner = self.lock_inner();
if inner.closed
|| inner.generation != route.handle.connection_token()
|| inner.writer.is_none()
{
return false;
}
inner.routes.get(key).is_some_and(|cached| {
cached.handle == route.handle && Arc::ptr_eq(&cached.sem, &route.sem)
})
}
fn invalidate_route(&self, key: &RouteKey, expected_handle: Option<RouteHandle>) {
let removed = {
let mut inner = self.lock_inner();
match inner.routes.get(key) {
Some(route) if expected_handle.is_none_or(|expected| expected == route.handle) => {
let removed = inner.remove_route(key);
if let Some(route) = &removed {
if inner.route_epochs.get(&route.handle.channel) == Some(&route.handle) {
inner.route_epochs.remove(&route.handle.channel);
}
inner.push_event_receivers.remove(&route.handle);
}
removed
}
_ => None,
}
};
if let Some(route) = removed {
route.sem.close();
}
}
fn finish_opening(&self, key: &RouteKey, result: Result<RouteState, SharedCallFailure>) {
let opening = self.lock_inner().openings.remove(key);
for waiter in opening.map(|o| o.waiters).unwrap_or_default() {
let _ = waiter.send(result.clone());
}
}
async fn close_handle(
self: &Arc<Self>,
handle: RouteHandle,
opts: &CloseRouteOptions,
) -> Result<(), CallError> {
self.validate_current_handle(handle)?;
let routes = {
let mut inner = self.lock_inner();
inner
.remove_route_by_handle(handle)
.into_iter()
.collect::<Vec<_>>()
};
if opts.drain {
self.drain_channel(handle, opts.drain_timeout).await;
}
for route in routes {
route.sem.close();
}
self.fail_channel_pending(handle, "route closed by close_handle");
self.send_route_goodbye(handle, false);
self.uninstall_route_handle(handle);
Ok(())
}
async fn close_route(self: &Arc<Self>, key: &RouteKey, opts: &CloseRouteOptions) {
let route = {
let mut inner = self.lock_inner();
if let Some(opening) = inner.openings.get_mut(key) {
opening.closed = true;
}
inner.remove_route(key)
};
let Some(route) = route else {
return;
};
if opts.drain {
self.drain_channel(route.handle, opts.drain_timeout).await;
}
route.sem.close();
self.fail_channel_pending(route.handle, "route closed by close_route");
self.send_route_goodbye(route.handle, false);
self.uninstall_route_handle(route.handle);
}
fn uninstall_route_handle(&self, handle: RouteHandle) {
let mut inner = self.lock_inner();
if inner.route_epochs.get(&handle.channel) == Some(&handle) {
inner.route_epochs.remove(&handle.channel);
inner.push_event_receivers.remove(&handle);
release_reverse_request_registry(handle);
}
}
fn fail_channel_pending(&self, handle: RouteHandle, reason: &str) {
let entries = {
let mut inner = self.lock_inner();
drain_pending_handle(&mut inner.pending, handle, true)
};
settle_pending_entries(entries, reason.to_string());
}
async fn drain_channel(&self, handle: RouteHandle, timeout: Duration) {
let deadline = Instant::now() + timeout;
loop {
let has_inflight = {
let inner = self.lock_inner();
inner.pending.iter().any(|(key, entry)| {
key.generation == handle.connection_token()
&& key.channel == handle.channel
&& key.epoch == handle.epoch
&& !entry.is_subscription()
})
};
if !has_inflight || Instant::now() >= deadline {
return;
}
sleep(Duration::from_millis(5)).await;
}
}
fn send_route_goodbye(self: &Arc<Self>, handle: RouteHandle, close_on_failure: bool) -> bool {
let writer = {
let inner = self.lock_inner();
if inner.closed
|| inner.generation != handle.connection_token()
|| inner.route_epochs.get(&handle.channel) != Some(&handle)
{
return false;
}
inner.writer.clone()
};
let Some(writer) = writer else {
return false;
};
let Ok(frame) = Frame::build(
FrameType::Goodbye,
Flags::new(false, Priority::Interactive, false),
handle.channel,
handle.epoch,
0,
Vec::new(),
) else {
return false;
};
if writer
.try_send(WriteCommand {
frame,
pending: None,
})
.is_ok()
{
return true;
}
if close_on_failure {
self.handle_generation_drop(
handle.connection_token(),
"failed to queue late route.open cleanup GOODBYE".to_string(),
);
}
false
}
fn emit_connection_state(&self, state: ConnectionState) {
let callbacks = self.lock_inner().callbacks.clone();
emit_callbacks(callbacks, state);
}
}
#[derive(Clone, Copy)]
enum InstallKind {
Initial,
Reconnect,
}
enum EnsureAction {
Wait,
Lead {
generation: u64,
stale_task: Option<JoinHandle<()>>,
},
}
enum ReconnectState {
Idle,
Inline {
generation: u64,
},
Background {
generation: u64,
task: JoinHandle<()>,
},
}
enum RouteInstall {
Cached(RouteState),
Discard { closed: bool },
}
enum RouteOpenAction {
Wait(oneshot::Receiver<Result<RouteState, SharedCallFailure>>),
Lead,
}
struct RouteOpenParams<'a> {
target: &'a RouteTarget,
identity: &'a BindIdentity,
consumer_identity: &'a Option<ConsumerIdentity>,
consumer_capabilities: &'a Option<Vec<String>>,
reverse_requests: &'a ReverseRequestRegistry,
}
struct RequestSend {
expected_handle: Option<RouteHandle>,
channel: u16,
epoch: u32,
body: Vec<u8>,
priority: Priority,
admission_class: AdmissionClass,
deadline: Instant,
retain_late_route_open: bool,
route_open_reverse_requests: Option<ReverseRequestRegistry>,
}
struct SubscriptionSend {
expected_handle: Option<RouteHandle>,
channel: u16,
epoch: u32,
body: Vec<u8>,
priority: Priority,
admission_class: AdmissionClass,
event_buffer: usize,
deadline: Instant,
permit: OwnedSemaphorePermit,
}
struct OpeningGuard {
shared: Arc<Shared>,
key: RouteKey,
finished: bool,
}
impl OpeningGuard {
fn new(shared: Arc<Shared>, key: RouteKey) -> Self {
Self {
shared,
key,
finished: false,
}
}
fn finish(&mut self, result: Result<RouteState, SharedCallFailure>) {
self.shared.finish_opening(&self.key, result);
self.finished = true;
}
}
impl Drop for OpeningGuard {
fn drop(&mut self) {
if !self.finished {
self.shared.finish_opening(
&self.key,
Err(SharedCallFailure::not_sent(
"route.open future was cancelled",
)),
);
}
}
}
struct InlineReconnectGuard {
shared: Arc<Shared>,
generation: u64,
finished: bool,
}
impl InlineReconnectGuard {
fn new(shared: Arc<Shared>, generation: u64) -> Self {
Self {
shared,
generation,
finished: false,
}
}
fn finish(&mut self) {
self.shared.finish_inline_reconnect(self.generation);
self.finished = true;
}
}
impl Drop for InlineReconnectGuard {
fn drop(&mut self) {
if !self.finished {
self.shared.finish_inline_reconnect(self.generation);
}
}
}
#[derive(Clone)]
struct RouteState {
handle: RouteHandle,
sem: Arc<Semaphore>,
}
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
struct RouteKey {
target: RouteTargetKey,
project_root: PathBuf,
harness: String,
session: String,
consumer_identity: Option<ConsumerIdentityKey>,
consumer_capabilities: Option<ConsumerCapabilitiesKey>,
}
impl RouteKey {
fn new(
target: &RouteTarget,
identity: &BindIdentity,
consumer_identity: Option<&ConsumerIdentity>,
consumer_capabilities: Option<&[String]>,
) -> Self {
Self {
target: RouteTargetKey::from(target),
project_root: identity.project_root.clone(),
harness: identity.harness.clone(),
session: identity.session.clone(),
consumer_identity: consumer_identity.map(ConsumerIdentityKey::from),
consumer_capabilities: consumer_capabilities.map(ConsumerCapabilitiesKey::from_slice),
}
}
fn target_label(&self) -> String {
match &self.target {
RouteTargetKey::ToolProvider { module_id } => format!("tool_provider:{module_id}"),
RouteTargetKey::ManagementSurface { module_id } => {
format!("management_surface:{module_id}")
}
RouteTargetKey::InternalService {
module_id,
service_id,
} => format!("internal_service:{module_id}:{service_id}"),
}
}
}
#[derive(Clone, Hash, PartialEq, Eq)]
struct ConsumerIdentityKey {
module_id: String,
launch_nonce: String,
}
impl fmt::Debug for ConsumerIdentityKey {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ConsumerIdentityKey")
.field("module_id", &self.module_id)
.field(
"launch_nonce",
&format_args!("<{} bytes redacted>", self.launch_nonce.len()),
)
.finish()
}
}
impl From<&ConsumerIdentity> for ConsumerIdentityKey {
fn from(value: &ConsumerIdentity) -> Self {
Self {
module_id: value.module_id.clone(),
launch_nonce: value.launch_nonce.clone(),
}
}
}
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
struct ConsumerCapabilitiesKey {
values: Vec<String>,
}
impl ConsumerCapabilitiesKey {
fn from_slice(values: &[String]) -> Self {
let mut values = values.to_vec();
values.sort();
values.dedup();
Self { values }
}
}
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
enum RouteTargetKey {
ToolProvider {
module_id: String,
},
ManagementSurface {
module_id: String,
},
InternalService {
module_id: String,
service_id: String,
},
}
impl From<&RouteTarget> for RouteTargetKey {
fn from(value: &RouteTarget) -> Self {
match value {
RouteTarget::ToolProvider { module_id } => Self::ToolProvider {
module_id: module_id.clone(),
},
RouteTarget::ManagementSurface { module_id } => Self::ManagementSurface {
module_id: module_id.clone(),
},
RouteTarget::InternalService {
module_id,
service_id,
} => Self::InternalService {
module_id: module_id.clone(),
service_id: service_id.clone(),
},
}
}
}
#[derive(Debug, Clone)]
struct SharedCallFailure {
kind: FailureKind,
message: String,
refusal: Option<RouteOpenRefused>,
}
impl SharedCallFailure {
fn not_sent(message: impl Into<String>) -> Self {
Self {
kind: FailureKind::NotSent,
message: message.into(),
refusal: None,
}
}
fn into_call_error(self) -> CallError {
match (self.kind, self.refusal) {
(FailureKind::NotSent, Some(refused)) => CallError::NotSent(Box::new(refused)),
(FailureKind::NotSent, None) => CallError::not_sent(self.message),
(FailureKind::OutcomeUnknown, _) => CallError::outcome_unknown(self.message),
}
}
}
impl From<CallError> for SharedCallFailure {
fn from(value: CallError) -> Self {
match value {
CallError::NotSent(err) => Self {
kind: FailureKind::NotSent,
message: err.to_string(),
refusal: err.downcast_ref::<RouteOpenRefused>().cloned(),
},
CallError::OutcomeUnknown(err) => Self {
kind: FailureKind::OutcomeUnknown,
message: err.to_string(),
refusal: None,
},
CallError::Module(body) => Self {
kind: FailureKind::OutcomeUnknown,
message: format!(
"unexpected module error during route.open: {} ({})",
body.code, body.message
),
refusal: None,
},
CallError::SubscriptionBackpressure(err) => Self {
kind: FailureKind::OutcomeUnknown,
message: err.to_string(),
refusal: None,
},
CallError::StaleRouteHandle(handle) => Self {
kind: FailureKind::NotSent,
message: format!("stale route handle: {handle:?}"),
refusal: None,
},
error @ (CallError::CapabilityUnprovided { .. }
| CallError::CapabilityAmbiguous { .. }
| CallError::InvalidCapabilityIdentifier { .. }) => Self {
kind: FailureKind::NotSent,
message: error.to_string(),
refusal: None,
},
}
}
}
#[derive(Debug, Clone, Copy)]
enum FailureKind {
NotSent,
OutcomeUnknown,
}
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
struct PendingKey {
generation: u64,
channel: u16,
epoch: u32,
corr: u64,
}
struct PendingEntry {
accepted: bool,
retain_late_route_open: bool,
expected_control_handle: Option<RouteHandle>,
route_open_reverse_requests: Option<ReverseRequestRegistry>,
completion: PendingCompletion,
}
enum PendingCompletion {
Unary(oneshot::Sender<PendingResult>),
Subscription {
events: mpsc::Sender<Vec<u8>>,
closed: oneshot::Sender<Result<(), CallError>>,
_permit: OwnedSemaphorePermit,
priority: Priority,
},
}
enum StreamDataDelivery {
NotSubscription,
Full,
Closed,
}
impl PendingEntry {
fn unary(
tx: oneshot::Sender<PendingResult>,
retain_late_route_open: bool,
expected_control_handle: Option<RouteHandle>,
route_open_reverse_requests: Option<ReverseRequestRegistry>,
) -> Self {
Self {
accepted: false,
retain_late_route_open,
expected_control_handle,
route_open_reverse_requests,
completion: PendingCompletion::Unary(tx),
}
}
fn subscription(
events: mpsc::Sender<Vec<u8>>,
closed: oneshot::Sender<Result<(), CallError>>,
permit: OwnedSemaphorePermit,
priority: Priority,
) -> Self {
Self {
accepted: false,
retain_late_route_open: false,
expected_control_handle: None,
route_open_reverse_requests: None,
completion: PendingCompletion::Subscription {
events,
closed,
_permit: permit,
priority,
},
}
}
fn completion_is_closed(&self) -> bool {
match &self.completion {
PendingCompletion::Unary(tx) => tx.is_closed(),
PendingCompletion::Subscription { closed, .. } => closed.is_closed(),
}
}
fn is_subscription(&self) -> bool {
matches!(&self.completion, PendingCompletion::Subscription { .. })
}
fn subscription_priority(&self) -> Option<Priority> {
match &self.completion {
PendingCompletion::Subscription { priority, .. } => Some(*priority),
PendingCompletion::Unary(_) => None,
}
}
fn try_send_stream_data(&self, body: Vec<u8>) -> Result<(), StreamDataDelivery> {
let PendingCompletion::Subscription { events, .. } = &self.completion else {
return Err(StreamDataDelivery::NotSubscription);
};
events.try_send(body).map_err(|err| match err {
mpsc::error::TrySendError::Full(_) => StreamDataDelivery::Full,
mpsc::error::TrySendError::Closed(_) => StreamDataDelivery::Closed,
})
}
fn settle_terminal(self, terminal: PendingTerminal) {
match self.completion {
PendingCompletion::Unary(tx) => {
let _ = tx.send(PendingResult::Terminal(terminal));
}
PendingCompletion::Subscription { closed, .. } => {
let result = match terminal {
PendingTerminal::Response { .. } | PendingTerminal::StreamEnd => Ok(()),
PendingTerminal::Error { body, .. } => Err(CallError::Module(body)),
};
let _ = closed.send(result);
}
}
}
fn settle_failure(self, reason: String) {
let accepted = self.accepted;
self.settle_call_error(classify_failure(accepted, reason));
}
fn settle_call_error(self, err: CallError) {
match self.completion {
PendingCompletion::Unary(tx) => {
let _ = tx.send(PendingResult::Failure {
accepted: self.accepted,
reason: err.to_string(),
});
}
PendingCompletion::Subscription { closed, .. } => {
let _ = closed.send(Err(err));
}
}
}
fn settle_subscription_result(self, result: Result<(), CallError>) {
match self.completion {
PendingCompletion::Subscription { closed, .. } => {
let _ = closed.send(result);
}
PendingCompletion::Unary(tx) => {
let _ = tx.send(PendingResult::Failure {
accepted: self.accepted,
reason: "subscription cancel matched a unary request".to_string(),
});
}
}
}
}
struct PendingRegistration {
shared: Arc<Shared>,
key: PendingKey,
active: bool,
retain_on_drop: bool,
}
impl PendingRegistration {
fn new(shared: Arc<Shared>, key: PendingKey, retain_on_drop: bool) -> Self {
Self {
shared,
key,
active: true,
retain_on_drop,
}
}
fn remove_pending(&mut self) -> Option<bool> {
if !self.active {
return None;
}
self.active = false;
self.shared
.lock_inner()
.pending
.remove(&self.key)
.map(|entry| entry.accepted)
}
fn disarm(&mut self) {
self.active = false;
}
}
impl Drop for PendingRegistration {
fn drop(&mut self) {
if self.retain_on_drop {
self.disarm();
} else {
let _ = self.remove_pending();
}
}
}
enum PendingResult {
Terminal(PendingTerminal),
Failure { accepted: bool, reason: String },
}
impl PendingResult {
fn into_call_result(self) -> Result<TerminalFrame, CallError> {
match self {
Self::Terminal(terminal) => Ok(terminal.into_terminal_frame()),
Self::Failure { accepted, reason } => Err(classify_failure(accepted, reason)),
}
}
}
enum PendingTerminal {
Response { generation: u64, body: Vec<u8> },
Error { body: ErrorBody, flags: Flags },
StreamEnd,
}
impl PendingTerminal {
fn into_terminal_frame(self) -> TerminalFrame {
match self {
Self::Response { generation, body } => TerminalFrame::Response { generation, body },
Self::Error { body, flags } => TerminalFrame::Error { body, flags },
Self::StreamEnd => TerminalFrame::StreamEnd,
}
}
}
#[derive(Debug)]
enum TerminalFrame {
Response { generation: u64, body: Vec<u8> },
Error { body: ErrorBody, flags: Flags },
StreamEnd,
}
struct WriteCommand {
frame: Frame,
pending: Option<PendingKey>,
}
struct OpenedConnection {
stream: TcpStream,
}
async fn open_connection(
path: &Path,
deadline: Duration,
) -> Result<OpenedConnection, ConsumerError> {
let conn =
connection_file::read_for_client(path).map_err(|source| ConsumerError::ConnectionFile {
path: path.to_path_buf(),
source,
})?;
open_connection_with_info(path, &conn, deadline).await
}
async fn open_connection_with_info(
path: &Path,
conn: &connection_file::ConnectionInfo,
deadline: Duration,
) -> Result<OpenedConnection, ConsumerError> {
let endpoint = conn
.endpoints
.first()
.ok_or_else(|| ConsumerError::NoEndpoint {
path: path.to_path_buf(),
})?;
let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
let mut stream = TcpStream::connect(&endpoint_label)
.await
.map_err(|source| ConsumerError::Connect {
path: path.to_path_buf(),
endpoint: endpoint_label.clone(),
source,
})?;
let _ = stream.set_nodelay(true);
authenticate_client(&mut stream, conn, deadline)
.await
.map_err(|source| ConsumerError::Auth {
path: path.to_path_buf(),
endpoint: endpoint_label,
source,
})?;
Ok(OpenedConnection { stream })
}
fn dispatch_reverse_request(shared: &Arc<Shared>, frame: Frame, handle: RouteHandle) {
let method = serde_json::from_slice::<serde_json::Value>(&frame.body)
.ok()
.and_then(|body| body.get("method")?.as_str().map(str::to_string));
let method_family = method
.as_deref()
.and_then(|method| method.split('/').next())
.filter(|family| !family.is_empty())
.map(str::to_string);
let handler = method_family.as_deref().and_then(|family| {
route_reverse_request_registry(handle.reverse_request_registry_id())
.ok()?
.handler(family)
});
let generation = handle.connection_token();
let shared = Arc::clone(shared);
tokio::spawn(async move {
let (frame_type, body) = if let (Some(handler), Some(method)) = (handler, method) {
let request_body = frame.body.clone();
let corr = frame.header.corr;
match tokio::spawn(async move {
handler(request_body, ReverseRequestContext { corr, method }).await
})
.await
{
Ok(Ok(body)) => (FrameType::Response, body),
Ok(Err(err)) => (
FrameType::Error,
serde_json::to_vec(&ErrorBody::new(REVERSE_REQUEST_UNHANDLED, err.to_string()))
.unwrap_or_default(),
),
Err(err) => (
FrameType::Error,
serde_json::to_vec(&ErrorBody::new(
REVERSE_REQUEST_UNHANDLED,
join_error_message(err),
))
.unwrap_or_default(),
),
}
} else {
let message = method_family.map_or_else(
|| "reverse request has no valid method".to_string(),
|family| format!("no reverse-request handler is registered for {family}"),
);
(
FrameType::Error,
serde_json::to_vec(&ErrorBody::new(REVERSE_REQUEST_UNHANDLED, message))
.unwrap_or_default(),
)
};
if shared.ingress_handle(generation, handle.channel, handle.epoch) != Some(handle) {
return;
}
let reply = match Frame::build_with_version(
frame.header.ver,
frame_type,
Flags::new(false, Priority::Interactive, false),
handle.channel,
handle.epoch,
frame.header.corr,
body,
) {
Ok(reply) => reply,
Err(_) => return,
};
let writer = shared.lock_inner().writer.clone();
if let Some(writer) = writer {
let _ = writer
.send(WriteCommand {
frame: reply,
pending: None,
})
.await;
}
});
}
fn join_error_message(error: tokio::task::JoinError) -> String {
if !error.is_panic() {
return error.to_string();
}
let payload = error.into_panic();
if let Some(message) = payload.downcast_ref::<String>() {
return message.clone();
}
if let Some(message) = payload.downcast_ref::<&str>() {
return (*message).to_string();
}
"reverse-request handler panicked".to_string()
}
async fn reader_loop(shared: Arc<Shared>, mut reader: OwnedReadHalf, generation: u64) {
loop {
match read_frame(&mut reader).await {
Ok(Some(frame)) => {
if !dispatch_frame(&shared, generation, frame).await {
return;
}
}
Ok(None) => {
shared.handle_generation_drop(generation, "subc connection closed".to_string());
return;
}
Err(err) => {
shared.handle_generation_drop(generation, err.to_string());
return;
}
}
}
}
async fn dispatch_frame(shared: &Arc<Shared>, generation: u64, frame: Frame) -> bool {
if !shared.record_inbound_if_current(generation) {
return false;
}
if frame.header.channel != 0
&& !shared.validate_ingress_handle(generation, frame.header.channel, frame.header.epoch)
{
return true;
}
if frame.header.ty == FrameType::Request && frame.header.channel != 0 {
if let Some(handle) =
shared.ingress_handle(generation, frame.header.channel, frame.header.epoch)
{
dispatch_reverse_request(shared, frame, handle);
}
return true;
}
let key = PendingKey {
generation,
channel: frame.header.channel,
epoch: frame.header.epoch,
corr: frame.header.corr,
};
if frame.header.channel == 0 && frame.header.ty == FrameType::Response {
if let Some(expected) = shared.pending_expected_control_handle(key) {
let echoes_expected = matches!(
serde_json::from_slice::<ClientControlResponse>(&frame.body),
Ok(ClientControlResponse::RoutePoll {
route_channel,
route_epoch,
..
}) if route_channel == expected.channel && route_epoch == expected.epoch
);
if !echoes_expected {
shared.count_dropped_route_frame();
return true;
}
}
}
if frame.header.channel == 0
&& frame.header.ty == FrameType::Response
&& shared.pending_expects_route_open(key)
{
if let Ok(ClientControlResponse::RouteOpen {
route_channel,
route_epoch,
}) = serde_json::from_slice::<ClientControlResponse>(&frame.body)
{
let reverse_requests = shared
.pending_route_open_reverse_requests(key)
.unwrap_or_default();
shared.install_ingress_handle(RouteHandle::new_consumer(
route_channel,
route_epoch,
generation,
reverse_requests,
));
}
}
match frame.header.ty {
FrameType::Response => shared.settle_pending(
key,
PendingTerminal::Response {
generation,
body: frame.body,
},
),
FrameType::Error => {
let body =
serde_json::from_slice::<ErrorBody>(&frame.body).unwrap_or_else(|err| ErrorBody {
code: "invalid_error_body".to_string(),
message: err.to_string(),
detail: None,
});
shared.settle_pending(
key,
PendingTerminal::Error {
body,
flags: frame.header.flags,
},
);
}
FrameType::StreamEnd => shared.settle_pending(key, PendingTerminal::StreamEnd),
FrameType::StreamData => shared.route_stream_data(key, frame.body),
FrameType::Push if frame.header.channel == 0 => {
shared.control_push(&frame.body);
}
FrameType::Push => shared.route_push(
RouteHandle::new(frame.header.channel, frame.header.epoch, generation),
frame.body,
),
FrameType::Goodbye if frame.header.channel == 0 => {
shared.handle_generation_drop(generation, "subc sent GOODBYE".to_string());
return false;
}
FrameType::Goodbye => {
let handle = RouteHandle::new(frame.header.channel, frame.header.epoch, generation);
shared.invalidate_routes_for_handle(handle);
let pending = {
let mut inner = shared.lock_inner();
drain_pending_handle(&mut inner.pending, handle, true)
};
settle_pending_entries(pending, "route closed by subc".to_string());
}
FrameType::Ping if frame.header.channel == 0 => {
if let Ok(pong) = Frame::build_with_version(
frame.header.ver,
FrameType::Pong,
frame.header.flags,
0,
0,
frame.header.corr,
Vec::new(),
) {
let writer = shared.lock_inner().writer.clone();
if let Some(writer) = writer {
let _ = writer
.send(WriteCommand {
frame: pong,
pending: None,
})
.await;
}
}
}
_ => {}
}
true
}
impl Shared {
fn record_inbound_if_current(&self, generation: u64) -> bool {
let inner = self.lock_inner();
if inner.closed || inner.generation != generation || inner.writer.is_none() {
return false;
}
self.last_inbound_ms
.store(epoch_millis(), Ordering::Release);
true
}
fn generation_is_current(&self, generation: u64) -> bool {
let inner = self.lock_inner();
!inner.closed && inner.generation == generation && inner.writer.is_some()
}
fn pending_expected_control_handle(&self, key: PendingKey) -> Option<RouteHandle> {
self.lock_inner()
.pending
.get(&key)
.and_then(|entry| entry.expected_control_handle)
}
fn count_dropped_route_frame(&self) {
let mut inner = self.lock_inner();
inner.dropped_route_frames = inner.dropped_route_frames.saturating_add(1);
}
fn pending_expects_route_open(&self, key: PendingKey) -> bool {
self.lock_inner()
.pending
.get(&key)
.is_some_and(|entry| entry.retain_late_route_open)
}
fn pending_route_open_reverse_requests(
&self,
key: PendingKey,
) -> Option<ReverseRequestRegistry> {
self.lock_inner()
.pending
.get(&key)
.and_then(|entry| entry.route_open_reverse_requests.clone())
}
fn ingress_handle(&self, generation: u64, channel: u16, epoch: u32) -> Option<RouteHandle> {
let inner = self.lock_inner();
let handle = inner.route_epochs.get(&channel).copied()?;
(handle.connection_token() == generation && handle.epoch == epoch).then_some(handle)
}
fn validate_ingress_handle(&self, generation: u64, channel: u16, epoch: u32) -> bool {
let mut inner = self.lock_inner();
let expected = RouteHandle::new(channel, epoch, generation);
if inner.route_epochs.get(&channel) == Some(&expected) {
true
} else {
inner.dropped_route_frames = inner.dropped_route_frames.saturating_add(1);
false
}
}
fn install_ingress_handle(&self, handle: RouteHandle) {
let mut inner = self.lock_inner();
if !inner.closed && inner.generation == handle.connection_token() && inner.writer.is_some()
{
if let Some(previous) = inner.route_epochs.insert(handle.channel, handle) {
if previous.reverse_request_registry_id() != handle.reverse_request_registry_id() {
release_reverse_request_registry(previous);
}
}
}
}
fn invalidate_routes_for_handle(&self, handle: RouteHandle) {
let removed = {
let mut inner = self.lock_inner();
if inner.route_epochs.get(&handle.channel) != Some(&handle) {
return;
}
inner.route_epochs.remove(&handle.channel);
inner.push_event_receivers.remove(&handle);
inner
.remove_route_by_handle(handle)
.into_iter()
.collect::<Vec<_>>()
};
for route in removed {
route.sem.close();
}
}
}
async fn writer_loop<W>(
shared: Arc<Shared>,
writer: W,
mut rx: mpsc::Receiver<WriteCommand>,
generation: u64,
) where
W: AsyncWrite + Unpin,
{
let mut writer = BufWriter::new(writer);
while let Some(command) = rx.recv().await {
if let Some(key) = command.pending {
if !shared.mark_pending_accepted(key) {
continue;
}
}
if let Err(err) = write_frame(&mut writer, &command.frame).await {
shared.handle_generation_drop(generation, err.to_string());
return;
}
while let Ok(command) = rx.try_recv() {
if let Some(key) = command.pending {
if !shared.mark_pending_accepted(key) {
continue;
}
}
if let Err(err) = write_frame(&mut writer, &command.frame).await {
shared.handle_generation_drop(generation, err.to_string());
return;
}
}
if let Err(err) = writer.flush().await.map_err(FrameIoError::Io) {
shared.handle_generation_drop(generation, err.to_string());
return;
}
}
}
fn route_open_consumer_identity(opts: &CallOptions) -> Option<ConsumerIdentity> {
opts.consumer_identity
.clone()
.or_else(consumer_identity_from_env)
}
fn close_route_consumer_identity(opts: &CloseRouteOptions) -> Option<ConsumerIdentity> {
opts.consumer_identity
.clone()
.or_else(consumer_identity_from_env)
}
fn route_open_consumer_capabilities(opts: &CallOptions) -> Option<Vec<String>> {
opts.reverse_requests.seal();
let capabilities = opts.reverse_requests.capabilities();
(!capabilities.is_empty()).then_some(capabilities)
}
fn close_route_consumer_capabilities(opts: &CloseRouteOptions) -> Option<Vec<String>> {
opts.reverse_requests.seal();
let capabilities = opts.reverse_requests.capabilities();
(!capabilities.is_empty()).then_some(capabilities)
}
fn consumer_identity_from_env() -> Option<ConsumerIdentity> {
let module_id = std::env::var(SUBC_MODULE_ID_ENV)
.ok()
.filter(|value| !value.is_empty())?;
let launch_nonce = crate::launch_nonce()
.ok()
.flatten()
.map(|nonce| nonce.value().to_string())?;
Some(ConsumerIdentity {
module_id,
launch_nonce,
})
}
fn validate_capability_for_resolution(capability: &str) -> Result<(), CallError> {
if is_valid_capability_identifier(capability) {
Ok(())
} else {
Err(CallError::InvalidCapabilityIdentifier {
capability: capability.to_string(),
})
}
}
fn capability_claimants(catalog: &CatalogList, capability: &str) -> Vec<String> {
let mut claimants = catalog
.modules
.iter()
.filter(|module| {
module.capabilities.as_ref().is_some_and(|capabilities| {
capabilities
.provides
.iter()
.any(|claim| claim == capability)
})
})
.map(|module| module.module_id.clone())
.collect::<Vec<_>>();
claimants.sort();
claimants
}
fn epoch_millis() -> u64 {
u64::try_from(
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis(),
)
.unwrap_or(u64::MAX)
}
fn default_random_unit() -> f64 {
use std::hash::{BuildHasher, RandomState};
static COUNTER: AtomicU64 = AtomicU64::new(0);
let bits = RandomState::new().hash_one(COUNTER.fetch_add(1, Ordering::Relaxed));
(bits >> 11) as f64 / (1u64 << 53) as f64
}
fn request_not_sent_after_route_open_failure(err: CallError) -> CallError {
match err {
CallError::OutcomeUnknown(source) => CallError::not_sent(format!(
"request not sent: route.open did not complete ({source})"
)),
other => other,
}
}
fn classify_failure(accepted: bool, reason: impl Into<String>) -> CallError {
if accepted {
CallError::outcome_unknown(reason)
} else {
CallError::not_sent(reason)
}
}
fn settle_pending_entries(entries: Vec<PendingEntry>, reason: String) {
for entry in entries {
entry.settle_failure(reason.clone());
}
}
fn drain_pending_generation(
pending: &mut HashMap<PendingKey, PendingEntry>,
generation: u64,
) -> Vec<PendingEntry> {
let keys = pending
.keys()
.copied()
.filter(|key| key.generation == generation)
.collect::<Vec<_>>();
keys.into_iter()
.filter_map(|key| pending.remove(&key))
.collect()
}
fn drain_pending_handle(
pending: &mut HashMap<PendingKey, PendingEntry>,
handle: RouteHandle,
include_subscriptions: bool,
) -> Vec<PendingEntry> {
let keys = pending
.iter()
.filter_map(|(key, entry)| {
(key.generation == handle.connection_token()
&& key.channel == handle.channel
&& key.epoch == handle.epoch
&& (include_subscriptions || !entry.is_subscription()))
.then_some(*key)
})
.collect::<Vec<_>>();
keys.into_iter()
.filter_map(|key| pending.remove(&key))
.collect()
}
fn drain_openings(openings: &mut HashMap<RouteKey, Opening>) -> Vec<Vec<OpeningWaiter>> {
openings
.drain()
.map(|(_, opening)| opening.waiters)
.collect()
}
fn fail_openings(openings: Vec<Vec<OpeningWaiter>>, failure: SharedCallFailure) {
for waiters in openings {
for waiter in waiters {
let _ = waiter.send(Err(failure.clone()));
}
}
}
fn emit_callbacks(callbacks: Vec<Callback>, state: ConnectionState) {
for callback in callbacks {
if let Ok(callback) = callback.lock() {
callback(state.clone());
}
}
}
pub fn is_retryable_route_open_code(code: &str) -> bool {
error_codes::is_retryable_route_open(code)
}
fn route_is_installed(inner: &Inner, handle: RouteHandle) -> bool {
handle.channel == 0 || inner.route_epochs.get(&handle.channel) == Some(&handle)
}
fn is_retryable_catalog_transport_error(err: &CallError) -> bool {
matches!(err, CallError::NotSent(_) | CallError::OutcomeUnknown(_))
}
fn is_reconnect_transient(err: &ConsumerError) -> bool {
match err {
ConsumerError::Connect { source, .. } => matches!(
source.kind(),
io::ErrorKind::ConnectionRefused
| io::ErrorKind::ConnectionReset
| io::ErrorKind::ConnectionAborted
| io::ErrorKind::TimedOut
| io::ErrorKind::NotConnected
| io::ErrorKind::AddrNotAvailable
),
ConsumerError::ConnectionFile { source, .. } => match source {
ConnectionFileError::Io { source, .. } => source.kind() == io::ErrorKind::NotFound,
_ => false,
},
ConsumerError::Auth { .. } => true,
ConsumerError::Discovery { .. }
| ConsumerError::NoEndpoint { .. }
| ConsumerError::Closed => false,
}
}
impl From<FrameBuildError> for CallError {
fn from(err: FrameBuildError) -> Self {
Self::not_sent(err.to_string())
}
}
#[cfg(test)]
mod tests {
use super::*;
use subc_protocol::FLAG_SUBSCRIPTION;
#[derive(Clone)]
struct InstrumentedWriter {
state: Arc<InstrumentedWriterState>,
fail_flush: bool,
}
#[derive(Default)]
struct InstrumentedWriterState {
bytes: Mutex<Vec<u8>>,
flushes: std::sync::atomic::AtomicUsize,
}
impl InstrumentedWriter {
fn new(fail_flush: bool) -> Self {
Self {
state: Arc::new(InstrumentedWriterState::default()),
fail_flush,
}
}
fn bytes(&self) -> Vec<u8> {
self.state
.bytes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone()
}
fn flush_count(&self) -> usize {
self.state.flushes.load(Ordering::SeqCst)
}
}
impl AsyncWrite for InstrumentedWriter {
fn poll_write(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<io::Result<usize>> {
self.state
.bytes
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.extend_from_slice(buf);
Poll::Ready(Ok(buf.len()))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
self.state.flushes.fetch_add(1, Ordering::SeqCst);
if self.fail_flush {
Poll::Ready(Err(io::Error::new(
io::ErrorKind::BrokenPipe,
"instrumented flush failure",
)))
} else {
Poll::Ready(Ok(()))
}
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
}
fn writer_test_shared() -> Arc<Shared> {
Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions {
reconnect_backoff: RetryBackoff {
max_attempts: 1,
..RetryBackoff::default()
},
..ConsumerOptions::default()
},
))
}
fn reverse_test_shared(
reverse_requests: ReverseRequestRegistry,
) -> (Arc<Shared>, mpsc::Receiver<WriteCommand>, RouteHandle) {
let shared = writer_test_shared();
let (writer, receiver) = mpsc::channel(16);
let handle = RouteHandle::new_consumer(17, 3, 1, reverse_requests);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.route_epochs.insert(handle.channel, handle);
}
(shared, receiver, handle)
}
fn reverse_request_frame(handle: RouteHandle, corr: u64, body: Vec<u8>) -> Frame {
Frame::build(
FrameType::Request,
Flags::new(false, Priority::Interactive, false),
handle.channel,
handle.epoch,
corr,
body,
)
.unwrap()
}
async fn capture_route_open(
reverse_requests: ReverseRequestRegistry,
) -> (serde_json::Value, RouteHandle) {
let shared = writer_test_shared();
let (writer, mut receiver) = mpsc::channel(4);
shared.lock_inner().writer = Some(writer);
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let opts = CallOptions {
reverse_requests,
..CallOptions::default()
};
let task = tokio::spawn(async move {
consumer
.open_route(
RouteTarget::ToolProvider {
module_id: "reverse-provider".to_string(),
},
BindIdentity::new(
PathBuf::from("/tmp/project"),
"test".to_string(),
"reverse".to_string(),
),
opts,
)
.await
});
let command = receiver.recv().await.expect("route.open frame");
let request = serde_json::from_slice(&command.frame.body).unwrap();
let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
route_channel: 17,
route_epoch: 3,
})
.unwrap();
assert!(
dispatch_frame(
&shared,
1,
response_frame(0, 0, command.frame.header.corr, response_body),
)
.await
);
let handle = task.await.unwrap().unwrap();
(request, handle)
}
#[tokio::test]
async fn neither_call_nor_subscribe_emits_bit_7_until_the_fleet_tolerates_it() {
let (shared, mut receiver, handle) = reverse_test_shared(ReverseRequestRegistry::new());
let call_shared = Arc::clone(&shared);
let call = tokio::spawn(async move {
call_shared
.send_request(RequestSend {
expected_handle: Some(handle),
channel: handle.channel,
epoch: handle.epoch,
body: b"ordinary".to_vec(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline: Instant::now() + Duration::from_secs(1),
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await
});
let ordinary = receiver.recv().await.unwrap();
assert_eq!(ordinary.frame.header.encode()[6] & FLAG_SUBSCRIPTION, 0);
assert!(
dispatch_frame(
&shared,
1,
response_frame(
handle.channel,
handle.epoch,
ordinary.frame.header.corr,
Vec::new()
),
)
.await
);
assert!(call.await.unwrap().is_ok());
let permit = Arc::new(Semaphore::new(1)).acquire_owned().await.unwrap();
let subscription = Arc::clone(&shared)
.send_subscription(SubscriptionSend {
expected_handle: Some(handle),
channel: handle.channel,
epoch: handle.epoch,
body: b"held".to_vec(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
event_buffer: 1,
deadline: Instant::now() + Duration::from_secs(1),
permit,
})
.await
.unwrap();
let held = receiver.recv().await.unwrap();
assert_eq!(
held.frame.header.encode()[6] & FLAG_SUBSCRIPTION,
0,
"a held-open subscribe must NOT set bit 7 until every module's decoder \
accepts it; restoring emission here is Phase 2 and needs a fleet census"
);
drop(subscription);
release_reverse_request_registry(handle);
}
#[tokio::test]
async fn registered_handlers_derive_route_open_consumer_capabilities() {
let registry = ReverseRequestRegistry::new();
registry
.on_request("elicitation", |_body, _ctx| async { Vec::new() })
.unwrap();
let (request, handle) = capture_route_open(registry).await;
assert_eq!(
request.get("consumer_capabilities"),
Some(&serde_json::json!(["elicitation"]))
);
release_reverse_request_registry(handle);
}
#[tokio::test]
async fn no_handlers_omit_consumer_capabilities_from_route_open() {
let (request, handle) = capture_route_open(ReverseRequestRegistry::new()).await;
assert!(!request
.as_object()
.unwrap()
.contains_key("consumer_capabilities"));
release_reverse_request_registry(handle);
}
#[tokio::test]
async fn reverse_request_without_handler_returns_typed_error_instead_of_silence() {
let (shared, mut receiver, handle) = reverse_test_shared(ReverseRequestRegistry::new());
let body = serde_json::to_vec(&serde_json::json!({
"jsonrpc": "2.0",
"id": 1,
"method": "elicitation/create",
"params": {}
}))
.unwrap();
assert!(dispatch_frame(&shared, 1, reverse_request_frame(handle, 800, body)).await);
let reply = tokio::time::timeout(Duration::from_secs(1), receiver.recv())
.await
.unwrap()
.unwrap()
.frame;
assert_eq!(reply.header.ty, FrameType::Error);
assert_eq!(reply.header.corr, 800);
let error: ErrorBody = serde_json::from_slice(&reply.body).unwrap();
assert_eq!(error.code, REVERSE_REQUEST_UNHANDLED);
release_reverse_request_registry(handle);
}
#[tokio::test]
async fn reverse_request_round_trip_preserves_raw_body_corr_and_response() {
let registry = ReverseRequestRegistry::new();
let seen = Arc::new(Mutex::new(None));
let seen_by_handler = Arc::clone(&seen);
let response = br#"{"jsonrpc":"2.0","id":9,"result":{"accepted":true}}"#.to_vec();
let expected_response = response.clone();
registry
.on_request("elicitation", move |body, ctx| {
*seen_by_handler
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()) =
Some((body, ctx.corr, ctx.method));
let response = response.clone();
async move { response }
})
.unwrap();
let (shared, mut receiver, handle) = reverse_test_shared(registry);
let request_body =
br#"{"jsonrpc":"2.0","id":9,"method":"elicitation/create","params":{"z":1,"a":2}}"#
.to_vec();
assert!(
dispatch_frame(
&shared,
1,
reverse_request_frame(handle, 801, request_body.clone()),
)
.await
);
let reply = tokio::time::timeout(Duration::from_secs(1), receiver.recv())
.await
.unwrap()
.unwrap()
.frame;
assert_eq!(reply.header.ty, FrameType::Response);
assert_eq!(reply.header.corr, 801);
assert_eq!(reply.body, expected_response);
assert_eq!(
seen.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone(),
Some((request_body, 801, "elicitation/create".to_string()))
);
release_reverse_request_registry(handle);
}
#[tokio::test]
async fn panicking_reverse_handler_returns_typed_refusal_and_sibling_request_resolves() {
let registry = ReverseRequestRegistry::new();
registry
.on_request("sampling", |_body, _ctx| async move {
panic!("person prompt exploded");
})
.unwrap();
let (shared, mut receiver, handle) = reverse_test_shared(registry);
let sibling_shared = Arc::clone(&shared);
let sibling = tokio::spawn(async move {
sibling_shared
.send_request(RequestSend {
expected_handle: Some(handle),
channel: handle.channel,
epoch: handle.epoch,
body: b"sibling".to_vec(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline: Instant::now() + Duration::from_secs(1),
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await
});
let sibling_request = receiver.recv().await.unwrap();
let reverse_body = serde_json::to_vec(&serde_json::json!({
"method": "sampling/createMessage",
"params": {}
}))
.unwrap();
assert!(
dispatch_frame(&shared, 1, reverse_request_frame(handle, 802, reverse_body),).await
);
assert!(
dispatch_frame(
&shared,
1,
response_frame(
handle.channel,
handle.epoch,
sibling_request.frame.header.corr,
b"ok".to_vec(),
),
)
.await
);
assert!(matches!(
sibling.await.unwrap().unwrap(),
TerminalFrame::Response { body, .. } if body == b"ok"
));
let refusal = tokio::time::timeout(Duration::from_secs(1), receiver.recv())
.await
.unwrap()
.unwrap()
.frame;
assert_eq!(refusal.header.ty, FrameType::Error);
let error: ErrorBody = serde_json::from_slice(&refusal.body).unwrap();
assert_eq!(error.code, REVERSE_REQUEST_UNHANDLED);
assert!(error.message.contains("person prompt exploded"));
assert!(shared.generation_is_current(1));
release_reverse_request_registry(handle);
}
#[tokio::test]
async fn writer_batches_ready_frames_into_one_flush() {
const FRAME_COUNT: usize = 8;
let shared = writer_test_shared();
let (live_writer, _live_rx) = mpsc::channel(1);
let (tx, rx) = mpsc::channel(FRAME_COUNT + 1);
let instrumented = InstrumentedWriter::new(false);
let observer = instrumented.clone();
let mut expected = Vec::with_capacity(FRAME_COUNT);
let mut keys = Vec::with_capacity(FRAME_COUNT);
{
let mut inner = shared.lock_inner();
inner.writer = Some(live_writer);
for index in 0..FRAME_COUNT {
let corr = index as u64 + 1;
let frame = response_frame(7, 1, corr, vec![index as u8; index + 1]);
let key = PendingKey {
generation: 1,
channel: 7,
epoch: 1,
corr,
};
let (pending_tx, _pending_rx) = oneshot::channel();
inner
.pending
.insert(key, PendingEntry::unary(pending_tx, false, None, None));
expected.push(frame.clone());
keys.push(key);
tx.try_send(WriteCommand {
frame,
pending: Some(key),
})
.expect("the burst should fit in the writer queue");
if index == 0 {
tx.try_send(WriteCommand {
frame: response_frame(7, 1, 999, b"skip".to_vec()),
pending: Some(PendingKey {
generation: 1,
channel: 7,
epoch: 1,
corr: 999,
}),
})
.expect("the skipped command should fit in the writer queue");
}
}
}
drop(tx);
writer_loop(Arc::clone(&shared), instrumented, rx, 1).await;
{
let inner = shared.lock_inner();
for key in keys {
assert!(
inner.pending.get(&key).is_some_and(|entry| entry.accepted),
"every written command must be marked accepted"
);
}
}
let mut wire = std::io::Cursor::new(observer.bytes());
for expected_frame in expected {
let actual = read_frame(&mut wire)
.await
.expect("the emitted frame should decode")
.expect("the emitted frame should be present");
assert_eq!(actual, expected_frame);
}
assert!(
read_frame(&mut wire)
.await
.expect("the end of the emitted burst should be clean")
.is_none(),
"the writer must not emit extra frames"
);
let flush_count = observer.flush_count();
assert_eq!(
flush_count, 1,
"a ready burst must be coalesced into one flush"
);
shared.close_sync("test complete");
}
#[tokio::test]
async fn writer_flush_failure_drops_generation_and_preserves_acceptance_classification() {
let shared = writer_test_shared();
let (live_writer, _live_rx) = mpsc::channel(1);
let accepted_key = PendingKey {
generation: 1,
channel: 3,
epoch: 1,
corr: 1,
};
let not_sent_key = PendingKey {
corr: 2,
..accepted_key
};
let (accepted_tx, accepted_rx) = oneshot::channel();
let (not_sent_tx, not_sent_rx) = oneshot::channel();
{
let mut inner = shared.lock_inner();
inner.writer = Some(live_writer);
inner.pending.insert(
accepted_key,
PendingEntry::unary(accepted_tx, false, None, None),
);
inner.pending.insert(
not_sent_key,
PendingEntry::unary(not_sent_tx, false, None, None),
);
}
let (tx, rx) = mpsc::channel(1);
tx.send(WriteCommand {
frame: response_frame(3, 1, accepted_key.corr, b"accepted".to_vec()),
pending: Some(accepted_key),
})
.await
.unwrap();
drop(tx);
writer_loop(Arc::clone(&shared), InstrumentedWriter::new(true), rx, 1).await;
assert!(
shared.lock_inner().writer.is_none(),
"a flush failure must drop the active generation"
);
let accepted_error = accepted_rx
.await
.expect("the accepted request should be settled")
.into_call_result()
.unwrap_err();
assert!(matches!(accepted_error, CallError::OutcomeUnknown(_)));
let not_sent_error = not_sent_rx
.await
.expect("the unwritten request should be settled")
.into_call_result()
.unwrap_err();
assert!(matches!(not_sent_error, CallError::NotSent(_)));
shared.close_sync("test complete");
}
#[test]
fn reconnect_classifier_treats_auth_failure_as_transient() {
let auth = ConsumerError::Auth {
path: PathBuf::from("/tmp/subc-connection.json"),
endpoint: "127.0.0.1:8757".to_string(),
source: subc_transport::AuthError::InvalidServerProof,
};
assert!(is_reconnect_transient(&auth), "rotation race must retry");
let absent = ConsumerError::ConnectionFile {
path: PathBuf::from("/tmp/subc-connection.json"),
source: ConnectionFileError::Io {
op: "read",
path: PathBuf::from("/tmp/subc-connection.json"),
source: io::Error::new(io::ErrorKind::NotFound, "gone"),
},
};
assert!(is_reconnect_transient(&absent));
}
#[tokio::test]
async fn newer_drop_supersedes_reconnect_and_ignores_stale_completion() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let stale_task = tokio::spawn(std::future::pending::<()>());
{
let mut inner = shared.lock_inner();
inner.generation = 2;
inner.reconnect = ReconnectState::Background {
generation: 1,
task: stale_task,
};
}
assert!(shared.spawn_reconnect(2));
assert!(matches!(
&shared.lock_inner().reconnect,
ReconnectState::Background { generation, .. } if *generation == 2
));
shared.finish_background_reconnect(1);
assert!(matches!(
&shared.lock_inner().reconnect,
ReconnectState::Background { generation, .. } if *generation == 2
));
shared.close_sync("test complete");
}
#[test]
fn retryable_route_open_codes_are_code_specific() {
for code in [
"module_reloading",
"module_warming",
"target_unavailable",
"module_timeout",
] {
assert!(is_retryable_route_open_code(code), "{code} should retry");
}
assert!(
!is_retryable_route_open_code(error_codes::UNKNOWN_MODULE),
"unknown_module is terminal"
);
assert!(!is_retryable_route_open_code(error_codes::MODULE_REMOVED));
assert!(!is_retryable_route_open_code("invalid_project_root"));
assert!(!is_retryable_route_open_code("route_rejected"));
assert!(
!is_retryable_route_open_code("capability_forbidden"),
"a capability policy refusal must not enter the route.open retry set"
);
}
#[test]
fn route_close_reason_classifier_accepts_capability_denied_and_fails_closed_for_unknown() {
assert_eq!(
RouteCloseReason::from_wire("capability_denied"),
RouteCloseReason::CapabilityDenied
);
assert_eq!(
RouteCloseReason::from_wire("capability_denied").disposition(),
RouteCloseDisposition::MustNotReopen
);
assert_eq!(
RouteCloseReason::from_wire("future_policy_reason").disposition(),
RouteCloseDisposition::MustNotReopen,
"an unknown close reason must receive the strictest handling"
);
assert_eq!(
RouteCloseReason::from_wire("reload").disposition(),
RouteCloseDisposition::MayReopen,
"control proves the classifier can distinguish a conservative default"
);
}
#[tokio::test]
async fn module_removed_fails_fast_while_module_reloading_retries_at_the_same_route_open_call_site(
) {
async fn reject_route_open_attempts(code: &str, deadline: Duration) -> (usize, CallError) {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut receiver) = mpsc::channel(4);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
}
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let target = RouteTarget::ToolProvider {
module_id: "retry-polarity".to_string(),
};
let identity = BindIdentity::new(
PathBuf::from("/tmp/project"),
"test".to_string(),
code.to_string(),
);
let options = CallOptions {
timeout: Duration::from_secs(2),
route_retry: RetryBackoff {
base: Duration::ZERO,
cap: Duration::ZERO,
max_attempts: 2,
},
route_retry_deadline: deadline,
..CallOptions::default()
};
let mut task =
tokio::spawn(async move { consumer.open_route(target, identity, options).await });
let mut attempts = 0usize;
let result = loop {
tokio::select! {
command = receiver.recv() => {
let Some(command) = command else {
break task.await.unwrap().expect_err("route.open must reject");
};
attempts += 1;
let body =
serde_json::to_vec(&ErrorBody::new(code, "test rejection")).unwrap();
assert!(
dispatch_frame(
&shared,
1,
Frame::build(
FrameType::Error,
Flags::new(false, Priority::Interactive, false),
0,
0,
command.frame.header.corr,
body,
)
.unwrap(),
)
.await
);
}
joined = &mut task => {
break joined.unwrap().expect_err("route.open must reject");
}
}
};
assert!(
receiver.try_recv().is_err(),
"a settled route.open must not queue another attempt"
);
(attempts, result)
}
let (reloading_attempts, reloading) =
reject_route_open_attempts(error_codes::MODULE_RELOADING, Duration::from_millis(150))
.await;
assert!(matches!(reloading, CallError::NotSent(_)));
assert!(
reloading_attempts > 2,
"reloading retries must run until the deadline, not an attempt cap \
(served {reloading_attempts} attempts against max_attempts=2)"
);
let (removed_attempts, removed) =
reject_route_open_attempts(error_codes::MODULE_REMOVED, Duration::from_millis(150))
.await;
assert!(matches!(removed, CallError::NotSent(_)));
assert_eq!(
removed_attempts, 1,
"terminal codes settle on the first answer"
);
}
fn seeded_random(seed: u64) -> Box<dyn Fn() -> f64 + Send + Sync> {
let state = AtomicU64::new(seed);
Box::new(move || {
let mut z = state
.fetch_add(0x9E37_79B9_7F4A_7C15, Ordering::Relaxed)
.wrapping_add(0x9E37_79B9_7F4A_7C15);
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^= z >> 31;
(z >> 11) as f64 / (1u64 << 53) as f64
})
}
fn route_open_stand_in(
random_unit: Box<dyn Fn() -> f64 + Send + Sync>,
) -> (Arc<Shared>, Arc<SubcConsumer>, mpsc::Receiver<WriteCommand>) {
let mut shared = Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
);
shared.random_unit = random_unit;
let shared = Arc::new(shared);
let (writer, receiver) = mpsc::channel(64);
shared.lock_inner().writer = Some(writer);
let consumer = Arc::new(SubcConsumer {
shared: Arc::clone(&shared),
});
(shared, consumer, receiver)
}
fn restart_target(module_id: &str) -> RouteTarget {
RouteTarget::ToolProvider {
module_id: module_id.to_string(),
}
}
fn restart_identity() -> BindIdentity {
BindIdentity::new(
PathBuf::from("/tmp/project"),
"test".to_string(),
"restart".to_string(),
)
}
fn route_open_answer(corr: u64, channel: u16, refusal: Option<ErrorBody>) -> Frame {
match refusal {
None => response_frame(
0,
0,
corr,
serde_json::to_vec(&ClientControlResponse::RouteOpen {
route_channel: channel,
route_epoch: 1,
})
.unwrap(),
),
Some(body) => Frame::build(
FrameType::Error,
Flags::new(false, Priority::Interactive, false),
0,
0,
corr,
serde_json::to_vec(&body).unwrap(),
)
.unwrap(),
}
}
async fn answer_route_opens<T>(
shared: &Arc<Shared>,
receiver: &mut mpsc::Receiver<WriteCommand>,
task: &mut JoinHandle<T>,
mut answer: impl FnMut(&str) -> Option<ErrorBody>,
) -> T {
let mut next_channel = 40u16;
loop {
tokio::select! {
command = receiver.recv() => {
let Some(command) = command else {
return (&mut *task).await.unwrap();
};
let request: serde_json::Value =
serde_json::from_slice(&command.frame.body).unwrap();
let module_id = request["target"]["module_id"].as_str().unwrap().to_string();
next_channel += 1;
let frame = route_open_answer(
command.frame.header.corr,
next_channel,
answer(&module_id),
);
assert!(dispatch_frame(shared, 1, frame).await);
}
joined = &mut *task => return joined.unwrap(),
}
}
}
fn reloading() -> ErrorBody {
ErrorBody::new(error_codes::MODULE_RELOADING, "module is reloading")
}
#[tokio::test(start_paused = true)]
async fn the_default_retry_deadline_outlasts_a_60s_restart_and_the_open_succeeds() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(1));
let started = Instant::now();
let options = CallOptions {
timeout: Duration::from_secs(120),
..CallOptions::default()
};
let mut task = tokio::spawn(async move {
consumer
.open_route(restart_target("restarting"), restart_identity(), options)
.await
});
let opened = answer_route_opens(&shared, &mut receiver, &mut task, |_| {
(started.elapsed() < Duration::from_secs(60)).then(reloading)
})
.await;
let handle =
opened.unwrap_or_else(|err| panic!("the open must survive the restart: {err}"));
assert!(handle.channel > 40);
assert!(started.elapsed() >= Duration::from_secs(60));
}
#[tokio::test(start_paused = true)]
async fn a_call_timeout_shorter_than_the_retry_deadline_ends_the_retries_at_the_call_timeout() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(2));
let started = Instant::now();
let options = CallOptions {
timeout: Duration::from_secs(10),
..CallOptions::default()
};
let mut task = tokio::spawn(async move {
consumer
.open_route(restart_target("restarting"), restart_identity(), options)
.await
});
let err = answer_route_opens(&shared, &mut receiver, &mut task, |_| {
(started.elapsed() < Duration::from_secs(60)).then(reloading)
})
.await
.expect_err("the call timeout must end the retries before the module is back");
let elapsed = started.elapsed();
assert!(matches!(err, CallError::NotSent(_)), "got {err:?}");
assert!(
elapsed >= Duration::from_secs(9) && elapsed <= Duration::from_secs(10),
"the retries must stop at the 10s call timeout, stopped after {elapsed:?}"
);
}
#[tokio::test(start_paused = true)]
async fn two_routes_refused_together_retry_at_different_jittered_times() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(42));
let started = Instant::now();
let mut task = tokio::spawn(async move {
let alpha = consumer.open_route(
restart_target("alpha"),
restart_identity(),
CallOptions::default(),
);
let beta = consumer.open_route(
restart_target("beta"),
restart_identity(),
CallOptions::default(),
);
let (alpha, beta) = tokio::join!(alpha, beta);
(alpha.is_ok(), beta.is_ok())
});
let mut first_open = HashMap::<String, Duration>::new();
let mut retried_at = HashMap::<String, Duration>::new();
let opened = answer_route_opens(&shared, &mut receiver, &mut task, |module_id| {
let at = started.elapsed();
if first_open.contains_key(module_id) {
retried_at.insert(module_id.to_string(), at);
None
} else {
first_open.insert(module_id.to_string(), at);
Some(reloading())
}
})
.await;
assert_eq!(opened, (true, true));
assert_eq!(first_open["alpha"], first_open["beta"], "refused together");
for at in retried_at.values() {
assert!(
*at >= Duration::from_millis(50) && *at <= Duration::from_millis(100),
"retry at {at:?} is outside the jitter window"
);
}
assert_ne!(
retried_at["alpha"], retried_at["beta"],
"routes refused together must not retry in lock step"
);
}
#[tokio::test(start_paused = true)]
async fn twenty_concurrent_opens_never_have_more_than_eight_outstanding_at_the_daemon() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(3));
let mut task = tokio::spawn(async move {
let mut opens = tokio::task::JoinSet::new();
for i in 0..20 {
let consumer = Arc::clone(&consumer);
opens.spawn(async move {
consumer
.open_route(
restart_target(&format!("module-{i}")),
restart_identity(),
CallOptions::default(),
)
.await
});
}
let mut opened = 0usize;
while let Some(result) = opens.join_next().await {
result.unwrap().unwrap();
opened += 1;
}
opened
});
let mut held = Vec::new();
let mut max_outstanding = 0usize;
let mut next_channel = 40u16;
let opened = loop {
tokio::select! {
command = receiver.recv() => {
let Some(command) = command else {
break (&mut task).await.unwrap();
};
held.push(command.frame.header.corr);
max_outstanding = max_outstanding.max(held.len());
}
() = sleep(Duration::from_millis(25)), if !held.is_empty() => {
for corr in held.drain(..) {
next_channel += 1;
assert!(dispatch_frame(&shared, 1, route_open_answer(corr, next_channel, None)).await);
}
}
joined = &mut task => break joined.unwrap(),
}
};
assert_eq!(opened, 20);
assert_eq!(max_outstanding, MAX_ROUTE_OPENS_IN_FLIGHT);
}
#[tokio::test(start_paused = true)]
async fn an_exhausted_deadline_names_module_reloading_not_the_later_admission_pressure() {
let (shared, consumer, mut receiver) = route_open_stand_in(seeded_random(4));
let options = CallOptions {
route_retry_deadline: Duration::from_secs(5),
..CallOptions::default()
};
let mut task = tokio::spawn(async move {
consumer
.open_route(restart_target("restarting"), restart_identity(), options)
.await
});
let mut opens = 0usize;
let err = answer_route_opens(&shared, &mut receiver, &mut task, |_| {
opens += 1;
Some(if opens <= 2 {
reloading().with_detail(serde_json::json!({ "reason": "restart" }))
} else {
ErrorBody::new(
error_codes::TARGET_UNAVAILABLE,
"connection already has 8 route.open binds in flight (limit 8); retry after one settles",
)
})
})
.await
.expect_err("the module never comes back");
assert!(opens > 2);
assert!(matches!(err, CallError::NotSent(_)), "got {err:?}");
assert_eq!(
err.route_open_refusal().map(|body| body.code.as_str()),
Some(error_codes::MODULE_RELOADING)
);
let message = err.to_string();
assert!(
message.starts_with(
"request not sent: route.open failed for target tool_provider:restarting: \
module_reloading (reason restart) (module is reloading); \
the most recent refusal was target_unavailable"
),
"got {message}"
);
}
#[tokio::test]
async fn close_route_flips_inflight_opening_so_a_racing_open_discards() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let key = RouteKey::new(
&RouteTarget::ToolProvider {
module_id: "m".into(),
},
&BindIdentity::new(PathBuf::from("/tmp/p"), "h", "s"),
None,
None,
);
shared.lock_inner().openings.insert(
key.clone(),
Opening {
waiters: Vec::new(),
closed: false,
},
);
shared
.close_route(&key, &CloseRouteOptions::default())
.await;
assert!(
shared
.lock_inner()
.openings
.get(&key)
.is_some_and(|o| o.closed),
"close_route must flip the in-flight opening's closed flag (close-beats-reopen)"
);
let absent = RouteKey::new(
&RouteTarget::ToolProvider {
module_id: "absent".into(),
},
&BindIdentity::new(PathBuf::from("/tmp/p"), "h", "s"),
None,
None,
);
shared
.close_route(&absent, &CloseRouteOptions::default())
.await;
}
#[test]
fn route_key_is_structured() {
let target = RouteTarget::InternalService {
module_id: "a\0b".into(),
service_id: "svc".into(),
};
let identity = BindIdentity::new(PathBuf::from("/tmp/project"), "h", "s");
let key = RouteKey::new(&target, &identity, None, None);
assert_eq!(key.project_root, PathBuf::from("/tmp/project"));
assert!(matches!(key.target, RouteTargetKey::InternalService { .. }));
}
#[tokio::test]
async fn route_channel_index_tracks_lookup_close_and_generation_drop() {
let shared = writer_test_shared();
let (writer, _rx) = mpsc::channel(32);
let mut expected = Vec::new();
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
for channel in 1..=8 {
let key = RouteKey::new(
&RouteTarget::ToolProvider {
module_id: format!("module-{channel}"),
},
&BindIdentity::new(
PathBuf::from("/tmp/project"),
"test",
format!("session-{channel}"),
),
None,
None,
);
let route = RouteState {
handle: RouteHandle::new(channel, channel.into(), 1),
sem: Arc::new(Semaphore::new(DEFAULT_ROUTE_WINDOW)),
};
inner.cache_route(key.clone(), route.clone());
expected.push((key, route));
}
}
for (key, expected_route) in &expected {
let resolved = shared
.route_state(expected_route.handle)
.expect("an indexed route handle should resolve");
assert_eq!(resolved.handle, expected_route.handle);
assert!(Arc::ptr_eq(&resolved.sem, &expected_route.sem));
let inner = shared.lock_inner();
assert_eq!(
inner.route_by_channel.get(&expected_route.handle.channel),
Some(key)
);
assert_eq!(
inner.route_epochs.get(&expected_route.handle.channel),
Some(&expected_route.handle)
);
assert!(inner
.routes
.get(key)
.is_some_and(|route| route.handle == expected_route.handle));
}
let (closed_key, closed_route) = &expected[3];
shared
.close_route(closed_key, &CloseRouteOptions::default())
.await;
{
let inner = shared.lock_inner();
assert!(!inner.routes.contains_key(closed_key));
assert!(!inner
.route_by_channel
.contains_key(&closed_route.handle.channel));
assert!(!inner
.route_epochs
.contains_key(&closed_route.handle.channel));
}
assert!(matches!(
shared.route_state(closed_route.handle),
Err(CallError::StaleRouteHandle(handle)) if handle == closed_route.handle
));
shared.handle_generation_drop(1, "test generation dropped".into());
{
let inner = shared.lock_inner();
assert!(inner.routes.is_empty());
assert!(inner.route_by_channel.is_empty());
assert!(inner.route_epochs.is_empty());
}
shared.close_sync("test complete");
}
#[tokio::test]
async fn stale_push_is_not_delivered_after_connection_generation_changes() {
let shared = writer_test_shared();
let old_handle = RouteHandle::new(7, 1, 1);
let (writer, _writer_rx) = mpsc::channel(1);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.route_epochs.insert(old_handle.channel, old_handle);
}
let mut pushes = shared
.register_push_events(old_handle)
.expect("the old live route should accept a receiver");
{
let mut inner = shared.lock_inner();
inner.generation = 2;
inner.close_routes();
inner.route_epochs.clear();
}
shared.route_push(old_handle, b"stale".to_vec());
assert!(
pushes.recv().await.is_none(),
"connection teardown must end the old receiver before a stale Push can arrive"
);
shared.close_sync("test complete");
}
#[test]
fn route_key_canonicalizes_consumer_capabilities() {
let target = RouteTarget::ToolProvider {
module_id: "aft".into(),
};
let identity = BindIdentity::new(PathBuf::from("/tmp/project"), "h", "s");
let left = RouteKey::new(
&target,
&identity,
None,
Some(&["sampling".to_string(), "elicitation".to_string()]),
);
let right = RouteKey::new(
&target,
&identity,
None,
Some(&[
"elicitation".to_string(),
"sampling".to_string(),
"sampling".to_string(),
]),
);
assert_eq!(left, right);
}
#[test]
fn drain_pending_channel_can_skip_subscriptions() {
let mut pending = HashMap::new();
let generation = 7;
let channel = 11;
let handle = RouteHandle::new(channel, 3, generation);
let unary_key = PendingKey {
generation,
channel,
epoch: handle.epoch,
corr: 1,
};
let subscription_key = PendingKey {
generation,
channel,
epoch: handle.epoch,
corr: 2,
};
let (unary_tx, _unary_rx) = oneshot::channel();
pending.insert(unary_key, PendingEntry::unary(unary_tx, false, None, None));
let (events_tx, _events_rx) = mpsc::channel(1);
let (closed_tx, _closed_rx) = oneshot::channel();
let permit = Arc::new(Semaphore::new(1))
.try_acquire_owned()
.expect("test semaphore permit should be available");
pending.insert(
subscription_key,
PendingEntry::subscription(events_tx, closed_tx, permit, Priority::Interactive),
);
let drained = drain_pending_handle(&mut pending, handle, false);
assert_eq!(drained.len(), 1);
assert!(pending.contains_key(&subscription_key));
let drained = drain_pending_handle(&mut pending, handle, true);
assert_eq!(drained.len(), 1);
assert!(pending.is_empty());
}
fn response_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
Frame::build(
FrameType::Response,
Flags::new(false, Priority::Interactive, false),
channel,
epoch,
corr,
body,
)
.unwrap()
}
#[tokio::test]
async fn stale_epoch_ingress_drops_without_settling_matching_corr() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, _rx) = mpsc::channel(4);
let current = RouteHandle::new(9, 2, 1);
let stale_key = PendingKey {
generation: 1,
channel: 9,
epoch: 1,
corr: 77,
};
let key = PendingKey {
generation: 1,
channel: 9,
epoch: 2,
corr: 77,
};
let (stale_tx, mut stale_response) = oneshot::channel();
let (tx, mut response) = oneshot::channel();
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.route_epochs.insert(9, current);
inner
.pending
.insert(stale_key, PendingEntry::unary(stale_tx, false, None, None));
inner
.pending
.insert(key, PendingEntry::unary(tx, false, None, None));
}
assert!(dispatch_frame(&shared, 1, response_frame(9, 1, 77, b"stale".to_vec())).await);
assert!(matches!(
response.try_recv(),
Err(oneshot::error::TryRecvError::Empty)
));
assert!(matches!(
stale_response.try_recv(),
Err(oneshot::error::TryRecvError::Empty)
));
assert!(shared.lock_inner().pending.contains_key(&stale_key));
assert!(shared.lock_inner().pending.contains_key(&key));
assert_eq!(shared.lock_inner().dropped_route_frames, 1);
assert!(dispatch_frame(&shared, 1, response_frame(9, 2, 77, b"current".to_vec())).await);
let PendingResult::Terminal(PendingTerminal::Response { body, .. }) =
response.await.unwrap()
else {
panic!("current epoch must settle its own pending request");
};
assert_eq!(body, b"current");
assert!(shared.lock_inner().pending.contains_key(&stale_key));
}
#[tokio::test]
async fn route_poll_response_must_echo_expected_handle_before_settling() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, _rx) = mpsc::channel(4);
let handle = RouteHandle::new(3, 9, 1);
let key = PendingKey {
generation: 1,
channel: 0,
epoch: 0,
corr: 88,
};
let (tx, mut response) = oneshot::channel();
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.route_epochs.insert(handle.channel, handle);
inner
.pending
.insert(key, PendingEntry::unary(tx, false, Some(handle), None));
}
let wrong = serde_json::to_vec(&ClientControlResponse::RoutePoll {
route_channel: handle.channel,
route_epoch: handle.epoch + 1,
status: Some("wrong".to_string()),
live: Some(true),
})
.unwrap();
assert!(dispatch_frame(&shared, 1, response_frame(0, 0, key.corr, wrong)).await);
assert!(matches!(
response.try_recv(),
Err(oneshot::error::TryRecvError::Empty)
));
assert!(shared.lock_inner().pending.contains_key(&key));
let correct = serde_json::to_vec(&ClientControlResponse::RoutePoll {
route_channel: handle.channel,
route_epoch: handle.epoch,
status: Some("ready".to_string()),
live: Some(true),
})
.unwrap();
assert!(dispatch_frame(&shared, 1, response_frame(0, 0, key.corr, correct)).await);
assert!(matches!(
response.await.unwrap(),
PendingResult::Terminal(PendingTerminal::Response { .. })
));
}
#[tokio::test]
async fn stale_connection_handle_emits_no_request_cancel_or_goodbye() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(4);
let stale = RouteHandle::new(4, 1, 1);
let current = RouteHandle::new(4, 1, 2);
{
let mut inner = shared.lock_inner();
inner.generation = 2;
inner.writer = Some(writer);
inner.route_epochs.insert(4, current);
}
let err = shared
.send_request(RequestSend {
expected_handle: Some(stale),
channel: stale.channel,
epoch: stale.epoch,
body: b"request".to_vec(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline: Instant::now() + Duration::from_millis(10),
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await
.unwrap_err();
assert!(matches!(err, CallError::StaleRouteHandle(handle) if handle == stale));
shared.send_cancel(stale, 8, Priority::Interactive);
assert!(!shared.send_route_goodbye(stale, false));
assert!(
rx.try_recv().is_err(),
"stale operations must not queue frames"
);
}
#[tokio::test]
async fn late_route_open_queues_goodbye_and_full_queue_closes_connection() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(2);
let key = PendingKey {
generation: 1,
channel: 0,
epoch: 0,
corr: 41,
};
let (tx, response) = oneshot::channel();
drop(response);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner
.pending
.insert(key, PendingEntry::unary(tx, true, None, None));
}
let body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
route_channel: 12,
route_epoch: 7,
})
.unwrap();
assert!(dispatch_frame(&shared, 1, response_frame(0, 0, 41, body)).await);
let cleanup = rx.recv().await.unwrap().frame;
assert_eq!(cleanup.header.ty, FrameType::Goodbye);
assert_eq!((cleanup.header.channel, cleanup.header.epoch), (12, 7));
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions {
reconnect_backoff: RetryBackoff {
max_attempts: 1,
..RetryBackoff::default()
},
..ConsumerOptions::default()
},
));
let (writer, _rx) = mpsc::channel(1);
let filler = response_frame(0, 0, 1, Vec::new());
writer
.try_send(WriteCommand {
frame: filler,
pending: None,
})
.unwrap();
let key = PendingKey {
generation: 1,
channel: 0,
epoch: 0,
corr: 42,
};
let (tx, response) = oneshot::channel();
drop(response);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner
.pending
.insert(key, PendingEntry::unary(tx, true, None, None));
}
let body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
route_channel: 13,
route_epoch: 8,
})
.unwrap();
assert!(dispatch_frame(&shared, 1, response_frame(0, 0, 42, body)).await);
assert!(shared.lock_inner().writer.is_none());
}
#[tokio::test]
async fn correlation_allocator_emits_max_once_then_closes_without_reuse() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions {
reconnect_backoff: RetryBackoff {
max_attempts: 1,
..RetryBackoff::default()
},
..ConsumerOptions::default()
},
));
let (writer, mut rx) = mpsc::channel(4);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.next_corr = Some(u64::MAX);
}
let request_shared = Arc::clone(&shared);
let request = tokio::spawn(async move {
request_shared
.send_request(RequestSend {
expected_handle: None,
channel: 0,
epoch: 0,
body: Vec::new(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline: Instant::now() + Duration::from_secs(1),
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await
});
let command = rx.recv().await.unwrap();
assert_eq!(command.frame.header.corr, u64::MAX);
assert!(dispatch_frame(&shared, 1, response_frame(0, 0, u64::MAX, Vec::new()),).await);
assert!(request.await.unwrap().is_ok());
let exhausted = shared
.send_request(RequestSend {
expected_handle: None,
channel: 0,
epoch: 0,
body: Vec::new(),
priority: Priority::Interactive,
admission_class: AdmissionClass::Normal,
deadline: Instant::now() + Duration::from_millis(10),
retain_late_route_open: false,
route_open_reverse_requests: None,
})
.await
.unwrap_err();
assert!(matches!(exhausted, CallError::NotSent(_)));
assert!(rx.try_recv().is_err());
assert!(shared.lock_inner().writer.is_none());
}
#[tokio::test]
async fn managed_call_deadline_bounds_flow_control_wait() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(4);
let target = RouteTarget::ToolProvider {
module_id: "flow-controlled".to_string(),
};
let identity = BindIdentity::new(
PathBuf::from("/tmp/project"),
"test".to_string(),
"deadline".to_string(),
);
let consumer_identity = Some(ConsumerIdentity {
module_id: "caller".to_string(),
launch_nonce: "nonce".to_string(),
});
let first_opts = CallOptions {
timeout: Duration::from_secs(1),
consumer_identity: consumer_identity.clone(),
..CallOptions::default()
};
let second_opts = CallOptions {
timeout: Duration::from_millis(25),
consumer_identity,
..CallOptions::default()
};
let key = RouteKey::new(
&target,
&identity,
first_opts.consumer_identity.as_ref(),
None,
);
let handle = RouteHandle::new(5, 3, 1);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.cache_route(
key,
RouteState {
handle,
sem: Arc::new(Semaphore::new(1)),
},
);
}
let first = tokio::spawn({
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let target = target.clone();
let identity = identity.clone();
async move {
consumer
.call(target, identity, b"first".to_vec(), first_opts)
.await
}
});
let first_frame = rx
.recv()
.await
.expect("the first request should enter the fake daemon queue");
assert_eq!(first_frame.frame.header.ty, FrameType::Request);
assert_eq!(first_frame.frame.body, b"first");
assert!(shared.mark_pending_accepted(
first_frame
.pending
.expect("request commands retain their pending key"),
));
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let result = tokio::time::timeout(
Duration::from_millis(250),
consumer.call(target, identity, b"second".to_vec(), second_opts),
)
.await
.expect("a flow-controlled call must finish at its own deadline")
.unwrap_err();
assert!(matches!(result, CallError::NotSent(_)));
assert!(
rx.try_recv().is_err(),
"the timed-out second request must not reach the fake daemon"
);
first.abort();
let _ = first.await;
}
#[tokio::test]
async fn admitted_route_open_emits_one_frame_without_retrying_daemon_errors() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions {
call_timeout: Duration::from_secs(1),
..ConsumerOptions::default()
},
));
let (writer, mut rx) = mpsc::channel(4);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
}
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let target = RouteTarget::ToolProvider {
module_id: "admitted-target".to_string(),
};
let identity = BindIdentity::new(
PathBuf::from("/tmp/project"),
"test".to_string(),
"admitted".to_string(),
);
let task = tokio::spawn(async move {
consumer
.open_route_with_admission_facts(
target,
identity,
serde_json::json!({"schema": 1, "verified_class": "member"}),
)
.await
});
let command = rx.recv().await.expect("one route.open must be queued");
let request: ClientControlRequest = serde_json::from_slice(&command.frame.body).unwrap();
let ClientControlRequest::RouteOpen {
admission_facts, ..
} = request
else {
panic!("expected route.open")
};
assert_eq!(
admission_facts,
Some(serde_json::json!({"schema": 1, "verified_class": "member"}))
);
let error_body = serde_json::to_vec(&ErrorBody {
code: "admission_facts_not_permitted".to_string(),
message: "not permitted".to_string(),
detail: None,
})
.unwrap();
assert!(
dispatch_frame(
&shared,
1,
Frame::build(
FrameType::Error,
Flags::new(false, Priority::Interactive, false),
0,
0,
command.frame.header.corr,
error_body,
)
.unwrap(),
)
.await
);
let result = task.await.unwrap();
assert!(matches!(result, Err(CallError::NotSent(_))));
assert!(rx.try_recv().is_err(), "one-shot route.open must not retry");
}
async fn open_admitted_route_for_test(
opts: Option<CallOptions>,
) -> (
Arc<Shared>,
SubcConsumer,
ClientControlRequest,
Result<RouteHandle, CallError>,
mpsc::Receiver<WriteCommand>,
) {
let shared = writer_test_shared();
let (writer, mut rx) = mpsc::channel(8);
let generation = {
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.generation
};
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let target = RouteTarget::ToolProvider {
module_id: "cerebellum".to_string(),
};
let identity = BindIdentity::new(
PathBuf::from("/tmp/project"),
"test".to_string(),
"admitted".to_string(),
);
let facts = serde_json::json!({"schema": 1, "verified_class": "member"});
let task = tokio::spawn(async move {
let result = match opts {
Some(opts) => {
consumer
.open_route_with_admission_facts_and_options(target, identity, facts, opts)
.await
}
None => {
consumer
.open_route_with_admission_facts(target, identity, facts)
.await
}
};
(consumer, result)
});
let command = rx.recv().await.expect("one route.open must be queued");
let request: ClientControlRequest = serde_json::from_slice(&command.frame.body).unwrap();
let response = serde_json::to_vec(&ClientControlResponse::RouteOpen {
route_channel: 21,
route_epoch: 4,
})
.unwrap();
assert!(
dispatch_frame(
&shared,
generation,
Frame::build(
FrameType::Response,
Flags::new(false, Priority::Interactive, false),
0,
0,
command.frame.header.corr,
response,
)
.unwrap(),
)
.await
);
let (consumer, result) = tokio::time::timeout(Duration::from_secs(1), task)
.await
.expect("admitted route.open resolves")
.unwrap();
(shared, consumer, request, result, rx)
}
#[tokio::test]
async fn admitted_route_open_refusal_keeps_the_daemon_code_and_detail() {
let shared = writer_test_shared();
let (writer, mut rx) = mpsc::channel(8);
let generation = {
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.generation
};
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let task = tokio::spawn(async move {
let result = consumer
.open_route_with_admission_facts(
RouteTarget::ToolProvider {
module_id: "cerebellum".to_string(),
},
BindIdentity::new(
PathBuf::from("/tmp/project"),
"test".to_string(),
"admitted".to_string(),
),
serde_json::json!({"schema": 1, "verified_class": "member"}),
)
.await;
(consumer, result)
});
let command = rx.recv().await.expect("one route.open must be queued");
let refusal = ErrorBody {
code: "module_warming".to_string(),
message: "cerebellum is not ready".to_string(),
detail: Some(serde_json::json!({"reason": "declared_not_ready"})),
};
assert!(
dispatch_frame(
&shared,
generation,
Frame::build(
FrameType::Error,
Flags::new(false, Priority::Interactive, false),
0,
0,
command.frame.header.corr,
serde_json::to_vec(&refusal).unwrap(),
)
.unwrap(),
)
.await
);
let (_consumer, result) = tokio::time::timeout(Duration::from_secs(1), task)
.await
.expect("admitted route.open resolves")
.unwrap();
let error = result.expect_err("a refused open is an error");
assert!(
error.is_not_sent(),
"a refused open never reached the module: {error}"
);
let body = error
.route_open_refusal()
.unwrap_or_else(|| panic!("the refusal's code must survive: {error}"));
assert_eq!(body.code, "module_warming");
assert_eq!(
body.detail,
Some(serde_json::json!({"reason": "declared_not_ready"}))
);
assert!(
error.to_string().contains("tool_provider:cerebellum"),
"the error names its target: {error}"
);
}
#[tokio::test]
async fn admitted_route_without_options_opens_and_declares_no_reverse_capabilities() {
let (_shared, _consumer, request, result, _rx) = open_admitted_route_for_test(None).await;
let ClientControlRequest::RouteOpen {
consumer_capabilities,
admission_facts,
..
} = request
else {
panic!("expected route.open")
};
assert_eq!(consumer_capabilities, None);
assert!(admission_facts.is_some());
let handle = result.expect("admitted route opens");
assert_eq!((handle.channel, handle.epoch), (21, 4));
}
#[tokio::test]
async fn admitted_route_with_options_delivers_elicitation_to_its_handler() {
let opts = CallOptions::default();
let seen = Arc::new(Mutex::new(None));
let seen_by_handler = Arc::clone(&seen);
let reply_body = br#"{"jsonrpc":"2.0","id":3,"result":{"action":"accept"}}"#.to_vec();
let expected_reply = reply_body.clone();
opts.reverse_requests
.on_request("elicitation", move |body, ctx| {
*seen_by_handler
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()) = Some((body, ctx.corr));
let reply_body = reply_body.clone();
async move { reply_body }
})
.unwrap();
let (shared, _consumer, request, result, mut rx) =
open_admitted_route_for_test(Some(opts)).await;
let ClientControlRequest::RouteOpen {
consumer_capabilities,
admission_facts,
..
} = request
else {
panic!("expected route.open")
};
assert_eq!(consumer_capabilities, Some(vec!["elicitation".to_string()]));
assert_eq!(
admission_facts,
Some(serde_json::json!({"schema": 1, "verified_class": "member"}))
);
let handle = result.expect("admitted route opens");
let request_body =
br#"{"jsonrpc":"2.0","id":3,"method":"elicitation/create","params":{}}"#.to_vec();
assert!(
dispatch_frame(
&shared,
handle.connection_token(),
reverse_request_frame(handle, 900, request_body.clone()),
)
.await
);
let reply = tokio::time::timeout(Duration::from_secs(1), rx.recv())
.await
.expect("the handler replies")
.unwrap()
.frame;
assert_eq!(reply.header.ty, FrameType::Response);
assert_eq!(reply.header.corr, 900);
assert_eq!(reply.header.channel, 21);
assert_eq!(reply.body, expected_reply);
assert_eq!(
seen.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone(),
Some((request_body, 900))
);
}
#[tokio::test]
async fn route_open_waiter_deadline_is_not_sent_without_writing() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(4);
let target = RouteTarget::ToolProvider {
module_id: "single-flight".to_string(),
};
let identity = BindIdentity::new(
PathBuf::from("/tmp/project"),
"test".to_string(),
"route-open".to_string(),
);
let opts = CallOptions {
timeout: Duration::from_millis(25),
consumer_identity: Some(ConsumerIdentity {
module_id: "caller".to_string(),
launch_nonce: "nonce".to_string(),
}),
..CallOptions::default()
};
let key = RouteKey::new(&target, &identity, opts.consumer_identity.as_ref(), None);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.openings.insert(
key,
Opening {
waiters: Vec::new(),
closed: false,
},
);
}
let consumer = SubcConsumer { shared };
let result = tokio::time::timeout(
Duration::from_millis(250),
consumer.open_route(target, identity, opts),
)
.await
.expect("a route.open waiter must finish at its own deadline")
.unwrap_err();
assert!(matches!(result, CallError::NotSent(_)));
assert!(
rx.try_recv().is_err(),
"a timed-out route.open waiter must not write a control frame"
);
}
async fn managed_op_with_unanswered_route_open(subscribe: bool) -> CallError {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(4);
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
}
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let target = RouteTarget::ToolProvider {
module_id: "slow-route-open".to_string(),
};
let identity = BindIdentity::new(
PathBuf::from("/tmp/project"),
"test".to_string(),
"slow-route-open".to_string(),
);
let task = tokio::spawn(async move {
if subscribe {
consumer
.subscribe(
target,
identity,
b"{}".to_vec(),
SubscribeOptions {
route_open_timeout: Duration::from_millis(200),
route_retry_deadline: Duration::from_millis(100),
..SubscribeOptions::default()
},
)
.await
.map(|_| ())
} else {
consumer
.call(
target,
identity,
b"{}".to_vec(),
CallOptions {
timeout: Duration::from_millis(200),
route_retry_deadline: Duration::from_millis(100),
..CallOptions::default()
},
)
.await
.map(|_| ())
}
});
let command = rx.recv().await.expect("route.open must be queued");
assert_eq!(command.frame.header.channel, 0);
let request: ClientControlRequest = serde_json::from_slice(&command.frame.body).unwrap();
assert!(matches!(request, ClientControlRequest::RouteOpen { .. }));
assert!(shared.mark_pending_accepted(command.pending.expect("route.open is tracked")));
let err = tokio::time::timeout(Duration::from_secs(2), task)
.await
.expect("the managed op must settle at its own deadline")
.unwrap()
.expect_err("an unanswered route.open cannot yield a route");
while let Ok(command) = rx.try_recv() {
assert_eq!(
command.frame.header.channel, 0,
"no data-plane frame may be written without an open route"
);
}
err
}
#[tokio::test]
async fn call_whose_route_open_times_out_after_send_is_not_sent() {
let err = managed_op_with_unanswered_route_open(false).await;
assert!(
matches!(err, CallError::NotSent(_)),
"call body never left the client, got {err:?}"
);
}
#[tokio::test]
async fn subscribe_whose_route_open_times_out_after_send_is_not_sent() {
let err = managed_op_with_unanswered_route_open(true).await;
assert!(
matches!(err, CallError::NotSent(_)),
"subscription request never left the client, got {err:?}"
);
}
#[tokio::test]
async fn route_poll_deadline_bounds_writer_capacity() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(1);
let handle = RouteHandle::new(8, 4, 1);
writer
.try_send(WriteCommand {
frame: response_frame(0, 0, 99, Vec::new()),
pending: None,
})
.expect("the fake daemon queue should accept its filler frame");
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.route_epochs.insert(handle.channel, handle);
}
let consumer = SubcConsumer { shared };
let result = tokio::time::timeout(
Duration::from_millis(250),
consumer.poll_route(&handle, PollKind::Liveness, Duration::from_millis(25)),
)
.await
.expect("a control request must finish at its own deadline")
.unwrap_err();
assert!(matches!(result, CallError::NotSent(_)));
assert_eq!(
rx.recv()
.await
.expect("the filler must still be the only queued frame")
.frame
.header
.corr,
99
);
assert!(
rx.try_recv().is_err(),
"the timed-out control request must not reach the fake daemon"
);
}
#[tokio::test]
async fn subscription_deadline_bounds_writer_capacity() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(1);
let handle = RouteHandle::new(9, 2, 1);
let route_sem = Arc::new(Semaphore::new(1));
writer
.try_send(WriteCommand {
frame: response_frame(0, 0, 100, Vec::new()),
pending: None,
})
.expect("the fake daemon queue should accept its filler frame");
{
let mut inner = shared.lock_inner();
inner.writer = Some(writer);
inner.cache_route(
RouteKey::new(
&RouteTarget::ToolProvider {
module_id: "subscriptions".to_string(),
},
&BindIdentity::new(
PathBuf::from("/tmp/project"),
"test".to_string(),
"subscription".to_string(),
),
None,
None,
),
RouteState {
handle,
sem: Arc::clone(&route_sem),
},
);
}
let consumer = SubcConsumer { shared };
let result = tokio::time::timeout(
Duration::from_millis(250),
consumer.subscribe_route(
&handle,
b"subscribe".to_vec(),
SubscribeOptions {
route_open_timeout: Duration::from_millis(25),
..SubscribeOptions::default()
},
),
)
.await
.expect("a subscription must finish at its route-open deadline");
let result = match result {
Ok(_) => panic!("a subscription blocked before writing must time out"),
Err(err) => err,
};
assert!(matches!(result, CallError::NotSent(_)));
assert_eq!(
rx.recv()
.await
.expect("the filler must still be the only queued frame")
.frame
.header
.corr,
100
);
assert!(
rx.try_recv().is_err(),
"the timed-out subscription must not reach the fake daemon"
);
assert!(
route_sem.try_acquire().is_ok(),
"a pre-write subscription timeout must release its route credit"
);
}
#[test]
fn catalog_list_deserializes_golden_reply_and_ignores_unknown_fields() {
let mut reply: serde_json::Value = serde_json::from_str(include_str!(
"../../subc-control/tests/golden/client_control_response_catalog_list.json"
))
.expect("the catalog.list golden reply must be valid JSON");
reply["future_top_level"] = serde_json::json!(true);
reply["modules"][0]["future_module_field"] = serde_json::json!("ignored");
let catalog: CatalogList =
serde_json::from_value(reply).expect("catalog.list should tolerate additive fields");
assert_eq!(catalog.generation, 7);
assert_eq!(catalog.modules.len(), 1);
assert!(catalog.subc_ops.iter().any(|op| op == "catalog.list"));
let tools = catalog.modules[0]
.roles
.iter()
.find_map(|role| match role {
subc_protocol::manifest::ProviderRole::ToolProvider { tools, .. } => Some(tools),
_ => None,
})
.expect("the golden module must advertise a tool_provider role");
let tool = tools
.first()
.expect("the golden tool_provider role must advertise a tool");
assert!(!tool.name.is_empty());
assert_eq!(
tool.schema.get("type").and_then(serde_json::Value::as_str),
Some("object")
);
assert!(matches!(
tool.execution_mode,
subc_protocol::manifest::ExecutionMode::Pure
));
}
#[tokio::test]
async fn catalog_list_sends_an_unfiltered_channel_zero_request() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(1);
shared.lock_inner().writer = Some(writer);
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let request = tokio::spawn(async move { consumer.catalog_list().await });
let command = rx
.recv()
.await
.expect("catalog.list must queue a channel-0 request");
assert_eq!(command.frame.header.channel, 0);
let body: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
assert_eq!(body["op"], "catalog.list");
assert!(
body.get("module_id").is_none(),
"catalog.list must request the complete catalog without a module filter"
);
let response = serde_json::to_vec(&ClientControlResponse::CatalogList {
generation: 9,
modules: Vec::new(),
subc_ops: vec!["catalog.list".to_string()],
})
.unwrap();
assert!(
dispatch_frame(
&shared,
1,
response_frame(0, 0, command.frame.header.corr, response),
)
.await
);
let catalog = request.await.unwrap().unwrap();
assert_eq!(catalog.generation, 9);
assert!(catalog.modules.is_empty());
}
#[tokio::test]
async fn spawn_snapshot_sends_a_channel_zero_request_and_returns_the_snapshot() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(1);
shared.lock_inner().writer = Some(writer);
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let request = tokio::spawn(async move { consumer.spawn_snapshot().await });
let command = rx
.recv()
.await
.expect("supervisor.spawn_snapshot must queue a channel-0 request");
assert_eq!(command.frame.header.channel, 0);
let body: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
assert_eq!(body["op"], "supervisor.spawn_snapshot");
let snapshot = SpawnSnapshot {
cursor: SpawnCursor {
daemon_incarnation: "incarnation-a".to_string(),
seq: 41,
},
ring_bound: 256,
live: vec![LiveSpawn {
module_id: "participant".to_string(),
spawn_generation: 3,
pid: 4242,
spawned_at_ms: 1_700_000_000_000,
}],
};
let response = serde_json::to_vec(&ClientControlResponse::SupervisorSpawnSnapshot {
snapshot: snapshot.clone(),
})
.unwrap();
assert!(
dispatch_frame(
&shared,
1,
response_frame(0, 0, command.frame.header.corr, response),
)
.await
);
assert_eq!(request.await.unwrap().unwrap(), snapshot);
}
#[tokio::test]
async fn spawn_snapshot_refusal_keeps_the_daemon_code() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, mut rx) = mpsc::channel(1);
shared.lock_inner().writer = Some(writer);
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let request = tokio::spawn(async move { consumer.spawn_snapshot().await });
let command = rx
.recv()
.await
.expect("supervisor.spawn_snapshot must queue a channel-0 request");
let refusal = serde_json::to_vec(&ErrorBody {
code: "op_not_permitted".to_string(),
message: "refused by the test".to_string(),
detail: None,
})
.unwrap();
let frame = Frame::build(
FrameType::Error,
Flags::new(false, Priority::Interactive, false),
0,
0,
command.frame.header.corr,
refusal,
)
.unwrap();
assert!(dispatch_frame(&shared, 1, frame).await);
let error = request.await.unwrap().unwrap_err();
assert_eq!(error.code(), Some("op_not_permitted"));
}
fn spawn_cursor(seq: u64) -> SpawnCursor {
SpawnCursor {
daemon_incarnation: "incarnation-a".to_string(),
seq,
}
}
fn spawn_event(seq: u64, kind: SpawnEventKind, generation: u64) -> SpawnEvent {
SpawnEvent {
cursor: spawn_cursor(seq),
kind,
module_id: "participant".to_string(),
spawn_generation: generation,
pid: 4242,
exit_code: (kind == SpawnEventKind::Exited).then_some(0),
exit_signal: None,
}
}
fn channel_zero_frame(ty: FrameType, corr: u64, body: Vec<u8>) -> Frame {
Frame::build(
ty,
Flags::new(false, Priority::Interactive, false),
0,
0,
corr,
body,
)
.unwrap()
}
fn spawn_error_frame(corr: u64, code: &str, detail: serde_json::Value) -> Frame {
channel_zero_frame(
FrameType::Error,
corr,
serde_json::to_vec(&ErrorBody {
code: code.to_string(),
message: "sent by the test".to_string(),
detail: Some(detail),
})
.unwrap(),
)
}
async fn spawn_subscribed(
since: Option<SpawnCursor>,
) -> (
Arc<Shared>,
mpsc::Receiver<WriteCommand>,
SpawnSubscription,
Frame,
) {
let shared = writer_test_shared();
let (writer, mut rx) = mpsc::channel(8);
shared.lock_inner().writer = Some(writer);
let consumer = SubcConsumer {
shared: Arc::clone(&shared),
};
let subscription = consumer
.spawn_subscribe(since)
.await
.expect("supervisor.spawn_subscribe must be sent");
std::mem::forget(consumer);
let command = rx
.recv()
.await
.expect("supervisor.spawn_subscribe must queue a channel-0 request");
(shared, rx, subscription, command.frame)
}
#[tokio::test]
async fn spawn_subscribe_sends_its_cursor_on_channel_zero_and_yields_events_until_stream_end() {
let (shared, _rx, mut subscription, request) =
spawn_subscribed(Some(spawn_cursor(7))).await;
assert_eq!(request.header.channel, 0);
assert_eq!(request.header.ty, FrameType::Request);
let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap();
assert_eq!(
body,
serde_json::json!({
"op": "supervisor.spawn_subscribe",
"since": {"daemon_incarnation": "incarnation-a", "seq": 7},
})
);
let corr = request.header.corr;
let events = [
spawn_event(8, SpawnEventKind::Exited, 3),
spawn_event(9, SpawnEventKind::Spawned, 4),
];
for event in &events {
let body = serde_json::to_vec(event).unwrap();
assert!(
dispatch_frame(
&shared,
1,
channel_zero_frame(FrameType::StreamData, corr, body)
)
.await
);
}
assert!(
dispatch_frame(
&shared,
1,
channel_zero_frame(FrameType::StreamEnd, corr, Vec::new())
)
.await
);
assert_eq!(subscription.next().await.unwrap(), Some(events[0].clone()));
assert_eq!(subscription.next().await.unwrap(), Some(events[1].clone()));
assert_eq!(subscription.next().await.unwrap(), None);
assert_eq!(
subscription.next().await.unwrap(),
None,
"a finished stream stays finished"
);
}
#[tokio::test]
async fn spawn_subscribe_without_a_cursor_omits_since() {
let (_shared, _rx, _subscription, request) = spawn_subscribed(None).await;
let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap();
assert_eq!(
body,
serde_json::json!({"op": "supervisor.spawn_subscribe"})
);
}
#[tokio::test]
async fn spawn_cursor_refusals_surface_as_coded_errors_with_their_detail() {
let (shared, _rx, mut subscription, request) =
spawn_subscribed(Some(spawn_cursor(7))).await;
let frame = spawn_error_frame(
request.header.corr,
"spawn_cursor_incarnation_mismatch",
serde_json::json!({"current_daemon_incarnation": "incarnation-b"}),
);
assert!(dispatch_frame(&shared, 1, frame).await);
let error = subscription.next().await.unwrap_err();
assert_eq!(error.code(), Some("spawn_cursor_incarnation_mismatch"));
let SpawnStreamError::CursorIncarnationMismatch {
current_daemon_incarnation,
body,
} = &error
else {
panic!("expected the incarnation refusal, got {error:?}");
};
assert_eq!(current_daemon_incarnation, "incarnation-b");
assert_eq!(body.code, "spawn_cursor_incarnation_mismatch");
assert_eq!(subscription.next().await.unwrap(), None);
let (shared, _rx, mut subscription, request) =
spawn_subscribed(Some(spawn_cursor(1))).await;
let frame = spawn_error_frame(
request.header.corr,
"spawn_cursor_too_old",
serde_json::json!({"oldest_retained_cursor": spawn_cursor(90)}),
);
assert!(dispatch_frame(&shared, 1, frame).await);
let error = subscription.next().await.unwrap_err();
assert_eq!(error.code(), Some("spawn_cursor_too_old"));
let SpawnStreamError::CursorTooOld {
oldest_retained_cursor,
..
} = &error
else {
panic!("expected the too-old refusal, got {error:?}");
};
assert_eq!(oldest_retained_cursor, &spawn_cursor(90));
}
#[tokio::test]
async fn a_lagged_spawn_stream_yields_its_queued_events_then_the_lag_error() {
let (shared, _rx, mut subscription, request) =
spawn_subscribed(Some(spawn_cursor(7))).await;
let corr = request.header.corr;
let queued = spawn_event(8, SpawnEventKind::Exited, 3);
assert!(
dispatch_frame(
&shared,
1,
channel_zero_frame(
FrameType::StreamData,
corr,
serde_json::to_vec(&queued).unwrap()
),
)
.await
);
let frame = spawn_error_frame(
corr,
"spawn_subscriber_lagged",
serde_json::json!({"first_undelivered_cursor": spawn_cursor(9)}),
);
assert!(dispatch_frame(&shared, 1, frame).await);
assert_eq!(subscription.next().await.unwrap(), Some(queued));
let error = subscription.next().await.unwrap_err();
assert_eq!(error.code(), Some("spawn_subscriber_lagged"));
let SpawnStreamError::SubscriberLagged {
first_undelivered_cursor,
..
} = &error
else {
panic!("expected the lag error, got {error:?}");
};
assert_eq!(first_undelivered_cursor, &spawn_cursor(9));
}
#[tokio::test]
async fn dropping_a_spawn_subscription_sends_a_channel_zero_cancel_for_its_corr() {
let (_shared, mut rx, subscription, request) =
spawn_subscribed(Some(spawn_cursor(7))).await;
drop(subscription);
let cancel = tokio::time::timeout(Duration::from_secs(1), rx.recv())
.await
.expect("dropping the subscription must queue a Cancel")
.expect("the writer stays open");
assert_eq!(cancel.frame.header.ty, FrameType::Cancel);
assert_eq!(cancel.frame.header.channel, 0);
assert_eq!(cancel.frame.header.corr, request.header.corr);
}
#[tokio::test]
async fn reconnect_completion_between_wait_decision_and_registration_wakes_caller() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/subc-client-rs-reconnect-wakeup"),
ConsumerOptions::default(),
));
let (sender, _receiver) = mpsc::channel(EGRESS_BUFFER);
{
let mut inner = shared.lock_inner();
inner.reconnect = ReconnectState::Inline {
generation: inner.generation,
};
}
let completing = Arc::clone(&shared);
*shared.before_wait.lock().unwrap() = Some(Box::new(move || {
completing.lock_inner().writer = Some(sender);
completing.notify.notify_waiters();
}));
let result = shared
.ensure_connected_for_call(Instant::now() + Duration::from_millis(30))
.await;
assert!(
result.is_ok(),
"connected caller should not time out: {result:?}"
);
assert!(shared.before_wait.lock().unwrap().is_none());
}
#[tokio::test]
async fn catalog_list_deadline_is_not_sent_when_reconnection_stays_down() {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/subc-client-rs-catalog-list-unavailable"),
ConsumerOptions {
call_timeout: Duration::from_millis(25),
reconnect_backoff: RetryBackoff {
base: Duration::from_millis(1),
cap: Duration::from_millis(1),
max_attempts: 100,
},
..ConsumerOptions::default()
},
));
let consumer = SubcConsumer { shared };
let result = tokio::time::timeout(Duration::from_millis(250), consumer.catalog_list())
.await
.expect("catalog.list must finish at its configured deadline")
.unwrap_err();
assert!(matches!(result, CallError::NotSent(_)));
}
}
#[cfg(test)]
mod launch_nonce_redaction_tests {
use super::*;
const NONCE: &str = "nonce-f00dfeed1234abcd";
#[test]
fn option_types_and_route_keys_never_print_the_nonce() {
let identity = ConsumerIdentity {
module_id: "wernicke".to_string(),
launch_nonce: NONCE.to_string(),
};
let call = CallOptions {
consumer_identity: Some(identity.clone()),
..CallOptions::default()
};
let key = ConsumerIdentityKey::from(&identity);
for printed in [format!("{call:?}"), format!("{key:?}")] {
assert!(!printed.contains(NONCE), "launch nonce printed: {printed}");
}
}
}