use std::collections::{BTreeMap, HashMap, VecDeque};
use std::future::pending;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;
use tokio::io::{AsyncRead, AsyncWrite};
use tokio::sync::{broadcast, mpsc, oneshot};
use tokio::task::JoinHandle;
use tokio::time::{Instant, sleep, sleep_until, timeout};
use crate::common::config::{
ConfigurationOpenApi, DEFAULT_BACKOFF, DEFAULT_RECONNECT_DELAY, DEFAULT_RETRIES,
DEFAULT_TIMEOUT,
};
use crate::common::errors::{
ConnectorError, ErrorCode, Interruption, ServerError, SessionEndCause, SessionEnded,
TransportError,
};
use crate::common::ids::AccountId;
use crate::open_api::messages::{
ProtoErrorRes, ProtoHeartbeatEvent, ProtoMessage, ProtoOaAccountAuthReq, ProtoOaAccountAuthRes,
ProtoOaApplicationAuthReq, ProtoOaApplicationAuthRes, ProtoOaErrorRes,
};
use crate::open_api::payload::{Payload, Request, account_of, is_historical, sessions_ended_by};
use crate::open_api::transport::{Connect, Framed};
pub const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(10);
pub const DEFAULT_COMMAND_CAPACITY: usize = 64;
pub const DEFAULT_EVENT_CAPACITY: usize = 1024;
pub const MAX_COMMAND_CAPACITY: usize = 65_536;
pub const MAX_EVENT_CAPACITY: usize = 65_536;
pub const REQUEST_LIMIT: u32 = 50;
pub const HISTORICAL_LIMIT: u32 = 5;
pub const BACKOFF_LIMIT: Duration = Duration::from_secs(30);
const LIMIT_WINDOW: Duration = Duration::from_millis(1100);
pub const RECONNECT_DELAY_LIMIT: Duration = Duration::from_secs(60);
const SHORTEST_INTERVAL: Duration = Duration::from_millis(1);
pub const SERVER_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30);
pub const DEFAULT_INBOUND_TIMEOUT: Duration = Duration::from_secs(75);
const LONGEST_WAIT: Duration = Duration::from_secs(30 * 365 * 86_400);
macro_rules! log_debug {
($($argument:tt)*) => {{
#[cfg(feature = "logging")]
tracing::debug!($($argument)*);
}};
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[must_use]
pub struct ConnectionOptions {
timeout: Duration,
heartbeat_interval: Duration,
command_capacity: usize,
event_capacity: usize,
reconnect_delay: Duration,
reconnect_attempts: Option<u32>,
request_limit: u32,
historical_limit: u32,
retries: u32,
backoff: Duration,
inbound_timeout: Option<Duration>,
}
impl Default for ConnectionOptions {
fn default() -> Self {
Self {
timeout: DEFAULT_TIMEOUT,
heartbeat_interval: HEARTBEAT_INTERVAL,
command_capacity: DEFAULT_COMMAND_CAPACITY,
event_capacity: DEFAULT_EVENT_CAPACITY,
reconnect_delay: DEFAULT_RECONNECT_DELAY,
reconnect_attempts: None,
request_limit: REQUEST_LIMIT,
historical_limit: HISTORICAL_LIMIT,
retries: DEFAULT_RETRIES,
backoff: DEFAULT_BACKOFF,
inbound_timeout: None,
}
}
}
impl From<&ConfigurationOpenApi> for ConnectionOptions {
fn from(configuration: &ConfigurationOpenApi) -> Self {
Self::default()
.timeout(configuration.timeout())
.reconnect_delay(configuration.reconnect_delay())
.retries(configuration.retries())
.backoff(configuration.backoff())
}
}
impl ConnectionOptions {
pub fn new() -> Self {
Self::default()
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout.max(SHORTEST_INTERVAL);
self
}
pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
self.heartbeat_interval = interval.max(SHORTEST_INTERVAL);
self
}
pub fn command_capacity(mut self, capacity: usize) -> Self {
self.command_capacity = capacity.clamp(1, MAX_COMMAND_CAPACITY);
self
}
pub fn event_capacity(mut self, capacity: usize) -> Self {
self.event_capacity = capacity.clamp(1, MAX_EVENT_CAPACITY);
self
}
pub fn reconnect_delay(mut self, delay: Duration) -> Self {
self.reconnect_delay = delay.max(SHORTEST_INTERVAL);
self
}
pub const fn reconnect_attempts(mut self, attempts: Option<u32>) -> Self {
self.reconnect_attempts = attempts;
self
}
pub fn request_limit(mut self, limit: u32) -> Self {
self.request_limit = limit.max(1);
self
}
pub fn historical_limit(mut self, limit: u32) -> Self {
self.historical_limit = limit.max(1);
self
}
pub const fn retries(mut self, retries: u32) -> Self {
self.retries = retries;
self
}
pub fn backoff(mut self, backoff: Duration) -> Self {
self.backoff = backoff.max(SHORTEST_INTERVAL);
self
}
pub fn inbound_timeout(mut self, period: Duration) -> Self {
self.inbound_timeout = Some(period.max(SHORTEST_INTERVAL));
self
}
fn silence(&self) -> Duration {
self.inbound_timeout.unwrap_or(DEFAULT_INBOUND_TIMEOUT)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum Event {
Message(ProtoMessage),
Disconnected(TransportError),
ReconnectFailed {
attempt: u32,
reason: TransportError,
},
Reconnected {
attempt: u32,
},
ReplayFailed {
key: ReplayKey,
error: ConnectorError,
},
SessionEnded(SessionEnded),
}
impl Event {
#[must_use]
pub fn decode<M: Payload>(&self) -> Option<M> {
match self {
Self::Message(message) => M::from_message(message).ok(),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub struct Stats {
pub requests_sent: u64,
pub heartbeats_sent: u64,
pub heartbeats_received: u64,
pub reconnects: u64,
pub throttle_retries: u64,
}
#[derive(Debug, Default)]
struct Counters {
requests_sent: AtomicU64,
heartbeats_sent: AtomicU64,
heartbeats_received: AtomicU64,
reconnects: AtomicU64,
throttle_retries: AtomicU64,
}
fn bump(counter: &AtomicU64) {
counter.fetch_add(1, Ordering::Relaxed);
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[non_exhaustive]
pub enum ReplayStage {
Application,
Account,
Subscription,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ReplayKey {
stage: ReplayStage,
id: String,
}
impl ReplayKey {
#[must_use]
pub fn new(stage: ReplayStage, id: impl Into<String>) -> Self {
Self {
stage,
id: id.into(),
}
}
#[must_use]
pub const fn stage(&self) -> ReplayStage {
self.stage
}
#[must_use]
pub fn id(&self) -> &str {
&self.id
}
}
#[derive(Clone, PartialEq, Eq)]
struct Stored {
payload_type: u32,
payload: Vec<u8>,
}
type Replays = BTreeMap<ReplayKey, Stored>;
type Reply = oneshot::Sender<Result<ProtoMessage, ConnectorError>>;
struct Outbound {
payload_type: u32,
payload: Vec<u8>,
historical: bool,
once: bool,
attempts: u32,
reply: Reply,
}
impl Outbound {
fn interrupted(&self, by: Interruption) -> ConnectorError {
if self.once {
ConnectorError::OutcomeUnknown(by)
} else {
match by {
Interruption::Timeout(limit) => ConnectorError::Timeout(limit),
Interruption::Transport(reason) => ConnectorError::Transport(reason),
Interruption::Skipped => ConnectorError::OutcomeUnknown(by),
}
}
}
}
#[derive(Default)]
struct Ended {
application: Option<SessionEndCause>,
accounts: BTreeMap<AccountId, SessionEndCause>,
}
impl Ended {
fn refusal(&self, payload_type: u32, payload: &[u8]) -> Option<SessionEnded> {
if payload_type == ProtoOaApplicationAuthReq::PAYLOAD_TYPE {
return None;
}
if let Some(cause) = &self.application {
return Some(SessionEnded::new(None, cause.clone()));
}
if payload_type == ProtoOaAccountAuthReq::PAYLOAD_TYPE {
return None;
}
let account = account_of(payload_type, payload)?;
let cause = self.accounts.get(&account)?;
Some(SessionEnded::new(Some(account), cause.clone()))
}
fn granted(&mut self, answer: &ProtoMessage) {
if answer.payload_type == ProtoOaApplicationAuthRes::PAYLOAD_TYPE {
self.application = None;
} else if answer.payload_type == ProtoOaAccountAuthRes::PAYLOAD_TYPE {
let payload = answer.payload.as_deref().unwrap_or_default();
if let Some(account) = account_of(answer.payload_type, payload) {
self.accounts.remove(&account);
}
}
}
}
fn token_refused(key: &ReplayKey, stored: &Stored, error: &ConnectorError) -> Option<SessionEnded> {
let ConnectorError::Server(error) = error else {
return None;
};
let dead = matches!(
error.code,
ErrorCode::ChAccessTokenInvalid | ErrorCode::OaAuthTokenExpired
);
if !dead || key.stage() != ReplayStage::Account {
return None;
}
let account = account_of(stored.payload_type, &stored.payload)?;
Some(SessionEnded::new(
Some(account),
SessionEndCause::TokenRefused(error.code.clone()),
))
}
enum Command {
Request(Outbound),
Replay(ReplayKey, Option<Stored>),
Reconnect,
Shutdown,
}
#[derive(Debug, Clone)]
pub struct Connection {
commands: mpsc::Sender<Command>,
events: broadcast::Sender<Event>,
counters: Arc<Counters>,
task: Arc<Mutex<Option<JoinHandle<()>>>>,
}
impl Connection {
pub async fn open<C: Connect>(
connector: C,
host: &str,
port: u16,
options: ConnectionOptions,
) -> Result<Self, ConnectorError> {
let stream = connector.connect(host, port).await?;
log_debug!(host, port, "connected");
let (commands, receiver) = mpsc::channel(options.command_capacity);
let (events, _) = broadcast::channel(options.event_capacity);
let counters = Arc::new(Counters::default());
let actor = Actor {
connector,
host: host.to_owned(),
port,
heartbeat_at: within(Instant::now(), options.heartbeat_interval),
silent_at: within(Instant::now(), options.silence()),
options,
commands: receiver,
events: events.clone(),
counters: Arc::clone(&counters),
queue: VecDeque::new(),
pending: HashMap::new(),
deadlines: VecDeque::new(),
replays: Replays::new(),
ended: Ended::default(),
throttles: [Throttle::default(), Throttle::default()],
paused_until: None,
send_at: None,
next_id: 0,
};
let task = tokio::spawn(actor.run(Framed::new(stream)));
Ok(Self {
commands,
events,
counters,
task: Arc::new(Mutex::new(Some(task))),
})
}
pub async fn request<R: Request>(&self, request: &R) -> Result<R::Response, ConnectorError> {
let message = self
.send(
R::PAYLOAD_TYPE,
request.encode_to_vec(),
R::HISTORICAL,
false,
)
.await?;
R::Response::from_message(&message)
}
pub async fn request_message(
&self,
payload_type: u32,
payload: Vec<u8>,
) -> Result<ProtoMessage, ConnectorError> {
self.send(payload_type, payload, is_historical(payload_type), false)
.await
}
pub async fn request_once(
&self,
payload_type: u32,
payload: Vec<u8>,
) -> Result<ProtoMessage, ConnectorError> {
self.send(payload_type, payload, is_historical(payload_type), true)
.await
}
async fn send(
&self,
payload_type: u32,
payload: Vec<u8>,
historical: bool,
once: bool,
) -> Result<ProtoMessage, ConnectorError> {
let (reply, response) = oneshot::channel();
let outbound = Outbound {
payload_type,
payload,
reply,
historical,
once,
attempts: 0,
};
self.commands
.send(Command::Request(outbound))
.await
.map_err(|_| ConnectorError::Closed)?;
response.await.map_err(|_| ConnectorError::Closed)?
}
pub async fn register_replay<R: Request>(
&self,
key: ReplayKey,
request: &R,
) -> Result<(), ConnectorError> {
let stored = Stored {
payload_type: R::PAYLOAD_TYPE,
payload: request.encode_to_vec(),
};
self.command(Command::Replay(key, Some(stored))).await
}
pub async fn remove_replay(&self, key: ReplayKey) -> Result<(), ConnectorError> {
self.command(Command::Replay(key, None)).await
}
pub async fn force_reconnect(&self) -> Result<(), ConnectorError> {
self.command(Command::Reconnect).await
}
async fn command(&self, command: Command) -> Result<(), ConnectorError> {
self.commands
.send(command)
.await
.map_err(|_| ConnectorError::Closed)
}
#[must_use]
pub fn events(&self) -> broadcast::Receiver<Event> {
self.events.subscribe()
}
pub async fn closed(&self) {
self.commands.closed().await;
}
#[must_use]
pub fn stats(&self) -> Stats {
Stats {
requests_sent: self.counters.requests_sent.load(Ordering::Relaxed),
heartbeats_sent: self.counters.heartbeats_sent.load(Ordering::Relaxed),
heartbeats_received: self.counters.heartbeats_received.load(Ordering::Relaxed),
reconnects: self.counters.reconnects.load(Ordering::Relaxed),
throttle_retries: self.counters.throttle_retries.load(Ordering::Relaxed),
}
}
pub async fn shutdown(&self) {
let _ = self.commands.send(Command::Shutdown).await;
let task = self
.task
.lock()
.unwrap_or_else(PoisonError::into_inner)
.take();
if let Some(task) = task {
let _ = task.await;
}
}
}
#[derive(Debug)]
enum Exit {
Shutdown,
Lost(TransportError),
}
struct Actor<C: Connect> {
connector: C,
host: String,
port: u16,
options: ConnectionOptions,
commands: mpsc::Receiver<Command>,
events: broadcast::Sender<Event>,
counters: Arc<Counters>,
queue: VecDeque<Outbound>,
pending: HashMap<String, Outbound>,
deadlines: VecDeque<(Option<Instant>, String)>,
replays: Replays,
ended: Ended,
throttles: [Throttle; 2],
paused_until: Option<Instant>,
send_at: Option<Instant>,
next_id: u64,
heartbeat_at: Option<Instant>,
silent_at: Option<Instant>,
}
impl<C: Connect> Actor<C> {
async fn run(mut self, mut framed: Framed<C::Stream>) {
let error = loop {
match self.serve(&mut framed).await {
Exit::Shutdown => break ConnectorError::Closed,
Exit::Lost(reason) => {
log_debug!(%reason, "connection lost");
close(&mut framed, self.options.timeout).await;
self.fail_in_flight(&reason);
let _ = self.events.send(Event::Disconnected(reason.clone()));
match self.reconnect(reason).await {
Ok(fresh) => framed = fresh,
Err(error) => break error,
}
}
}
};
self.fail_all(&error);
close(&mut framed, self.options.timeout).await;
}
async fn reconnect(
&mut self,
reason: TransportError,
) -> Result<Framed<C::Stream>, ConnectorError> {
let mut last = reason;
let mut delay = self.options.reconnect_delay;
let longest = RECONNECT_DELAY_LIMIT.max(delay);
let mut attempt = 0_u32;
loop {
if self
.options
.reconnect_attempts
.is_some_and(|limit| attempt >= limit)
{
return Err(ConnectorError::Transport(last));
}
attempt = attempt.saturating_add(1);
hold(&mut self.commands, &mut self.replays, &last, sleep(delay)).await?;
delay = delay.saturating_mul(2).min(longest);
match self.restore(&last).await? {
Ok((framed, replays_sent)) => {
log_debug!(attempt, "reconnected");
bump(&self.counters.reconnects);
let seeded = replays_sent
.min(usize::try_from(self.options.request_limit).unwrap_or(usize::MAX));
if let Some(throttle) = self.throttles.first_mut() {
throttle
.sent
.extend(std::iter::repeat_n(Instant::now(), seeded));
}
self.heartbeat_at = within(Instant::now(), self.options.heartbeat_interval);
self.silent_at = within(Instant::now(), self.options.silence());
let _ = self.events.send(Event::Reconnected { attempt });
return Ok(framed);
}
Err(reason) => {
log_debug!(attempt, %reason, "reconnection failed");
let _ = self.events.send(Event::ReconnectFailed {
attempt,
reason: reason.clone(),
});
last = reason;
}
}
}
}
async fn restore(
&mut self,
reason: &TransportError,
) -> Result<Result<(Framed<C::Stream>, usize), TransportError>, ConnectorError> {
let limit = self.options.timeout;
let spacing = LIMIT_WINDOW / self.options.request_limit;
let connect = self.connector.connect(&self.host, self.port);
let mut framed = match hold(&mut self.commands, &mut self.replays, reason, connect).await? {
Ok(stream) => Framed::new(stream),
Err(reason) => return Ok(Err(reason)),
};
let mut sent = Replays::new();
loop {
let next = self
.replays
.iter()
.find(|(key, stored)| sent.get(*key) != Some(*stored))
.map(|(key, stored)| (key.clone(), stored.clone()));
let Some((key, stored)) = next else {
return Ok(Ok((framed, sent.len())));
};
sent.insert(key.clone(), stored.clone());
let id = self.fresh_id();
let mut pushed = Vec::new();
let step = Box::pin(replay(
&mut framed,
id,
stored.clone(),
limit,
&self.events,
&mut pushed,
));
let answer = match hold(&mut self.commands, &mut self.replays, reason, step).await? {
Ok(answer) => answer,
Err(reason) => return Ok(Err(reason)),
};
for session in pushed {
self.end(session);
}
match answer {
Ok(answer) => self.ended.granted(&answer),
Err(error) => {
log_debug!(%error, "replayed request failed");
let refused = token_refused(&key, &stored, &error);
let _ = self.events.send(Event::ReplayFailed { key, error });
if let Some(session) = refused {
self.end(session);
}
}
}
hold(
&mut self.commands,
&mut self.replays,
reason,
sleep(spacing),
)
.await?;
}
}
async fn serve(&mut self, framed: &mut Framed<C::Stream>) -> Exit {
loop {
if let Err(reason) = self.flush(framed).await {
return Exit::Lost(reason);
}
let wake = self.next_wake();
let capacity = self.options.command_capacity;
tokio::select! {
incoming = framed.receive() => match incoming {
Ok(message) => self.heard(message),
Err(reason) => return Exit::Lost(reason),
},
command = self.commands.recv(), if self.queue.len() < capacity => match command {
Some(Command::Request(outbound)) => self.queue.push_back(outbound),
Some(Command::Replay(key, stored)) => store(&mut self.replays, key, stored),
Some(Command::Reconnect) => return Exit::Lost(TransportError::Closed),
Some(Command::Shutdown) | None => return Exit::Shutdown,
},
() = reached(wake) => {
if let Err(reason) = self.tick(framed).await {
return Exit::Lost(reason);
}
}
}
}
}
fn end(&mut self, session: SessionEnded) {
log_debug!(%session, "authorisation ended");
if let Some(account) = session.account {
self.replays.retain(|_, stored| {
account_of(stored.payload_type, &stored.payload) != Some(account)
});
self.ended.accounts.insert(account, session.cause.clone());
} else {
for stored in self.replays.values() {
if let Some(account) = account_of(stored.payload_type, &stored.payload) {
self.ended.accounts.insert(account, session.cause.clone());
}
}
self.replays.clear();
self.ended.application = Some(session.cause.clone());
}
let _ = self.events.send(Event::SessionEnded(session));
}
fn fresh_id(&mut self) -> String {
self.next_id = self.next_id.saturating_add(1);
self.next_id.to_string()
}
fn heard(&mut self, message: ProtoMessage) {
self.silent_at = within(Instant::now(), self.options.silence());
self.dispatch(message);
}
fn next_wake(&self) -> Option<Instant> {
let deadline = self.deadlines.front().and_then(|(deadline, _)| *deadline);
[self.heartbeat_at, self.silent_at, deadline, self.send_at]
.into_iter()
.flatten()
.min()
}
fn next_ready(&mut self, now: Instant) -> Option<usize> {
self.send_at = None;
if let Some(until) = self.paused_until {
if until > now {
if !self.queue.is_empty() {
self.send_at = Some(until);
}
return None;
}
self.paused_until = None;
}
let limits = [self.options.request_limit, self.options.historical_limit];
let mut ready = [None, None];
for ((throttle, limit), ready) in self.throttles.iter_mut().zip(limits).zip(&mut ready) {
*ready = throttle.ready_at(now, limit);
}
for (index, outbound) in self.queue.iter().enumerate() {
match ready
.get(usize::from(outbound.historical))
.copied()
.flatten()
{
None => return Some(index),
Some(at) => self.send_at = Some(self.send_at.map_or(at, |known| known.min(at))),
}
}
None
}
async fn flush(&mut self, framed: &mut Framed<C::Stream>) -> Result<(), TransportError> {
loop {
let now = Instant::now();
let Some(outbound) = self
.next_ready(now)
.and_then(|index| self.queue.remove(index))
else {
return Ok(());
};
if outbound.reply.is_closed() {
continue;
}
if let Some(ended) = self.ended.refusal(outbound.payload_type, &outbound.payload) {
let _ = outbound.reply.send(Err(ended.into()));
continue;
}
let id = self.fresh_id();
let message = ProtoMessage {
payload_type: outbound.payload_type,
payload: Some(outbound.payload.clone()),
client_msg_id: Some(id.clone()),
};
match write(framed, &message, self.options.timeout).await {
Ok(()) => {
bump(&self.counters.requests_sent);
if let Some(throttle) = self.throttles.get_mut(usize::from(outbound.historical))
{
throttle.sent.push_back(now);
}
self.heartbeat_at = within(now, self.options.heartbeat_interval);
self.deadlines
.push_back((within(now, self.options.timeout), id.clone()));
self.pending.insert(id, outbound);
}
Err(error @ TransportError::FrameTooLong { .. }) => {
let _ = outbound.reply.send(Err(error.into()));
}
Err(reason) => {
let error = outbound.interrupted(Interruption::Transport(reason.clone()));
let _ = outbound.reply.send(Err(error));
return Err(reason);
}
}
}
}
async fn tick(&mut self, framed: &mut Framed<C::Stream>) -> Result<(), TransportError> {
let now = Instant::now();
while let Some((deadline, id)) = self.deadlines.pop_front() {
if deadline.is_none_or(|deadline| deadline > now) {
self.deadlines.push_front((deadline, id));
break;
}
if let Some(outbound) = self.pending.remove(&id) {
let error = outbound.interrupted(Interruption::Timeout(self.options.timeout));
let _ = outbound.reply.send(Err(error));
}
}
if self.silent_at.is_some_and(|silent_at| now >= silent_at) {
let silence = self.options.silence();
let late = timeout(Duration::ZERO, framed.receive())
.await
.unwrap_or(Err(TransportError::Idle(silence)));
self.heard(late?);
}
if self
.heartbeat_at
.is_some_and(|heartbeat_at| now >= heartbeat_at)
{
let beat = ProtoHeartbeatEvent::default().to_message(None);
write(framed, &beat, self.options.timeout).await?;
bump(&self.counters.heartbeats_sent);
self.heartbeat_at = within(Instant::now(), self.options.heartbeat_interval);
}
Ok(())
}
fn dispatch(&mut self, message: ProtoMessage) {
if message.payload_type == ProtoHeartbeatEvent::PAYLOAD_TYPE {
bump(&self.counters.heartbeats_received);
return;
}
let waiting = message
.client_msg_id
.as_ref()
.and_then(|id| self.pending.remove(id));
let Some(mut outbound) = waiting else {
let sessions = sessions_ended_by(&message);
let _ = self.events.send(Event::Message(message));
for session in sessions {
self.end(session);
}
return;
};
self.ended.granted(&message);
if outbound.once {
let _ = self.events.send(Event::Message(message.clone()));
}
let result = outcome(message);
if !outbound.once && over_limit(&result) && outbound.attempts < self.options.retries {
let pause = self
.options
.backoff
.saturating_mul(1 << outbound.attempts.min(16))
.min(BACKOFF_LIMIT.max(self.options.backoff));
let until = after(Instant::now(), pause);
log_debug!(
payload_type = outbound.payload_type,
?pause,
"limit exceeded"
);
self.paused_until = Some(self.paused_until.map_or(until, |known| known.max(until)));
outbound.attempts += 1;
bump(&self.counters.throttle_retries);
self.queue.push_front(outbound);
} else {
let _ = outbound.reply.send(result);
}
}
fn fail_in_flight(&mut self, reason: &TransportError) {
self.deadlines.clear();
for (_, outbound) in self.pending.drain() {
let error = outbound.interrupted(Interruption::Transport(reason.clone()));
let _ = outbound.reply.send(Err(error));
}
for outbound in self.queue.drain(..) {
let _ = outbound.reply.send(Err(reason.clone().into()));
}
self.reset_pacing();
}
fn close_in_flight(&mut self, error: &ConnectorError) {
self.deadlines.clear();
for (_, outbound) in self.pending.drain() {
let _ = outbound.reply.send(Err(error.clone()));
}
for outbound in self.queue.drain(..) {
let _ = outbound.reply.send(Err(error.clone()));
}
self.reset_pacing();
}
fn reset_pacing(&mut self) {
self.paused_until = None;
self.send_at = None;
for throttle in &mut self.throttles {
throttle.sent.clear();
}
}
fn fail_all(&mut self, error: &ConnectorError) {
self.close_in_flight(error);
self.commands.close();
while let Ok(command) = self.commands.try_recv() {
if let Command::Request(outbound) = command {
let _ = outbound.reply.send(Err(error.clone()));
}
}
}
}
#[derive(Default)]
struct Throttle {
sent: VecDeque<Instant>,
}
impl Throttle {
fn ready_at(&mut self, now: Instant, limit: u32) -> Option<Instant> {
while self
.sent
.front()
.is_some_and(|sent| now.duration_since(*sent) >= LIMIT_WINDOW)
{
self.sent.pop_front();
}
let limit = usize::try_from(limit).unwrap_or(usize::MAX);
let excess = (self.sent.len() + 1).checked_sub(limit)?;
let blocking = excess.checked_sub(1)?;
self.sent
.get(blocking)
.map(|sent| after(*sent, LIMIT_WINDOW))
}
}
fn after(start: Instant, period: Duration) -> Instant {
start.checked_add(period.min(LONGEST_WAIT)).unwrap_or(start)
}
fn within(start: Instant, period: Duration) -> Option<Instant> {
if period > LONGEST_WAIT {
return None;
}
start.checked_add(period)
}
async fn reached(wake: Option<Instant>) {
match wake {
Some(wake) => sleep_until(wake).await,
None => pending().await,
}
}
fn over_limit(result: &Result<ProtoMessage, ConnectorError>) -> bool {
matches!(
result,
Err(ConnectorError::Server(error)) if matches!(
error.code,
ErrorCode::RequestFrequencyExceeded | ErrorCode::BlockedPayloadType
)
)
}
fn store(replays: &mut Replays, key: ReplayKey, stored: Option<Stored>) {
match stored {
Some(stored) => {
replays.insert(key, stored);
}
None => {
replays.remove(&key);
}
}
}
async fn write<S: AsyncRead + AsyncWrite + Unpin>(
framed: &mut Framed<S>,
message: &ProtoMessage,
limit: Duration,
) -> Result<(), TransportError> {
timeout(limit, framed.send(message))
.await
.unwrap_or(Err(TransportError::WriteTimeout(limit)))
}
async fn close<S: AsyncRead + AsyncWrite + Unpin>(framed: &mut Framed<S>, limit: Duration) {
let _ = timeout(limit, framed.close()).await;
}
async fn hold<F: Future>(
commands: &mut mpsc::Receiver<Command>,
replays: &mut Replays,
reason: &TransportError,
work: F,
) -> Result<F::Output, ConnectorError> {
tokio::pin!(work);
loop {
tokio::select! {
output = &mut work => return Ok(output),
command = commands.recv() => match command {
Some(Command::Request(outbound)) => {
let _ = outbound.reply.send(Err(reason.clone().into()));
}
Some(Command::Replay(key, stored)) => store(replays, key, stored),
Some(Command::Reconnect) => {}
Some(Command::Shutdown) | None => return Err(ConnectorError::Closed),
},
}
}
}
async fn replay<S: AsyncRead + AsyncWrite + Unpin>(
framed: &mut Framed<S>,
id: String,
stored: Stored,
limit: Duration,
events: &broadcast::Sender<Event>,
pushed: &mut Vec<SessionEnded>,
) -> Result<Result<ProtoMessage, ConnectorError>, TransportError> {
let message = ProtoMessage {
payload_type: stored.payload_type,
payload: Some(stored.payload),
client_msg_id: Some(id.clone()),
};
write(framed, &message, limit).await?;
Ok(
match timeout(limit, answer_to(framed, &id, events, pushed)).await {
Ok(answer) => outcome(answer?),
Err(_) => Err(ConnectorError::Timeout(limit)),
},
)
}
async fn answer_to<S: AsyncRead + AsyncWrite + Unpin>(
framed: &mut Framed<S>,
id: &str,
events: &broadcast::Sender<Event>,
pushed: &mut Vec<SessionEnded>,
) -> Result<ProtoMessage, TransportError> {
loop {
let message = framed.receive().await?;
if message.client_msg_id.as_deref() == Some(id) {
return Ok(message);
}
if message.payload_type != ProtoHeartbeatEvent::PAYLOAD_TYPE {
pushed.extend(sessions_ended_by(&message));
let _ = events.send(Event::Message(message));
}
}
}
fn outcome(message: ProtoMessage) -> Result<ProtoMessage, ConnectorError> {
if message.payload_type == ProtoOaErrorRes::PAYLOAD_TYPE {
Err(ProtoOaErrorRes::from_message(&message)
.map_or_else(|error| error, |response| ServerError::from(response).into()))
} else if message.payload_type == ProtoErrorRes::PAYLOAD_TYPE {
Err(ProtoErrorRes::from_message(&message)
.map_or_else(|error| error, |response| ServerError::from(response).into()))
} else {
Ok(message)
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use super::{
ConnectionOptions, DEFAULT_INBOUND_TIMEOUT, Event, LIMIT_WINDOW, LONGEST_WAIT,
MAX_COMMAND_CAPACITY, MAX_EVENT_CAPACITY, ReplayKey, ReplayStage,
SERVER_HEARTBEAT_INTERVAL, Throttle, after, outcome, over_limit, within,
};
use crate::common::config::ConfigurationOpenApi;
use crate::common::errors::{ConnectorError, ErrorCode, TransportError};
use crate::open_api::messages::{
ProtoErrorRes, ProtoMessage, ProtoOaErrorRes, ProtoOaSpotEvent, ProtoOaVersionRes,
};
use crate::open_api::payload::Payload;
use tokio::time::Instant;
#[test]
fn options_take_timing_and_retries_of_the_configuration() {
let configuration = ConfigurationOpenApi::builder()
.client_id("id")
.client_secret("secret")
.access_token("token")
.timeout(Duration::from_secs(3))
.reconnect_delay(Duration::from_secs(2))
.retries(7)
.backoff(Duration::from_millis(250))
.build()
.expect("configuration");
let options = ConnectionOptions::from(&configuration);
assert_eq!(
options,
ConnectionOptions::new()
.timeout(Duration::from_secs(3))
.reconnect_delay(Duration::from_secs(2))
.retries(7)
.backoff(Duration::from_millis(250))
);
}
#[test]
fn silence_allowed_follows_the_pace_of_the_server_unless_it_is_set() {
assert_eq!(DEFAULT_INBOUND_TIMEOUT, Duration::from_secs(75));
assert!(DEFAULT_INBOUND_TIMEOUT >= SERVER_HEARTBEAT_INTERVAL * 2);
assert_eq!(ConnectionOptions::new().silence(), DEFAULT_INBOUND_TIMEOUT);
let quick = ConnectionOptions::new()
.heartbeat_interval(Duration::from_secs(2))
.timeout(Duration::from_secs(1));
assert_eq!(quick.silence(), DEFAULT_INBOUND_TIMEOUT);
assert_eq!(
quick.clone().inbound_timeout(Duration::ZERO).silence(),
Duration::from_millis(1)
);
assert_eq!(
quick.inbound_timeout(Duration::from_secs(90)).silence(),
Duration::from_secs(90)
);
}
#[test]
fn options_raise_degenerate_values() {
let options = ConnectionOptions::new()
.timeout(Duration::ZERO)
.heartbeat_interval(Duration::ZERO)
.command_capacity(0)
.event_capacity(0)
.reconnect_delay(Duration::ZERO)
.reconnect_attempts(Some(4))
.request_limit(0)
.historical_limit(0)
.retries(0)
.backoff(Duration::ZERO);
assert_eq!(options.timeout, Duration::from_millis(1));
assert_eq!(options.heartbeat_interval, Duration::from_millis(1));
assert_eq!(options.command_capacity, 1);
assert_eq!(options.event_capacity, 1);
let largest = ConnectionOptions::new()
.command_capacity(usize::MAX)
.event_capacity(usize::MAX);
assert_eq!(largest.command_capacity, MAX_COMMAND_CAPACITY);
assert_eq!(largest.event_capacity, MAX_EVENT_CAPACITY);
assert_eq!(options.reconnect_delay, Duration::from_millis(1));
assert_eq!(options.reconnect_attempts, Some(4));
assert_eq!(options.request_limit, 1);
assert_eq!(options.historical_limit, 1);
assert_eq!(options.retries, 0);
assert_eq!(options.backoff, Duration::from_millis(1));
assert_eq!(ConnectionOptions::new().reconnect_attempts, None);
}
#[test]
fn error_responses_become_server_errors() {
let open_api = ProtoOaErrorRes {
error_code: "CH_CLIENT_AUTH_FAILURE".to_owned(),
..ProtoOaErrorRes::default()
};
assert!(matches!(
outcome(open_api.to_message(None)),
Err(ConnectorError::Server(error)) if error.code == ErrorCode::ChClientAuthFailure
));
let common = ProtoErrorRes {
error_code: "UNSUPPORTED_MESSAGE".to_owned(),
..ProtoErrorRes::default()
};
assert!(matches!(
outcome(common.to_message(None)),
Err(ConnectorError::Server(error)) if error.code == ErrorCode::UnsupportedMessage
));
for payload_type in [50, 2142] {
let malformed = ProtoMessage {
payload_type,
payload: Some(vec![0xff]),
client_msg_id: None,
};
assert!(matches!(
outcome(malformed),
Err(ConnectorError::Transport(TransportError::Decode(_)))
));
}
let fine = ProtoOaVersionRes::default().to_message(None);
assert_eq!(outcome(fine.clone()), Ok(fine));
}
#[test]
fn event_decodes_only_the_message_it_carries() {
let spot = ProtoOaSpotEvent {
ctid_trader_account_id: 1,
symbol_id: 2,
..ProtoOaSpotEvent::default()
};
let event = Event::Message(spot.to_message(None));
assert_eq!(event.decode::<ProtoOaSpotEvent>(), Some(spot));
assert_eq!(event.decode::<ProtoOaVersionRes>(), None);
let lost = Event::Disconnected(TransportError::Closed);
assert_eq!(lost.decode::<ProtoOaSpotEvent>(), None);
let again = Event::Reconnected { attempt: 1 };
assert_eq!(again.decode::<ProtoOaSpotEvent>(), None);
}
#[test]
fn replay_keys_order_by_stage_then_name() {
let mut keys = [
ReplayKey::new(ReplayStage::Subscription, "a"),
ReplayKey::new(ReplayStage::Account, "2"),
ReplayKey::new(ReplayStage::Application, "z"),
ReplayKey::new(ReplayStage::Account, "1"),
];
keys.sort();
let order: Vec<(ReplayStage, &str)> =
keys.iter().map(|key| (key.stage(), key.id())).collect();
assert_eq!(
order,
vec![
(ReplayStage::Application, "z"),
(ReplayStage::Account, "1"),
(ReplayStage::Account, "2"),
(ReplayStage::Subscription, "a"),
]
);
}
#[test]
fn throttle_names_the_moment_the_next_request_may_go() {
let start = Instant::now();
let mut throttle = Throttle::default();
assert_eq!(throttle.ready_at(start, 2), None);
throttle.sent.push_back(start);
assert_eq!(throttle.ready_at(start, 2), None);
throttle.sent.push_back(start + Duration::from_millis(10));
assert_eq!(throttle.ready_at(start, 2), Some(start + LIMIT_WINDOW));
throttle.sent.push_back(start + Duration::from_millis(20));
assert_eq!(
throttle.ready_at(start, 2),
Some(start + Duration::from_millis(10) + LIMIT_WINDOW)
);
let later = start + LIMIT_WINDOW + Duration::from_millis(10);
assert_eq!(throttle.ready_at(later, 2), None);
assert_eq!(throttle.sent.len(), 1);
}
#[test]
fn waits_too_long_for_the_clock_end_at_the_longest_wait() {
let start = Instant::now();
let brief = Duration::from_secs(90);
assert_eq!(after(start, brief), start + brief);
assert_eq!(after(start, LONGEST_WAIT), start + LONGEST_WAIT);
assert_eq!(after(start, Duration::MAX), start + LONGEST_WAIT);
let far = after(start, Duration::MAX);
assert_eq!(after(far, Duration::MAX), far + LONGEST_WAIT);
assert_eq!(within(start, brief), Some(start + brief));
assert_eq!(within(start, LONGEST_WAIT), Some(start + LONGEST_WAIT));
assert_eq!(within(start, LONGEST_WAIT + Duration::from_nanos(1)), None);
assert_eq!(within(start, Duration::MAX), None);
let options = ConnectionOptions::new()
.timeout(Duration::MAX)
.heartbeat_interval(Duration::MAX);
assert_eq!(options.silence(), DEFAULT_INBOUND_TIMEOUT);
assert_eq!(
options.inbound_timeout(Duration::MAX).silence(),
Duration::MAX
);
let mut throttle = Throttle::default();
throttle.sent.push_back(far);
assert_eq!(throttle.ready_at(far, 1), Some(far + LIMIT_WINDOW));
}
#[test]
fn only_limit_errors_are_retried() {
let refusal = |code: &str| {
outcome(
ProtoOaErrorRes {
error_code: code.to_owned(),
..ProtoOaErrorRes::default()
}
.to_message(None),
)
};
assert!(over_limit(&refusal("REQUEST_FREQUENCY_EXCEEDED")));
assert!(over_limit(&refusal("BLOCKED_PAYLOAD_TYPE")));
assert!(!over_limit(&refusal("INVALID_REQUEST")));
assert!(!over_limit(&Ok(
ProtoOaVersionRes::default().to_message(None)
)));
assert!(!over_limit(&Err(ConnectorError::Closed)));
}
}